diff --git a/docs/docs/00200-core-concepts/00500-authentication/00200-Auth0.md b/docs/docs/00200-core-concepts/00500-authentication/00200-Auth0.md
new file mode 100644
index 00000000000..81737fb1ac4
--- /dev/null
+++ b/docs/docs/00200-core-concepts/00500-authentication/00200-Auth0.md
@@ -0,0 +1,267 @@
+---
+title: Auth0
+---
+
+import { StepByStep, Step, StepText, StepCode } from "@site/src/components/Steps";
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+This guilde will walk you through integrating Auth0 authentication with your SpacetimeDB application.
+
+## Prerequisites
+
+We assume you have the following prerequisites in place:
+
+- A working SpacetimeDB project, follow our [React Quickstart Guide](../../00100-intro/00200-quickstarts/00100-react.md) if you need help setting this up.
+
+## Getting started
+
+
+
+
+ Install the Auth0 React SDK into your React application.
+
+
+
+
+```bash
+npm add @auth0/auth0-react
+```
+
+
+```bash
+yarn add @auth0/auth0-react
+```
+
+
+```bash
+pnpm add @auth0/auth0-react
+```
+
+
+```bash
+bun add @auth0/auth0-react
+```
+
+
+
+
+
+
+
+1. Head to the [Auth0 Dashboard](https://manage.auth0.com/dashboard/)
+2. Click on **Applications** > **Applications** > **Create Application**
+3. In the popup, enter a name for your app, select `Single Page Web Application`
+ as the app type and click Create
+4. Switch to the **Settings** tab on the **Application Details** page
+5. Save the Domain and Client ID values from the dashboard somewhere handy, you'll
+ need them later
+6. Finally, on the Settings tab of your Application Details page, configure the
+ URLs from the table on the right
+
+
+
+
+
+| URL Type | URL |
+| --------------------- | ----------------------- |
+| Allowed Callback URLs | `http://localhost:5173` |
+| Allowed Logout URLs | `http://localhost:5173` |
+| Allowed Web Origins | `http://localhost:5173` |
+
+
+
+
+
+
+
+Create an `AutoLogin` component that automatically handles user login and ID token
+retrieval using the Auth0 React SDK. This component will redirect unauthenticated
+users to the Auth0 login page and provide the ID token to its children via context.
+
+
+
+```tsx
+import { useAuth0 } from '@auth0/auth0-react';
+import { createContext, useContext, useEffect, useMemo, useState } from 'react';
+
+const IdTokenContext = createContext(undefined);
+
+export function useIdToken() {
+ const ctx = useContext(IdTokenContext);
+ if (!ctx) {
+ throw new Error('useIdToken must be used within an IdTokenProvider');
+ }
+ return ctx;
+}
+
+export function AutoLogin({ children }: { children: React.ReactNode }) {
+ const { isLoading, isAuthenticated, loginWithRedirect, getIdTokenClaims } =
+ useAuth0();
+
+ const [idToken, setIdToken] = useState(null);
+ const [error, setError] = useState(null);
+
+ useEffect(() => {
+ if (!isLoading && !isAuthenticated) {
+ loginWithRedirect().catch(err => setError(err));
+ }
+ }, [isLoading, isAuthenticated, loginWithRedirect]);
+
+ useEffect(() => {
+ let cancelled = false;
+
+ const run = async () => {
+ if (isLoading) return;
+
+ // IMPORTANT: If not authenticated, ensure token is cleared
+ if (!isAuthenticated) {
+ if (!cancelled) setIdToken(null);
+ return;
+ }
+
+ try {
+ const claims = await getIdTokenClaims();
+ const token = claims?.__raw ?? null;
+
+ if (!token) {
+ throw new Error('Auth0 returned no ID token (__raw missing).');
+ }
+
+ if (!cancelled) {
+ setIdToken(token);
+ }
+ } catch (e) {
+ if (!cancelled) setError(e as Error);
+ }
+ };
+
+ run();
+ return () => {
+ cancelled = true;
+ };
+ }, [isLoading, isAuthenticated, getIdTokenClaims]);
+
+ const value = useMemo(() => {
+ return idToken ?? undefined;
+ }, [idToken]);
+
+ const ready = !isLoading && isAuthenticated && !!idToken && !error;
+
+ if (error) {
+ return (
+
+
Authentication error
+
{error.message}
+
+ );
+ }
+
+ if (!ready) {
+ return
Loading...
;
+ }
+
+ return (
+ {children}
+ );
+}
+```
+
+
+
+
+
+
+Wrap your application with the `Auth0Provider` component to `main.tsx` to enable
+authentication.
+
+:::info
+Don't forget to remove any existing `SpacetimeDBProvider` wrapper from this
+file as we will be adding it somewhere else later in this guide.
+:::
+
+
+
+```tsx
+import { Auth0Provider } from '@auth0/auth0-react';
+import { StrictMode } from 'react';
+import { createRoot } from 'react-dom/client';
+import App from './App.tsx';
+import { AutoLogin } from './AutoLogin.tsx';
+
+createRoot(document.getElementById('root')!).render(
+
+
+
+
+
+
+
+);
+```
+
+
+
+
+
+Update your `App.tsx` file to use the `useIdToken` hook from the `AutoLogin`
+component to get the ID token and pass it to the `DbConnection` builder.
+
+
+
+```tsx
+import { useMemo } from 'react';
+import { Identity } from 'spacetimedb';
+import { SpacetimeDBProvider } from 'spacetimedb/react';
+import { useIdToken } from './AutoLogin';
+import { DbConnection, ErrorContext } from './module_bindings';
+
+const onConnect = (_conn: DbConnection, identity: Identity) => {
+ console.log(
+ 'Connected to SpacetimeDB with identity:',
+ identity.toHexString()
+ );
+};
+
+const onDisconnect = () => {
+ console.log('Disconnected from SpacetimeDB');
+};
+
+const onConnectError = (_ctx: ErrorContext, err: Error) => {
+ console.log('Error connecting to SpacetimeDB:', err);
+};
+
+export default function App() {
+ const idToken = useIdToken();
+
+ const connectionBuilder = useMemo(() => {
+ return DbConnection.builder()
+ .withUri('')
+ .withModuleName('')
+ .withToken(idToken)
+ .onConnect(onConnect)
+ .onDisconnect(onDisconnect)
+ .onConnectError(onConnectError);
+ }, [idToken]);
+
+ return (
+
+
+
SpacetimeDB React App
+
You can now use SpacetimeDB in your app!
+
+
+ );
+}
+```
+
+
+
+
+
diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md
index ee0e02bf837..6a31c04897d 100644
--- a/docs/llms/docs-benchmark-analysis.md
+++ b/docs/llms/docs-benchmark-analysis.md
@@ -1,80 +1,99 @@
# Benchmark Failure Analysis
-Generated from: `C:\Users\Tyler\Developer\SpacetimeDB\tools\xtask-llm-benchmark\../../docs/llms/docs-benchmark-details.json`
+Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../docs/llms/docs-benchmark-details.json`
## Summary
-- **Total failures analyzed**: 26
+- **Total failures analyzed**: 24
## Analysis
-# SpacetimeDB Benchmark Test Failures Analysis
+# Analysis of SpacetimeDB Benchmark Failures
## Rust Failures
### 1. Root Causes
-- **Compile/Publish Errors (3 failures)**:
- - The primary issue across the failures is related to the use of `ScheduleAt::every_micros` versus `ScheduleAt::RepeatMicros`, which indicates a lack of clarity in the documentation about the correct method of using scheduled types.
- - Another issue is the incorrect implementation of `pub` for some fields and missing `#[derive(SpacetimeType)]` for structs, which has led to schema mismatches.
+1. **Missing Primary Keys and Public Modifiers**:
+ - Several tests (e.g., `t_005_update`, `t_006_delete`, `t_007_crud`, etc.) fail due to missing access modifiers on the struct fields. Notably, primary keys should have `pub` to be accessible.
+
+2. **Schedule At Usage**:
+ - In scheduled tests like `t_002_scheduled_table` and `t_017_scheduled_columns`, the documentation does not clearly explain how to properly set up scheduled reducers and columns.
-- **Other Failures (1 failure)**:
- - The test `t_003_struct_in_table` has a mismatch where the expected reducer setup differs from what's provided. This highlights insufficient documentation around initial setup requirements for reducers.
+3. **Table Definitions**:
+ - Errors in identifying the tables may suggest that the documentation lacks details on how to ensure tables are created or seeded correctly before executing tests.
+
+4. **General Error Handling**:
+ - Many errors include warnings about instability, indicating the documentation hasn’t adequately prepared users for expected limitations or how to work around them.
### 2. Recommendations
-- **Documentation Updates**:
- - **Scheduled Types Documentation**: Enhance the section on scheduled types in the documentation to clarify the use of `ScheduleAt::every_micros` and `ScheduleAt::RepeatMicros`. Example for addition:
- ```markdown
- ### Scheduled Types
- - Use `ScheduleAt::every_micros(interval)` for non-repeating intervals.
- - Use `ScheduleAt::RepeatMicros(interval)` for repeating intervals. Ensure proper usage to avoid publishing errors.
- ```
-
- - **Section on Structs and Reducers**: Update the section dealing with struct fields to illustrate the necessity of using `pub` where it applies and clarifying how reducers must align:
- ```markdown
- ### Struct Definitions
- - Struct fields must be marked as `pub` to ensure they are accessible within the SpacetimeDB context.
- - Reducers must be defined properly; ensure that each reducer matches expected configurations in your schemas.
+#### a. Update Documentation for Struct Fields
+- **Section**: Struct Field Modifiers
+- **Change**: Reinforce that all fields in structs representing tables should be declared as `pub` (public).
+ ```rust
+ #[primary_key]
+ pub id: i32,
```
-- **Example Code Alignment**: Revise example code throughout documentation to align with the latest syntax and ensure that all required attributes are included.
+#### b. Clarify Schedules and Reducers
+- **Section**: Scheduling and Reducers
+- **Change**: Provide specific examples that detail correct usage of scheduled reducers and column definitions.
+ ```rust
+ #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]
+ ```
+
+#### c. Table Creation and Seeding
+- **Section**: Database Setup
+- **Change**: Include a walkthrough for initializing and seeding tables prior to executing tests.
+
+#### d. Error Handling in Tests
+- **Section**: Error Messages
+- **Change**: Update the documentation to clarify the potential implications of unstable commands and how to handle them, including fallbacks or alternative methods.
### 3. Priority
-- **High Impact Fixes**:
- 1. Scheduled Types Documentation (to prevent compile errors).
- 2. Structs and Reducers Section (to ensure schema and function alignment).
+- **High**: Improvements on struct field access modifiers and scheduling examples.
+- **Medium**: Recommendations on table setup and seeding.
+- **Low**: Enhancements to error handling and stability notes.
---
## C# Failures
### 1. Root Causes
-- **Table Naming Issues (19 failures)**:
- - The primary issue causing the failures is the inconsistency in the use of table names (e.g., `entities` vs. `Entity`). Lack of clear guidelines on naming conventions has led to widespread discrepancies.
+1. **Missing Public Modifiers**:
+ - Similar to Rust, many C# errors arise from the lack of `public` modifiers for struct fields, which can affect accessibility (e.g., `t_004_insert`, `t_006_delete`, etc.).
-- **Timeout Issues (3 failures)**:
- - Additionally, the timeout failures indicate that certain processes aren’t being documented well in terms of what expectations exist for execution time and potential pitfalls leading to these issues.
+2. **Table Name Consistency Issues**:
+ - Documented table names must match the expected names in the declarations to avoid runtime errors regarding nonexistent tables.
+
+3. **Redundant Modifiers**:
+ - There are inconsistencies where the `Public = true` attribute is unnecessary in certain contexts, leading to confusion.
+
+4. **Unstable Command Warnings**:
+ - Like Rust, frequent unstable command warnings highlight the need for better communication regarding command limitations.
### 2. Recommendations
-- **Documentation Updates**:
- - **Table Naming Conventions**: Introduce a comprehensive section specifying the naming conventions for tables. Example for addition:
- ```markdown
- ### Table Naming Conventions
- - Table names should be singular and PascalCase (e.g., `User` instead of `users`).
- - Ensure that when creating and querying tables, the names are consistently used to avoid schema parity issues.
+#### a. Overview of Struct Fields
+- **Section**: Struct Field Modifiers
+- **Change**: Emphasize that all fields must be marked as `public` to ensure accessibility within the library.
+ ```csharp
+ [PrimaryKey] public int Id;
```
+
+#### b. Consistent Table Naming
+- **Section**: Table Naming Conventions
+- **Change**: Outline naming conventions clearly to ensure consistency between struct definitions and database references.
- - **Timeout Handling Guidance**: Provide clearer information on how to handle potential timeout issues within operations:
- ```markdown
- ### Handling Timeouts
- - If encountering timeout errors during transactions, consider optimizing the initial data load or query processes.
- - Implement logging to help identify which part of your transaction is leading to the timeout.
- ```
+#### c. Clean Up Redundant Modifiers
+- **Section**: Table Attribute Usage
+- **Change**: Simplify the examples that use `Public = true`, focusing on when it is truly necessary.
+
+#### d. Addressing Unstable Command Handling
+- **Section**: Managing Instabilities
+- **Change**: Provide guidance on how to manage and respond to warnings during command execution.
### 3. Priority
-- **High Impact Fixes**:
- 1. Table Naming Conventions (most immediate to fix widespread errors).
- 2. Timeout Handling Guidance (to improve performance and reliability in operations).
-
----
+- **High**: Adjustments on struct field access and consistent naming conventions.
+- **Medium**: Cleanup on redundant modifiers and reasserting proper access control.
+- **Low**: Instructions on managing unstable commands.
-This structured approach will help improve the accessibility and clarity of the SpacetimeDB documentation, directly addressing the root causes of current benchmark test failures.
+With these changes, we expect a decrease in benchmark test failures and enhanced reliability in user implementations.
diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md
index 00356a7efde..4ffc84b4e9c 100644
--- a/docs/llms/docs-benchmark-comment.md
+++ b/docs/llms/docs-benchmark-comment.md
@@ -2,13 +2,13 @@
| Language | Mode | Category | Tests Passed | Task Pass % |
|----------|------|----------|--------------|-------------|
-| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬇️ -8.3% |
-| Rust | rustdoc_json | schema | 29/34 | 80.0% |
-| Rust | rustdoc_json | **total** | 54/61 | **81.8%** ⬇️ -4.5% |
-| C# | docs | basics | 7/27 | 16.0% ⬇️ -84.0% |
-| C# | docs | schema | 18/34 | 48.5% ⬇️ -31.5% |
-| C# | docs | **total** | 25/61 | **30.8%** ⬇️ -60.2% |
+| Rust | rustdoc_json | basics | 17/25 | 67.4% ⬇️ -16.0% |
+| Rust | rustdoc_json | schema | 32/34 | 90.0% ⬆️ +10.0% |
+| Rust | rustdoc_json | **total** | 49/59 | **77.7%** ⬇️ -4.2% |
+| C# | docs | basics | 11/25 | 58.3% ⬇️ -2.8% |
+| C# | docs | schema | 10/34 | 31.5% |
+| C# | docs | **total** | 21/59 | **46.1%** ⬇️ -1.5% |
_Compared against master branch baseline_
-Generated at: 2026-01-16T19:34:55.608Z
+Generated at: 2026-01-20T11:45:02.790Z
diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json
index e7d25c4bce3..8b21dbe308c 100644
--- a/docs/llms/docs-benchmark-details.json
+++ b/docs/llms/docs-benchmark-details.json
@@ -19,7 +19,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\nfn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n let _ = count;\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n let _ = name;\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n let _ = (count, name);\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n let _ = (active, ratio, label);\n}",
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-000-empty-reducers-golden",
@@ -35,15 +35,15 @@
"llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:16.167437181Z",
- "finished_at": "2026-01-17T02:30:49.255347669Z"
+ "started_at": "2026-01-20T11:37:28.499651182Z",
+ "finished_at": "2026-01-20T11:38:00.018940431Z"
},
"t_001_basic_tables": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -53,7 +53,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = products)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = notes)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
+ "llm_output": "#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = products)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = notes)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-001-basic-tables-golden",
@@ -69,15 +69,15 @@
"llm_db": "basics-t-001-basic-tables-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:52.376540604Z",
- "finished_at": "2026-01-17T02:32:37.875429994Z"
+ "started_at": "2026-01-20T11:37:54.816456207Z",
+ "finished_at": "2026-01-20T11:38:36.428942445Z"
},
"t_002_scheduled_table": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -87,7 +87,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::from_now_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(ctx: &ReducerContext, scheduled_id: u64) {\n if let Some(mut row) = ctx.db.tick_timer().scheduled_id().find(scheduled_id) {\n row.scheduled_at = ScheduleAt::from_now_micros(50_000);\n let _ = ctx.db.tick_timer().scheduled_id().update(row);\n }\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) {\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-002-scheduled-table-golden",
@@ -99,14 +99,14 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.8\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.53\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `from_now_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::from_now_micros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:24:35\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m24\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if let Some(mut row) = ctx.db.tick_timer().scheduled_id().find(scheduled_id) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `from_now_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:25:40\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m25\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m row.scheduled_at = ScheduleAt::from_now_micros(50_000);\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:26:24\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m26\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m let _ = ctx.db.tick_timer().scheduled_id().update(row);\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0422`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.105\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.43\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling zmij v1.0.16\n Compiling itoa v1.0.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling bytemuck v1.24.0\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling cc v1.2.53\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:42.103317349Z",
- "finished_at": "2026-01-17T02:32:41.832312901Z"
+ "started_at": "2026-01-20T11:37:28.500159508Z",
+ "finished_at": "2026-01-20T11:38:37.795185128Z"
},
"t_003_struct_in_table": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -115,8 +115,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
- "passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entities().insert(Entity { id, pos: Position { x, y } });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-003-struct-in-table-golden",
@@ -125,27 +125,22 @@
"work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
- "pass": false,
- "partial": 0.0,
+ "pass": true,
+ "partial": 1.0,
"notes": {
"golden_db": "basics-t-003-struct-in-table-golden",
"llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
- "reducers_diff": {
- "only_golden": [],
- "only_llm": [
- "add_entity()"
- ]
- },
- "reducers_equal": false,
- "server": "http://127.0.0.1:46587",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:49.255385009Z",
- "finished_at": "2026-01-17T02:32:37.661244218Z"
+ "started_at": "2026-01-20T11:37:44.218456731Z",
+ "finished_at": "2026-01-20T11:38:11.539831563Z"
},
"t_004_insert": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -155,7 +150,7 @@
"model_name": "GPT-5",
"total_tests": 2,
"passed_tests": 2,
- "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().insert(User { id, name, age, active });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\nfn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().insert(User { id, name, age, active });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-004-insert-golden",
@@ -179,7 +174,7 @@
"llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
"query": "SELECT id, name, age, active FROM users WHERE id=1",
"reducer": "insert_user",
- "server": "http://127.0.0.1:46587"
+ "server": "http://127.0.0.1:44879"
}
},
"schema_parity": {
@@ -190,15 +185,15 @@
"llm_db": "basics-t-004-insert-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:53.628765951Z",
- "finished_at": "2026-01-17T02:32:37.475748331Z"
+ "started_at": "2026-01-20T11:37:59.598721381Z",
+ "finished_at": "2026-01-20T11:38:23.749988602Z"
},
"t_005_update": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -207,8 +202,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.users().id().update(user);\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.users().id().find(id).is_some() {\n let user = User { id, name, age, active };\n let _ = ctx.db.users().id().update(user);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-005-update-golden",
@@ -216,30 +211,21 @@
"work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm",
"scorer_details": {
- "data_parity_update_user": {
- "pass": true,
- "partial": 1.0,
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [
- 1,
- "Alice2",
- 31,
- false
- ],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-gpt-5-llm",
- "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT id, name, age, active FROM users WHERE id=1",
- "reducer": "update_user",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44879/v1/database/c2002acda4c1b763fdbd6e258508ec915c0ae1db4e0e95ced8de20ed71cbad40/sql)\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
},
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
+ "data_parity_update_user": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:44879/v1/database/c2002acda4c1b763fdbd6e258508ec915c0ae1db4e0e95ced8de20ed71cbad40/call/update_user)\n",
+ "phase": "call_reducer_golden"
}
},
"schema_parity": {
@@ -250,15 +236,15 @@
"llm_db": "basics-t-005-update-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:20.539096519Z",
- "finished_at": "2026-01-17T02:30:53.033151846Z"
+ "started_at": "2026-01-20T11:37:28.499401408Z",
+ "finished_at": "2026-01-20T11:38:04.091138125Z"
},
"t_006_delete": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -267,7 +253,7 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
+ "passed_tests": 2,
"llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(&id);\n}",
"category": "basics",
"route_api_model": "gpt-5",
@@ -276,11 +262,17 @@
"work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm",
"scorer_details": {
- "seed_users_row": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
+ "tables_diff": null,
+ "tables_equal": true
}
},
"delete_user_count_zero": {
@@ -292,23 +284,19 @@
"sql": "SELECT COUNT(*) AS n FROM users WHERE id=1"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44879/v1/database/c200ff1db701577fa1b0dc3777b570e8bcbb279ecd4bb271a1e79dcab01f7cc8/sql)\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.746281172Z",
- "finished_at": "2026-01-17T02:30:42.103247549Z"
+ "started_at": "2026-01-20T11:37:28.499804787Z",
+ "finished_at": "2026-01-20T11:37:54.816322787Z"
},
"t_007_crud": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -317,8 +305,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
- "passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let mut u1 = ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _ = ctx.db.users().id().update(u1);\n ctx.db.users().id().delete(&2);\n}",
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.users();\n\n let mut alice = users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _bob = users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n alice.name = \"Alice2\".to_string();\n alice.age = 31;\n alice.active = false;\n\n let _alice_updated = users.id().update(alice);\n\n users.id().delete(&2);\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-007-crud-golden",
@@ -326,6 +314,15 @@
"work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm",
"scorer_details": {
+ "crud_total_count_one": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
"crud_row_id2_deleted": {
"pass": true,
"partial": 1.0,
@@ -343,38 +340,23 @@
"llm_db": "basics-t-007-crud-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
},
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
"crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-gpt-5-llm",
- "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT id, name, age, active FROM users WHERE id=1",
- "reducer": "crud",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44879/v1/database/c2003aca40f03ddd5eadb189dd4f9e6421eca24702dd4d150a5cf556aa660898/sql)\n",
+ "phase": "sql_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:16.167798252Z",
- "finished_at": "2026-01-17T02:30:53.400172120Z"
+ "started_at": "2026-01-20T11:37:28.500451378Z",
+ "finished_at": "2026-01-20T11:38:02.358850437Z"
},
"t_008_index_lookup": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -383,8 +365,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\nfn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: u.id, name: u.name });\n }\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: user.id, name: user.name });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-008-index-lookup-golden",
@@ -393,25 +375,19 @@
"work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm",
"scorer_details": {
"index_lookup_projection_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "id | name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
- "llm_out": "id | name ----+--------- 1 | \"Alice\"",
- "query": "SELECT id, name FROM results WHERE id=1",
- "reducer": "lookup_user_name",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44879/v1/database/c20049c9666019d1c42d903a4a26f74289d15fba3542503d42b148e5ab480e34/sql)\n",
+ "phase": "sql_golden"
}
},
"seed_user_row": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:44879/v1/database/c20049c9666019d1c42d903a4a26f74289d15fba3542503d42b148e5ab480e34/sql)\n",
+ "phase": "sql_golden",
"sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
},
@@ -423,15 +399,15 @@
"llm_db": "basics-t-008-index-lookup-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.748044021Z",
- "finished_at": "2026-01-17T02:30:54.245888026Z"
+ "started_at": "2026-01-20T11:37:28.500011938Z",
+ "finished_at": "2026-01-20T11:38:04.251101942Z"
},
"t_009_init": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -441,7 +417,7 @@
"model_name": "GPT-5",
"total_tests": 4,
"passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let users = ctx.db.users();\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-009-init-golden",
@@ -449,13 +425,17 @@
"work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm",
"scorer_details": {
- "init_total_two": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
+ "tables_diff": null,
+ "tables_equal": true
}
},
"init_seed_bob": {
@@ -476,23 +456,19 @@
"sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true"
}
},
- "schema_parity": {
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.930019173Z",
- "finished_at": "2026-01-17T02:30:41.209696143Z"
+ "started_at": "2026-01-20T11:37:28.500301711Z",
+ "finished_at": "2026-01-20T11:37:59.598680415Z"
},
"t_010_connect": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -502,7 +478,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-010-connect-golden",
@@ -518,76 +494,44 @@
"llm_db": "basics-t-010-connect-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:53.626651147Z",
- "finished_at": "2026-01-17T02:32:36.216348757Z"
+ "started_at": "2026-01-20T11:38:00.018971459Z",
+ "finished_at": "2026-01-20T11:38:37.795118005Z"
},
"t_011_helper_function": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
"task": "t_011_helper_function",
"lang": "rust",
- "golden_published": true,
+ "golden_published": false,
"model_name": "GPT-5",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.results().insert(ResultRow { id, sum });\n}",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
"scorer_details": {
- "helper_func_sum_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "id | sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-gpt-5-llm",
- "llm_out": "id | sum ----+----- 1 | 5",
- "query": "SELECT id, sum FROM results WHERE id=1",
- "reducer": "compute_sum",
- "server": "http://127.0.0.1:46587"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1 AND sum=5"
+ "error": "POST https://api.openai.com/v1/responses -> 502 Bad Gateway: \r\n502 Bad Gateway\r\n\r\n
502 Bad Gateway
\r\n
cloudflare
\r\n\r\n\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:42.105283707Z",
- "finished_at": "2026-01-17T02:31:52.745496413Z"
+ "started_at": "2026-01-20T11:39:18.169116330Z",
+ "finished_at": "2026-01-20T11:39:18.169116330Z"
},
"t_012_spacetime_product_type": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -596,8 +540,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow { id, value: Score { left, right } });\n}",
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-012-spacetime-product-type-golden",
@@ -605,21 +549,22 @@
"work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm",
"scorer_details": {
- "product_type_row_count": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1"
- }
- },
- "product_type_row_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c20064919d5938e3c53955f56eff54581cc88db76e6afe6cedff286135db8edb/sql)\n",
- "phase": "sql_golden"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)",
+ "query": "SELECT id, value FROM results WHERE id=1",
+ "reducer": "set_score",
+ "server": "http://127.0.0.1:44879"
}
},
"schema_parity": {
@@ -630,15 +575,24 @@
"llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "product_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992696821Z",
- "finished_at": "2026-01-17T02:28:16.167373502Z"
+ "started_at": "2026-01-20T11:38:39.232679507Z",
+ "finished_at": "2026-01-20T11:39:04.546485640Z"
},
"t_013_spacetime_sum_type": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -648,7 +602,7 @@
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().id().delete(&id);\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-013-spacetime-sum-type-golden",
@@ -664,7 +618,7 @@
"llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
@@ -692,13 +646,13 @@
"llm_out": "id | value ----+--------------- 1 | (Circle = 10)",
"query": "SELECT id, value FROM results WHERE id=1",
"reducer": "set_circle",
- "server": "http://127.0.0.1:46587"
+ "server": "http://127.0.0.1:44879"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:12.198644503Z",
- "finished_at": "2026-01-17T02:30:53.626583041Z"
+ "started_at": "2026-01-20T11:38:11.539865429Z",
+ "finished_at": "2026-01-20T11:38:45.519136534Z"
},
"t_014_elementary_columns": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -708,7 +662,7 @@
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-014-elementary-columns-golden",
@@ -725,37 +679,37 @@
"sql": "SELECT COUNT(*) AS n FROM primitives WHERE id=1"
}
},
- "schema_parity": {
+ "elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
+ "args": [],
"golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
"llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT id, count, total, price, ratio, active, name FROM primitives WHERE id=1",
+ "reducer": "seed",
+ "server": "http://127.0.0.1:44879"
}
},
- "elementary_columns_row_parity": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
"golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
"llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
- "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT id, count, total, price, ratio, active, name FROM primitives WHERE id=1",
- "reducer": "seed",
- "server": "http://127.0.0.1:46587"
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992406952Z",
- "finished_at": "2026-01-17T02:28:06.754249465Z"
+ "started_at": "2026-01-20T11:38:36.429061442Z",
+ "finished_at": "2026-01-20T11:39:11.867582553Z"
},
"t_015_product_type_columns": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -764,8 +718,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n ctx.db.profiles().insert(Profile { id: 1, home, work, pos });\n}",
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let table = ctx.db.profiles();\n for row in table.iter() {\n table.id().delete(&row.id);\n }\n table.insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-015-product-type-columns-golden",
@@ -782,6 +736,20 @@
"sql": "SELECT COUNT(*) AS n FROM profiles WHERE id=1"
}
},
+ "product_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
+ "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)",
+ "query": "SELECT id, home, work, pos FROM profiles WHERE id=1",
+ "reducer": "seed",
+ "server": "http://127.0.0.1:44879"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -790,23 +758,15 @@
"llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
- },
- "product_type_columns_row_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c200f79f71d0434e7e09b201e09674fe9fc25e018d5ba8eae68ccedefeafa4b2/sql)\n",
- "phase": "sql_golden"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.993007169Z",
- "finished_at": "2026-01-17T02:28:12.198565927Z"
+ "started_at": "2026-01-20T11:38:02.358890164Z",
+ "finished_at": "2026-01-20T11:38:41.468835390Z"
},
"t_016_sum_type_columns": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -816,7 +776,7 @@
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawings();\n if drawings.count() == 0 {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawings();\n if drawings.id().find(1).is_none() {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-016-sum-type-columns-golden",
@@ -824,6 +784,15 @@
"work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm",
"scorer_details": {
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE id=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -832,7 +801,7 @@
"llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
@@ -848,22 +817,13 @@
"llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))",
"query": "SELECT id, a, b FROM drawings WHERE id=1",
"reducer": "seed",
- "server": "http://127.0.0.1:46587"
- }
- },
- "sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE id=1"
+ "server": "http://127.0.0.1:44879"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:06.754312132Z",
- "finished_at": "2026-01-17T02:30:52.376483288Z"
+ "started_at": "2026-01-20T11:38:37.795562384Z",
+ "finished_at": "2026-01-20T11:39:07.364526387Z"
},
"t_017_scheduled_columns": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -873,7 +833,7 @@
"model_name": "GPT-5",
"total_tests": 2,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n // Scheduled reducer invoked with the scheduled row.\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-017-scheduled-columns-golden",
@@ -885,14 +845,14 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.8\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.53\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0412]\u001b[0m\u001b[0m\u001b[1m: cannot find type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:42\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::RepeatMicros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `register_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m746\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `register_reducer`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `invoke_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m44\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn invoke_reducer<'a, A: Args<'a>>(\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--------------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound in this function\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m45\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m reducer: impl Reducer<'a, A>,\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `invoke_reducer`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0277, E0412, E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0277`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.105\n Compiling quote v1.0.43\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling zmij v1.0.16\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.53\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:32\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:12\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:35\n |\n20 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991765459Z",
- "finished_at": "2026-01-17T02:28:17.929994467Z"
+ "started_at": "2026-01-20T11:38:23.750056422Z",
+ "finished_at": "2026-01-20T11:39:05.499267292Z"
},
"t_018_constraints": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -901,8 +861,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = accounts, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.accounts().try_insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n let _ = ctx.db.accounts().try_insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}",
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = accounts, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let accounts = ctx.db.accounts();\n accounts.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n accounts.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-018-constraints-golden",
@@ -910,17 +870,18 @@
"work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
+ "constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
"notes": {
+ "args": [],
"golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
"llm_db": "schema-t-018-constraints-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT id, email, name FROM accounts WHERE id=1",
+ "reducer": "seed",
+ "server": "http://127.0.0.1:44879"
}
},
"constraints_seed_two_rows": {
@@ -932,18 +893,23 @@
"sql": "SELECT COUNT(*) AS n FROM accounts WHERE id=2"
}
},
- "constraints_row_parity_after_seed": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c2007d48b169e0048c4dd9c53c8d47e40c7761a8814064fe65c573b1ce844864/sql)\n",
- "phase": "sql_golden"
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991438731Z",
- "finished_at": "2026-01-17T02:28:15.400501818Z"
+ "started_at": "2026-01-20T11:38:04.251191001Z",
+ "finished_at": "2026-01-20T11:38:39.232633743Z"
},
"t_019_many_to_many": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -953,7 +919,7 @@
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 5,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let users = ctx.db.users();\n users.insert(User { user_id: 1, name: \"Alice\".to_string() });\n users.insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n let groups = ctx.db.groups();\n groups.insert(Group { group_id: 10, title: \"Admin\".to_string() });\n groups.insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n let memberships = ctx.db.memberships();\n memberships.insert(Membership { id: 1, user_id: 1, group_id: 10 });\n memberships.insert(Membership { id: 2, user_id: 1, group_id: 20 });\n memberships.insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.memberships().iter() {\n ctx.db.memberships().id().delete(&row.id);\n }\n for row in ctx.db.users().iter() {\n ctx.db.users().user_id().delete(&row.user_id);\n }\n for row in ctx.db.groups().iter() {\n ctx.db.groups().group_id().delete(&row.group_id);\n }\n\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-019-many-to-many-golden",
@@ -970,6 +936,24 @@
"sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=20"
}
},
+ "m2m_has_1_10": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=10"
+ }
+ },
+ "m2m_has_2_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=2 AND group_id=20"
+ }
+ },
"memberships_three_rows": {
"pass": true,
"partial": 1.0,
@@ -987,33 +971,15 @@
"llm_db": "schema-t-019-many-to-many-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
- },
- "m2m_has_1_10": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=10"
- }
- },
- "m2m_has_2_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=2 AND group_id=20"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.990785156Z",
- "finished_at": "2026-01-17T02:28:17.183262809Z"
+ "started_at": "2026-01-20T11:38:37.795196374Z",
+ "finished_at": "2026-01-20T11:39:18.169070627Z"
},
"t_020_ecs": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -1023,7 +989,7 @@
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 5,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = positions)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.entities().id().find(1).is_none() {\n let _ = ctx.db.entities().insert(Entity { id: 1 });\n }\n match ctx.db.positions().entity_id().find(1) {\n Some(mut p) => {\n p.x = 0;\n p.y = 0;\n let _ = ctx.db.positions().entity_id().update(p);\n }\n None => {\n let _ = ctx.db.positions().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n }\n match ctx.db.velocities().entity_id().find(1) {\n Some(mut v) => {\n v.vx = 1;\n v.vy = 0;\n let _ = ctx.db.velocities().entity_id().update(v);\n }\n None => {\n let _ = ctx.db.velocities().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n }\n\n if ctx.db.entities().id().find(2).is_none() {\n let _ = ctx.db.entities().insert(Entity { id: 2 });\n }\n match ctx.db.positions().entity_id().find(2) {\n Some(mut p) => {\n p.x = 10;\n p.y = 0;\n let _ = ctx.db.positions().entity_id().update(p);\n }\n None => {\n let _ = ctx.db.positions().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n }\n match ctx.db.velocities().entity_id().find(2) {\n Some(mut v) => {\n v.vx = -2;\n v.vy = 3;\n let _ = ctx.db.velocities().entity_id().update(v);\n }\n None => {\n let _ = ctx.db.velocities().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.positions().iter() {\n if let Some(vel) = ctx.db.velocities().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n match ctx.db.next_positions().entity_id().find(pos.entity_id) {\n Some(mut np) => {\n np.x = nx;\n np.y = ny;\n let _ = ctx.db.next_positions().entity_id().update(np);\n }\n None => {\n let _ = ctx.db.next_positions().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n }\n}",
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = positions)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entities().try_insert(Entity { id: 1 });\n let _ = ctx.db.positions().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocities().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.entities().try_insert(Entity { id: 2 });\n let _ = ctx.db.positions().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocities().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.positions().iter() {\n let vel_opt = ctx.db.velocities().entity_id().find(pos.entity_id);\n let (vx, vy) = match vel_opt {\n Some(v) => (v.vx, v.vy),\n None => (0, 0),\n };\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: pos.x + vx,\n y: pos.y + vy,\n };\n match ctx.db.next_positions().entity_id().find(pos.entity_id) {\n Some(_) => {\n ctx.db.next_positions().entity_id().update(next);\n }\n None => {\n ctx.db.next_positions().insert(next);\n }\n }\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-020-ecs-golden",
@@ -1040,6 +1006,19 @@
"sql": "SELECT COUNT(*) AS n FROM positions"
}
},
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:44879",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
"ecs_next_pos_entity1": {
"pass": true,
"partial": 1.0,
@@ -1058,19 +1037,6 @@
"sql": "SELECT COUNT(*) AS n FROM next_positions WHERE entity_id=2 AND x=8 AND y=3"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"ecs_step_next_positions_count": {
"pass": true,
"partial": 1.0,
@@ -1082,8 +1048,8 @@
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991130819Z",
- "finished_at": "2026-01-17T02:28:20.539029094Z"
+ "started_at": "2026-01-20T11:38:04.091179109Z",
+ "finished_at": "2026-01-20T11:38:49.424891409Z"
},
"t_021_multi_column_index": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -1093,7 +1059,7 @@
"model_name": "GPT-5",
"total_tests": 4,
"passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = logs, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.logs();\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = logs, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let logs = ctx.db.logs();\n logs.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n logs.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n logs.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-021-multi-column-index-golden",
@@ -1101,6 +1067,15 @@
"work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm",
"scorer_details": {
+ "mcindex_lookup_u7_d2": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=2"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1109,42 +1084,33 @@
"llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:44879",
"tables_diff": null,
"tables_equal": true
}
},
- "mcindex_lookup_u7_d1": {
+ "mcindex_seed_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=1"
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
}
},
- "mcindex_lookup_u7_d2": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=2"
- }
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=1"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992084803Z",
- "finished_at": "2026-01-17T02:28:17.745571962Z"
+ "started_at": "2026-01-20T11:38:41.468877405Z",
+ "finished_at": "2026-01-20T11:39:06.843028607Z"
}
}
}
@@ -1335,14 +1301,14 @@
"modes": [
{
"mode": "docs",
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"models": [
{
"name": "GPT-5",
"route_api_model": "gpt-5",
"tasks": {
"t_000_empty_reducers": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
@@ -1365,25 +1331,25 @@
"llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:24.282055723Z",
- "finished_at": "2026-01-17T02:40:56.269475914Z"
+ "started_at": "2026-01-20T11:40:38.627840084Z",
+ "finished_at": "2026-01-20T11:41:53.163695681Z"
},
"t_001_basic_tables": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-001-basic-tables-golden",
@@ -1399,25 +1365,25 @@
"llm_db": "basics-t-001-basic-tables-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:28.640515207Z",
- "finished_at": "2026-01-17T02:43:58.187104574Z"
+ "started_at": "2026-01-20T11:42:28.882852074Z",
+ "finished_at": "2026-01-20T11:42:59.200268751Z"
},
"t_002_scheduled_table": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-002-scheduled-table-golden",
@@ -1433,18 +1399,18 @@
"llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:25.159472748Z",
- "finished_at": "2026-01-17T02:43:07.893721504Z"
+ "started_at": "2026-01-20T11:41:53.163765037Z",
+ "finished_at": "2026-01-20T11:42:42.620843109Z"
},
"t_003_struct_in_table": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
@@ -1467,18 +1433,18 @@
"llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:07.893755608Z",
- "finished_at": "2026-01-17T02:43:39.643821282Z"
+ "started_at": "2026-01-20T11:42:26.327003812Z",
+ "finished_at": "2026-01-20T11:42:50.247572423Z"
},
"t_004_insert": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
@@ -1492,93 +1458,71 @@
"llm_db": "basics-t-004-insert-gpt-5-llm",
"work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm",
- "scorer_details": {
- "data_parity_insert_user": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200933266559154a4d1e8adf441af2b283882c3206ed34f5c4ef907d29de6a7/sql)\n",
- "phase": "sql_golden"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "openai",
- "started_at": "2026-01-17T02:43:58.187152683Z",
- "finished_at": "2026-01-17T02:44:26.839511728Z"
- },
- "t_005_update": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
- "task": "t_005_update",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "GPT-5",
- "total_tests": 3,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}",
- "category": "basics",
- "route_api_model": "gpt-5",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm",
- "scorer_details": {
- "data_parity_update_user": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.6.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(User )\n at SpacetimeDB.Internal.TableHandles.User.IdUniqueIndex.Update(User )\n at Module.UpdateUser(ReducerContext , Int32 , String , Int32 , Boolean )\n at ModuleRegistration.UpdateUser.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:35677/v1/database/c200523e08ccb74ff8d13ab66f492b457bb8aeee5e4fdafc5d1fec7431a9b4e9/call/UpdateUser)\n",
- "phase": "call_reducer_golden"
- }
- },
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gpt-5-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
},
- "seed_users_row": {
+ "data_parity_insert_user": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200523e08ccb74ff8d13ab66f492b457bb8aeee5e4fdafc5d1fec7431a9b4e9/sql)\n",
- "phase": "sql_golden",
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200f1fafb4a6308ce93a00b478f97681b9d11f82fd154e514fd396948c20284/sql)\n",
+ "phase": "sql_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T11:42:41.469803194Z",
+ "finished_at": "2026-01-20T11:43:06.726644222Z"
+ },
+ "t_005_update": {
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
+ "task": "t_005_update",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "LLM call timed out",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:19.102421878Z",
- "finished_at": "2026-01-17T02:42:52.237316726Z"
+ "started_at": "2026-01-20T11:45:02.758494766Z",
+ "finished_at": "2026-01-20T11:45:02.758494766Z"
},
"t_006_delete": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-006-delete-golden",
@@ -1586,50 +1530,50 @@
"work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200afd4d21d2937f5acdc2945f293cc58c431009ae9ca9c30defd4affdca498/sql)\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
"delete_user_count_zero": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2004b74d3be00407c2d02282ee08b25e5e8242063995dea613d8fc1467ef09e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200afcd9b0050324ceb0c66947319cc0aa12d6e4b415b3de217e63152a66ea3/sql)\n",
"phase": "sql"
}
},
- "seed_users_row": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200052ec3b2af2555b2766443d690ff70a4d07ffd410be917b433381552b936/sql)\n",
- "phase": "sql_golden",
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:46139",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:56.269507302Z",
- "finished_at": "2026-01-17T02:41:28.991775660Z"
+ "started_at": "2026-01-20T11:40:38.633211180Z",
+ "finished_at": "2026-01-20T11:41:56.372828438Z"
},
"t_007_crud": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user)\n {\n user.Name = \"Alice2\";\n user.Age = 31;\n user.Active = false;\n ctx.Db.User.Id.Update(user);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-007-crud-golden",
@@ -1641,15 +1585,15 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2006b22042e3fba4f36b1b44b80a07da826122ccc12419c53c511830a0c78f9/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200bb4f64fe48d0739436998bbe9df656d8a4e70079f648f9e3fc9b577ba96a/sql)\n",
"phase": "sql_golden"
}
},
- "crud_row_id2_deleted": {
+ "crud_total_count_one": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003c5aaa4b9c829e64c505f35cb7241f299a733d0f5a89da64241c5e052096/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2008f92e44b859735223ad129c2070bec0afa9c6f4e8d8563a3a4722cf315a0/sql)\n",
"phase": "sql"
}
},
@@ -1661,33 +1605,33 @@
"llm_db": "basics-t-007-crud-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
},
- "crud_total_count_one": {
+ "crud_row_id2_deleted": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003c5aaa4b9c829e64c505f35cb7241f299a733d0f5a89da64241c5e052096/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2008f92e44b859735223ad129c2070bec0afa9c6f4e8d8563a3a4722cf315a0/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:51.462439194Z",
- "finished_at": "2026-01-17T02:41:35.805469514Z"
+ "started_at": "2026-01-20T11:41:56.563021891Z",
+ "finished_at": "2026-01-20T11:42:28.882665901Z"
},
"t_008_index_lookup": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user is User u)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = u.Id,\n Name = u.Name\n });\n }\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-008-index-lookup-golden",
@@ -1695,6 +1639,14 @@
"work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm",
"scorer_details": {
+ "index_lookup_projection_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200975331a02734d29ed787b74c37af9c2198dd3b99c0a4e802172b07767c21/sql)\n",
+ "phase": "sql_golden"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1703,7 +1655,7 @@
"llm_db": "basics-t-008-index-lookup-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
@@ -1712,26 +1664,18 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009fb018392f6be9608948e77d6c1cbdd60e04fc795fc595009cb76a175394/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200975331a02734d29ed787b74c37af9c2198dd3b99c0a4e802172b07767c21/sql)\n",
"phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
- },
- "index_lookup_projection_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009fb018392f6be9608948e77d6c1cbdd60e04fc795fc595009cb76a175394/sql)\n",
- "phase": "sql_golden"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:41:28.991836824Z",
- "finished_at": "2026-01-17T02:42:19.102358640Z"
+ "started_at": "2026-01-20T11:40:38.638873012Z",
+ "finished_at": "2026-01-20T11:41:56.558627065Z"
},
"t_009_init": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_009_init",
"lang": "csharp",
"golden_published": true,
@@ -1746,14 +1690,6 @@
"work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm",
"scorer_details": {
- "init_seed_bob": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1762,41 +1698,49 @@
"llm_db": "basics-t-009-init-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
},
+ "init_total_two": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200d4e79c70bfe54c7c0de4c6f0671c364004662cac09ef21b295e102bc706e/sql)\n",
+ "phase": "sql"
+ }
+ },
"init_seed_alice": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200d4e79c70bfe54c7c0de4c6f0671c364004662cac09ef21b295e102bc706e/sql)\n",
"phase": "sql"
}
},
- "init_total_two": {
+ "init_seed_bob": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200d4e79c70bfe54c7c0de4c6f0671c364004662cac09ef21b295e102bc706e/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:41:35.805507675Z",
- "finished_at": "2026-01-17T02:42:25.159425259Z"
+ "started_at": "2026-01-20T11:41:56.558730690Z",
+ "finished_at": "2026-01-20T11:42:26.326975636Z"
},
"t_010_connect": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"connected\",\n });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"disconnected\",\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"Event\")]\n public partial struct Event\n {\n [PrimaryKey]\n [AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-010-connect-golden",
@@ -1812,25 +1756,25 @@
"llm_db": "basics-t-010-connect-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:39.643859323Z",
- "finished_at": "2026-01-17T02:44:14.452325815Z"
+ "started_at": "2026-01-20T11:42:42.620873588Z",
+ "finished_at": "2026-01-20T11:43:17.522655691Z"
},
"t_011_helper_function": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-011-helper-function-golden",
@@ -1838,14 +1782,6 @@
"work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm",
"scorer_details": {
- "helper_func_sum_abs": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200bacfa9e399c2f18f42a36dcfe8479e933d44e6e749cbcc699afdf2cd855d/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1854,33 +1790,41 @@
"llm_db": "basics-t-011-helper-function-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
},
+ "helper_func_sum_abs": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2006be2a1a8cbac23587ce04bfea5fb3978206522ac94b229cf4d03f7181993/sql)\n",
+ "phase": "sql"
+ }
+ },
"helper_func_sum_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200b67e605283bc59b71366b2f40ab582a2bbbfc946e2c8510ce84f55424a14/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c20036d918a78db9840195746d13ce8987c3f47746ba44954d6162fc2d2fa90c/sql)\n",
"phase": "sql_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:52.237342945Z",
- "finished_at": "2026-01-17T02:43:28.640296252Z"
+ "started_at": "2026-01-20T11:42:08.626209992Z",
+ "finished_at": "2026-01-20T11:42:41.469759097Z"
},
"t_012_spacetime_product_type": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-012-spacetime-product-type-golden",
@@ -1888,6 +1832,14 @@
"work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm",
"scorer_details": {
+ "product_type_row_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2004f87de77d55a11535102093b18571a47e4184def1f9e3f32678bb8dea5a6/sql)\n",
+ "phase": "sql"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1896,7 +1848,7 @@
"llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
@@ -1905,32 +1857,24 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200000593a1c1f41d9a7ca9ff85b3f10b7eb54ba57159f084fd8b1d4196e453/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200f4efd97d4d0a75aec1abf28482cf3546aeb405da22e238f5fbb8727ddd58/sql)\n",
"phase": "sql_golden"
}
- },
- "product_type_row_count": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200773151fae81e0c0863ae9d0a4534b2934c3b4023363ffb2b7c1930309b46/sql)\n",
- "phase": "sql"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:56.019269384Z",
- "finished_at": "2026-01-17T02:39:27.502611490Z"
+ "started_at": "2026-01-20T11:44:24.426665842Z",
+ "finished_at": "2026-01-20T11:44:54.866711307Z"
},
"t_013_spacetime_sum_type": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(SpacetimeDB.ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-013-spacetime-sum-type-golden",
@@ -1942,10 +1886,18 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009a86be1f0f78642d2ade147db198738ef88aa551d11bb76e42c1c8900e9a/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200a0da5189c8fa804767b2bd511533e9fe477226152c15c21a57b945a3cc22/sql)\n",
"phase": "sql"
}
},
+ "sum_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200a45e7bc6ca39a049370c62e31842dd6dc73e4ec8bdfe8a29014365ff5e7d/sql)\n",
+ "phase": "sql_golden"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1954,33 +1906,25 @@
"llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
- },
- "sum_type_row_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d82761d2c72dc9a864fd786af31b4c9ef43bb79b633450e8e561c3eea2ce/sql)\n",
- "phase": "sql_golden"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:02.303793183Z",
- "finished_at": "2026-01-17T02:40:51.462059047Z"
+ "started_at": "2026-01-20T11:43:17.522678607Z",
+ "finished_at": "2026-01-20T11:44:03.839181513Z"
},
"t_014_elementary_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-014-elementary-columns-golden",
@@ -1992,7 +1936,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003f9c20afc908b6f3fae929b8555ec0e90cf407b107f224de84971189e7b3/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2004bbe0e1e70629621361df3638d5aacddc199728eb7ed4ab73e61373eda15/sql)\n",
"phase": "sql_golden"
}
},
@@ -2000,7 +1944,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009b868c8501f40d75f3046c80b4fa8bc7018d8baf65ff12c991bbab6d605a/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2001ee71f9441acae664d521e6271de513e5a48ceae65971593b75f6ed68a11/sql)\n",
"phase": "sql"
}
},
@@ -2012,25 +1956,25 @@
"llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:36.670677504Z",
- "finished_at": "2026-01-17T02:39:15.092747325Z"
+ "started_at": "2026-01-20T11:43:51.587828598Z",
+ "finished_at": "2026-01-20T11:44:24.426626210Z"
},
"t_015_product_type_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-015-product-type-columns-golden",
@@ -2038,24 +1982,11 @@
"work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"product_type_columns_row_count": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2007bfa13561ce0c8cb477e4e11171bc3733d8ba01683aecd94ed2d3f231319/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c20035b004bf29a05e46367e77f78b84e04395f2d343aca9756590f01ecbff73/sql)\n",
"phase": "sql"
}
},
@@ -2063,17 +1994,30 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200e5ea78db1095b274afeafb8041e139a0d7f11834b194b863882d4a7c3328/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c20082df7a9ef45a5d64aea7f9e7c8449154e6d3233ee5e2c5a3262650883df7/sql)\n",
"phase": "sql_golden"
}
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:46139",
+ "tables_diff": null,
+ "tables_equal": true
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:39:15.092804541Z",
- "finished_at": "2026-01-17T02:40:02.303741426Z"
+ "started_at": "2026-01-20T11:42:50.247602659Z",
+ "finished_at": "2026-01-20T11:43:37.383787950Z"
},
"t_016_sum_type_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
@@ -2092,7 +2036,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2004da484f449e95616d5eb1c6832e7e06faf68e5ebbe454e63475d86740e6c/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2002f2972d824a7dfc951c2e60a51fc742d6d2738117abeceb7d9b6d013922e/sql)\n",
"phase": "sql_golden"
}
},
@@ -2100,7 +2044,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ad55b1dba50ea8bb36552e0bc9480b27d8060b57d8ed5e9e4d86cb6b5f8/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2007235d9c559fe12b2aef851e2af1662962e2e28260aa3746eb8c43d72730b/sql)\n",
"phase": "sql"
}
},
@@ -2112,25 +2056,25 @@
"llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:39:27.502655041Z",
- "finished_at": "2026-01-17T02:40:24.282007063Z"
+ "started_at": "2026-01-20T11:44:03.839214689Z",
+ "finished_at": "2026-01-20T11:44:43.638959909Z"
},
"t_017_scheduled_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 2,
"passed_tests": 1,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-017-scheduled-columns-golden",
@@ -2138,14 +2082,6 @@
"work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20024b2c639ed44059f68c872d4ae8686e43def045ff88fb49f452645a17338/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -2154,25 +2090,33 @@
"llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "scheduled_seeded_one_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200e8dbd82cb96dc374a6b3272001149e3bfcbaf304766aaf1e167942e1ceb2/sql)\n",
+ "phase": "sql"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:01.497464655Z",
- "finished_at": "2026-01-17T02:38:36.670468145Z"
+ "started_at": "2026-01-20T11:43:37.383821107Z",
+ "finished_at": "2026-01-20T11:44:30.220362656Z"
},
"t_018_constraints": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [\"Name\"])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-018-constraints-golden",
@@ -2188,7 +2132,7 @@
"llm_db": "schema-t-018-constraints-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
@@ -2197,7 +2141,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200db49191274d7a3a7263567507f9945c4e802cb3173928c835966c8331803/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200dbc01789ef0e4e4522a39a52ac84d6e360c14296376df58ac6b3a8a28b54/sql)\n",
"phase": "sql_golden"
}
},
@@ -2205,24 +2149,24 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20025629920e94d036ee70c22a88bf38265ab06fbb5cfc2c505e147018ad40c/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200fc3f2b12088a9f4ff90c71e4fcb938c4b6cdd64f191c0579bf103eeef3b2/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:37:43.947065043Z",
- "finished_at": "2026-01-17T02:38:24.847760366Z"
+ "started_at": "2026-01-20T11:43:06.726672562Z",
+ "finished_at": "2026-01-20T11:43:51.587797418Z"
},
"t_019_many_to_many": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-019-many-to-many-golden",
@@ -2234,23 +2178,28 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200e5c9cea81ae8bcbaaaaead0a2e0c7bc5b6b9327a1e015295646919705f94/sql)\n",
"phase": "sql"
}
},
- "m2m_has_2_20": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
- "phase": "sql"
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:46139",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "memberships_three_rows": {
+ "m2m_has_2_20": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200e5c9cea81ae8bcbaaaaead0a2e0c7bc5b6b9327a1e015295646919705f94/sql)\n",
"phase": "sql"
}
},
@@ -2258,37 +2207,32 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200e5c9cea81ae8bcbaaaaead0a2e0c7bc5b6b9327a1e015295646919705f94/sql)\n",
"phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "memberships_three_rows": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c200e5c9cea81ae8bcbaaaaead0a2e0c7bc5b6b9327a1e015295646919705f94/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:36:42.262573385Z",
- "finished_at": "2026-01-17T02:37:43.946992007Z"
+ "started_at": "2026-01-20T11:43:56.646507798Z",
+ "finished_at": "2026-01-20T11:44:36.014588130Z"
},
"t_020_ecs": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n var nx = pos.X + vel.Value.VX;\n var ny = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var row = existing.Value;\n row.X = nx;\n row.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(row);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n InsertEntityWithComponents(ctx, 1, 0, 0, 1, 0);\n InsertEntityWithComponents(ctx, 2, 10, 0, -2, 3);\n }\n\n private static void InsertEntityWithComponents(ReducerContext ctx, int id, int x, int y, int vx, int vy)\n {\n if (ctx.Db.Entity.Id.Find(id) is not Entity)\n {\n ctx.Db.Entity.Insert(new Entity { Id = id });\n }\n\n if (ctx.Db.Position.EntityId.Find(id) is Position existingPos)\n {\n existingPos.X = x;\n existingPos.Y = y;\n ctx.Db.Position.EntityId.Update(existingPos);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = id, X = x, Y = y });\n }\n\n if (ctx.Db.Velocity.EntityId.Find(id) is Velocity existingVel)\n {\n existingVel.VX = vx;\n existingVel.VY = vy;\n ctx.Db.Velocity.EntityId.Update(existingVel);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = id, VX = vx, VY = vy });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is not Velocity vel)\n {\n continue;\n }\n\n var nx = pos.X + vel.VX;\n var ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition next)\n {\n next.X = nx;\n next.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(next);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-020-ecs-golden",
@@ -2300,23 +2244,28 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2001636e22feb22354131f60f08004aa617f5433a869fb9bf997102509ca16b/sql)\n",
"phase": "sql"
}
},
- "ecs_seed_positions_count": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
- "phase": "sql"
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:46139",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "ecs_next_pos_entity2": {
+ "ecs_seed_positions_count": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2001636e22feb22354131f60f08004aa617f5433a869fb9bf997102509ca16b/sql)\n",
"phase": "sql"
}
},
@@ -2324,30 +2273,25 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2001636e22feb22354131f60f08004aa617f5433a869fb9bf997102509ca16b/sql)\n",
"phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "ecs_next_pos_entity2": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2001636e22feb22354131f60f08004aa617f5433a869fb9bf997102509ca16b/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:36:42.266606487Z",
- "finished_at": "2026-01-17T02:38:01.497277347Z"
+ "started_at": "2026-01-20T11:42:59.200296357Z",
+ "finished_at": "2026-01-20T11:43:56.646461704Z"
},
"t_021_multi_column_index": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
@@ -2362,22 +2306,6 @@
"work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm",
"scorer_details": {
- "mcindex_lookup_u7_d1": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
- "phase": "sql"
- }
- },
- "mcindex_seed_count": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -2386,23 +2314,39 @@
"llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:46139",
"tables_diff": null,
"tables_equal": true
}
},
+ "mcindex_lookup_u7_d1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2007b07f66bf5bafd24f8d927454e783f25d3a06c00a527f4245262d9c7bdb4/sql)\n",
+ "phase": "sql"
+ }
+ },
"mcindex_lookup_u7_d2": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2007b07f66bf5bafd24f8d927454e783f25d3a06c00a527f4245262d9c7bdb4/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46139/v1/database/c2007b07f66bf5bafd24f8d927454e783f25d3a06c00a527f4245262d9c7bdb4/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:24.847804258Z",
- "finished_at": "2026-01-17T02:38:56.019226354Z"
+ "started_at": "2026-01-20T11:44:30.220391018Z",
+ "finished_at": "2026-01-20T11:45:02.758450473Z"
}
}
}
diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json
index 30e341ff193..7ce93c1ab54 100644
--- a/docs/llms/docs-benchmark-summary.json
+++ b/docs/llms/docs-benchmark-summary.json
@@ -1,21 +1,21 @@
{
"version": 1,
- "generated_at": "2026-01-17T02:44:26.905Z",
+ "generated_at": "2026-01-20T11:45:02.790Z",
"by_language": {
"csharp": {
"modes": {
"docs": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "bdf3ee21189f531e8de36a4dfe9a85d96b0d893667fb5bf61fe979ce8282d0ce",
"models": {
"GPT-5": {
"categories": {
"basics": {
"tasks": 12,
- "total_tests": 27,
- "passed_tests": 12,
- "pass_pct": 44.444443,
- "task_pass_equiv": 7.333334,
- "task_pass_pct": 61.111115
+ "total_tests": 25,
+ "passed_tests": 11,
+ "pass_pct": 44.0,
+ "task_pass_equiv": 7.0000005,
+ "task_pass_pct": 58.333336
},
"schema": {
"tasks": 10,
@@ -28,11 +28,11 @@
},
"totals": {
"tasks": 22,
- "total_tests": 61,
- "passed_tests": 22,
- "pass_pct": 36.065575,
- "task_pass_equiv": 10.483333,
- "task_pass_pct": 47.651512
+ "total_tests": 59,
+ "passed_tests": 21,
+ "pass_pct": 35.59322,
+ "task_pass_equiv": 10.15,
+ "task_pass_pct": 46.13636
}
}
}
@@ -48,28 +48,28 @@
"categories": {
"basics": {
"tasks": 12,
- "total_tests": 27,
- "passed_tests": 25,
- "pass_pct": 92.59259,
- "task_pass_equiv": 10.0,
- "task_pass_pct": 83.33333
+ "total_tests": 25,
+ "passed_tests": 17,
+ "pass_pct": 68.0,
+ "task_pass_equiv": 8.083334,
+ "task_pass_pct": 67.361115
},
"schema": {
"tasks": 10,
"total_tests": 34,
- "passed_tests": 29,
- "pass_pct": 85.29412,
- "task_pass_equiv": 8.0,
- "task_pass_pct": 80.0
+ "passed_tests": 32,
+ "pass_pct": 94.117645,
+ "task_pass_equiv": 9.0,
+ "task_pass_pct": 90.0
}
},
"totals": {
"tasks": 22,
- "total_tests": 61,
- "passed_tests": 54,
- "pass_pct": 88.52459,
- "task_pass_equiv": 18.0,
- "task_pass_pct": 81.818184
+ "total_tests": 59,
+ "passed_tests": 49,
+ "pass_pct": 83.05085,
+ "task_pass_equiv": 17.083334,
+ "task_pass_pct": 77.65152
}
}
}
diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css
index 81afdbbbc6d..5cebb854662 100644
--- a/docs/src/css/custom.css
+++ b/docs/src/css/custom.css
@@ -99,6 +99,9 @@
--ifm-tabs-padding-vertical: 8px;
--ifm-navbar-height: 57px;
+
+ --ifm-footer-padding-vertical: 0;
+ --ifm-footer-padding-horizontal: 0;
}
:root .alert--info {
@@ -747,4 +750,4 @@ a.card {
.step-code {
margin-top: 16px;
}
-}
\ No newline at end of file
+}