diff --git a/.cargo/config.toml b/.cargo/config.toml index 755c9d30397..5ee0dda9f02 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"] [alias] bump-versions = "run -p upgrade-version --" +ci = "run -p ci --" [target.x86_64-pc-windows-msvc] # Use a different linker. Otherwise, the build fails with some obscure linker error that diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cefb6f9e59..4bb309a5136 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,9 +111,14 @@ jobs: if: runner.os == 'Windows' - name: Install python deps run: python -m pip install -r smoketests/requirements.txt - - name: Run smoketests + - name: Run smoketests (windows) # Note: clear_database and replication only work in private run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams + if: runner.os == 'Windows' + - name: Run smoketests (Linux) + # Note: clear_database and replication only work in private + run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams + if: runner.os != 'Windows' - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose -f .github/docker-compose.yml down @@ -167,34 +172,12 @@ jobs: with: run_install: true - - name: Create /stdb dir - run: | - sudo mkdir /stdb - sudo chmod 777 /stdb - - name: Build typescript module sdk working-directory: crates/bindings-typescript run: pnpm build - - name: Run cargo test - #Note: Unreal tests will be run separately - run: cargo test --all -- --skip unreal - - # The fallocate tests have been flakely when running in parallel - - name: Run fallocate tests - run: cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1 - - - name: Check that the test outputs are up-to-date - run: bash tools/check-diff.sh - - - name: Ensure C# autogen bindings are up-to-date - run: | - cargo run -p spacetimedb-codegen --example regen-csharp-moduledef - bash tools/check-diff.sh crates/bindings-csharp - - - name: C# bindings tests - working-directory: crates/bindings-csharp - run: dotnet test -warnaserror + - name: Run tests + run: cargo ci test lints: name: Lints @@ -224,30 +207,8 @@ jobs: with: global-json-file: global.json - - name: Run cargo fmt - run: cargo fmt --all -- --check - - - name: Run cargo clippy - run: cargo clippy --all --tests --benches -- -D warnings - - - name: Run C# formatting check - working-directory: crates/bindings-csharp - run: | - dotnet tool restore - dotnet csharpier --check . - - - name: Run `cargo doc` for bindings crate - # `bindings` is the only crate we care strongly about documenting, - # since we link to its docs.rs from our website. - # We won't pass `--no-deps`, though, - # since we want everything reachable through it to also work. - # This includes `sats` and `lib`. - working-directory: crates/bindings - env: - # Make `cargo doc` exit with error on warnings, most notably broken links - RUSTDOCFLAGS: '--deny warnings' - run: | - cargo doc + - name: Run ci lint + run: cargo ci lint wasm_bindings: name: Build and test wasm bindings @@ -273,20 +234,7 @@ jobs: save-if: false - name: Run bindgen tests - run: cargo test -p spacetimedb-codegen - - # Make sure the `Cargo.lock` file reflects the latest available versions. - # This is what users would end up with on a fresh module, so we want to - # catch any compile errors arising from a different transitive closure - # of dependencies than what is in the workspace lock file. - # - # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - - name: Update dependencies - run: cargo update - - - name: Build module-test - run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test - + run: cargo ci wasm-bindings publish_checks: name: Check that packages are publishable @@ -350,6 +298,7 @@ jobs: - name: Build spacetimedb-update run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update + if: runner.os == 'Windows' - name: Run self-install env: @@ -362,6 +311,13 @@ jobs: # happens very frequently on the `macos-runner`, but we haven't seen it on any others). cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help + if: runner.os == 'Windows' + + - name: Test spacetimedb-update + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: cargo ci update-flow --target=${{ matrix.target }} --github-token-auth + if: runner.os != 'Windows' unreal_engine_tests: name: Unreal Engine Tests @@ -386,20 +342,20 @@ jobs: # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle. options: --user 0:0 steps: -# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back -# commits though. -# - name: Find Git ref -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# shell: bash -# run: | -# PR_NUMBER="${{ github.event.inputs.pr_number || null }}" -# if test -n "${PR_NUMBER}"; then -# GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" -# else -# GIT_REF="${{ github.ref }}" -# fi -# echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + # Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back + # commits though. + # - name: Find Git ref + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # shell: bash + # run: | + # PR_NUMBER="${{ github.event.inputs.pr_number || null }}" + # if test -n "${PR_NUMBER}"; then + # GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" + # else + # GIT_REF="${{ github.ref }}" + # fi + # echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - name: Checkout sources uses: actions/checkout@v4 with: @@ -452,6 +408,33 @@ jobs: cargo test -- --test-threads=1 ' + ci_command_docs: + name: Check CI command docs + runs-on: ubuntu-latest + steps: + - name: Find Git ref + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + PR_NUMBER="${{ github.event.inputs.pr_number || null }}" + if test -n "${PR_NUMBER}"; then + GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" + else + GIT_REF="${{ github.ref }}" + fi + echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" + + - name: Checkout sources + uses: actions/checkout@v4 + with: + ref: ${{ env.GIT_REF }} + + - uses: dsherret/rust-toolchain-file@v1 + + - name: Check for docs change + run: cargo ci self-docs --check + cli_docs: name: Check CLI docs permissions: read-all @@ -515,15 +498,7 @@ jobs: - name: Check for docs change run: | - cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md - pnpm format - git status - if git diff --exit-code HEAD -- docs/docs/cli-reference.md; then - echo "No docs changes detected" - else - echo "It looks like the CLI docs have changed:" - exit 1 - fi + cargo ci cli-docs unity-testsuite: # Skip if this is an external contribution. @@ -609,7 +584,7 @@ jobs: enable_pr_comment: ${{ github.event_name == 'pull_request' }} target_path: sdks/csharp env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' - name: Start SpacetimeDB run: | @@ -643,12 +618,12 @@ jobs: - name: Run Unity tests uses: game-ci/unity-test-runner@v4 with: - unityVersion: 2022.3.32f1 # Adjust Unity version to a valid tag - projectPath: demo/Blackholio/client-unity # Path to the Unity project subdirectory + unityVersion: 2022.3.32f1 # Adjust Unity version to a valid tag + projectPath: demo/Blackholio/client-unity # Path to the Unity project subdirectory githubToken: ${{ secrets.GITHUB_TOKEN }} testMode: playmode useHostNetwork: true - artifactsPath: "" + artifactsPath: '' env: UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} diff --git a/Cargo.lock b/Cargo.lock index 29568ab1fc3..566644cdf0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -871,6 +871,18 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "ci" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap 4.5.50", + "duct", + "log", + "regex", +] + [[package]] name = "ciborium" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index b4cf0f4d9a5..0aba47e1cf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,62 +1,63 @@ [workspace] members = [ - "crates/auth", - "crates/bench", - "crates/bindings-sys", - "crates/bindings", - "crates/bindings-macro", - "crates/cli", - "crates/client-api", - "crates/client-api-messages", - "crates/commitlog", - "crates/core", - "crates/data-structures", - "crates/datastore", - "crates/durability", - "crates/execution", - "crates/expr", - "crates/fs-utils", - "crates/lib", - "crates/metrics", - "crates/paths", - "crates/pg", - "crates/physical-plan", - "crates/primitives", - "crates/query", - "crates/sats", - "crates/schema", - "sdks/rust", - "sdks/unreal", - "crates/snapshot", - "crates/sqltest", - "crates/sql-parser", - "crates/standalone", - "crates/subscription", - "crates/table", - "crates/testing", - "crates/update", - "crates/vm", - "modules/benchmarks", - "modules/keynote-benchmarks", - "modules/perf-test", - "modules/module-test", - "modules/quickstart-chat", - "modules/sdk-test", - "modules/sdk-test-connect-disconnect", - "modules/sdk-test-procedure", - "modules/sdk-test-view", - "sdks/rust/tests/test-client", - "sdks/rust/tests/test-counter", - "sdks/rust/tests/connect_disconnect_client", - "sdks/rust/tests/procedure-client", - "sdks/rust/tests/view-client", - "tools/upgrade-version", - "tools/license-check", - "tools/replace-spacetimedb", - "tools/generate-client-api", - "tools/gen-bindings", - "crates/bindings-typescript/test-app/server", - "crates/bindings-typescript/test-react-router-app/server", + "crates/auth", + "crates/bench", + "crates/bindings-sys", + "crates/bindings", + "crates/bindings-macro", + "crates/cli", + "crates/client-api", + "crates/client-api-messages", + "crates/commitlog", + "crates/core", + "crates/data-structures", + "crates/datastore", + "crates/durability", + "crates/execution", + "crates/expr", + "crates/fs-utils", + "crates/lib", + "crates/metrics", + "crates/paths", + "crates/pg", + "crates/physical-plan", + "crates/primitives", + "crates/query", + "crates/sats", + "crates/schema", + "sdks/rust", + "sdks/unreal", + "crates/snapshot", + "crates/sqltest", + "crates/sql-parser", + "crates/standalone", + "crates/subscription", + "crates/table", + "crates/testing", + "crates/update", + "crates/vm", + "modules/benchmarks", + "modules/keynote-benchmarks", + "modules/perf-test", + "modules/module-test", + "modules/quickstart-chat", + "modules/sdk-test", + "modules/sdk-test-connect-disconnect", + "modules/sdk-test-procedure", + "modules/sdk-test-view", + "sdks/rust/tests/test-client", + "sdks/rust/tests/test-counter", + "sdks/rust/tests/connect_disconnect_client", + "sdks/rust/tests/procedure-client", + "sdks/rust/tests/view-client", + "tools/ci", + "tools/upgrade-version", + "tools/license-check", + "tools/replace-spacetimedb", + "tools/generate-client-api", + "tools/gen-bindings", + "crates/bindings-typescript/test-app/server", + "crates/bindings-typescript/test-react-router-app/server", ] default-members = ["crates/cli", "crates/standalone", "crates/update"] # cargo feature graph resolver. v3 is default in edition2024 but workspace @@ -164,10 +165,16 @@ colored = "2.0.0" console = { version = "0.15.6" } convert_case = "0.6.0" crc32c = "0.6.4" -criterion = { version = "0.5.1", features = ["async", "async_tokio", "html_reports"] } +criterion = { version = "0.5.1", features = [ + "async", + "async_tokio", + "html_reports", +] } crossbeam-channel = "0.5" crossbeam-queue = "0.3.12" -cursive = { version = "0.20", default-features = false, features = ["crossterm-backend"] } +cursive = { version = "0.20", default-features = false, features = [ + "crossterm-backend", +] } decorum = { version = "0.3.1", default-features = false, features = ["std"] } derive_more = "0.99" dialoguer = { version = "0.11", default-features = false } @@ -191,14 +198,19 @@ futures-util = "0.3" getrandom02 = { package = "getrandom", version = "0.2" } git2 = "0.19" glob = "0.3.1" -hashbrown = { version = "0.16.1", default-features = false, features = ["equivalent", "inline-more", "rayon", "serde"] } +hashbrown = { version = "0.16.1", default-features = false, features = [ + "equivalent", + "inline-more", + "rayon", + "serde", +] } headers = "0.4" heck = "0.4" hex = "0.4.3" home = "0.5" hostname = "^0.3" http = "1.0" -http-body-util= "0.1.3" +http-body-util = "0.1.3" humantime = "2.1.0" hyper = "1.0" hyper-util = { version = "0.1", features = ["tokio"] } @@ -227,7 +239,10 @@ paste = "1.0" percent-encoding = "2.3" petgraph = { version = "0.6.5", default-features = false } pin-project-lite = "0.2.9" -pgwire = { version = "0.34.2", default-features = false, features = ["server-api", "pg-ext-types"] } +pgwire = { version = "0.34.2", default-features = false, features = [ + "server-api", + "pg-ext-types", +] } postgres-types = "0.2.5" pretty_assertions = { version = "1.4", features = ["unstable"] } proc-macro2 = "1.0" @@ -257,7 +272,10 @@ second-stack = "0.3" self-replace = "1.5" semver = "1" serde = { version = "1.0.136", features = ["derive"] } -serde_json = { version = "1.0.128", features = ["raw_value", "arbitrary_precision"] } +serde_json = { version = "1.0.128", features = [ + "raw_value", + "arbitrary_precision", +] } serde_path_to_error = "0.1.9" serde_with = { version = "3.3.0", features = ["base64", "hex"] } serial_test = "2.0.0" @@ -274,7 +292,9 @@ sqllogictest-engines = "0.17" sqlparser = "0.38.0" strum = { version = "0.25.0", features = ["derive"] } syn = { version = "2", features = ["full", "extra-traits"] } -syntect = { version = "5.0.0", default-features = false, features = ["default-fancy"] } +syntect = { version = "5.0.0", default-features = false, features = [ + "default-fancy", +] } tabled = "0.14.0" tar = "0.4" tempdir = "0.3.7" @@ -327,14 +347,14 @@ openssl = { version = "0.10", features = ["vendored"] } version = "39" default-features = false features = [ - "addr2line", - "async", - "cache", - "cranelift", - "demangle", - "parallel-compilation", - "runtime", - "std", + "addr2line", + "async", + "cache", + "cranelift", + "demangle", + "parallel-compilation", + "runtime", + "std", ] [workspace.dependencies.tracing-tracy] @@ -343,13 +363,13 @@ version = "0.10.4" # and reconnecting, from the tracy client to the database. # TODO(George): Need to be able to remove "broadcast" in some build configurations. features = [ - "enable", - "system-tracing", - "context-switch-tracing", - "sampling", - "code-transfer", - "broadcast", - "ondemand", + "enable", + "system-tracing", + "context-switch-tracing", + "sampling", + "code-transfer", + "broadcast", + "ondemand", ] [workspace.lints.rust] diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml new file mode 100644 index 00000000000..e907526a42c --- /dev/null +++ b/tools/ci/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci" +version = "0.1.0" +edition.workspace = true + +[dependencies] +log.workspace = true +anyhow.workspace = true +chrono = { workspace = true, features=["clock"] } +clap.workspace = true +regex.workspace = true +duct.workspace = true diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 00000000000..080dc354f33 --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,148 @@ +# SpacetimeDB's cargo ci + +## Overview + +This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. + +## `cargo ci` + +SpacetimeDB CI tasks + +This tool provides several subcommands for automating CI workflows in SpacetimeDB. + +It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally + +**Usage:** +```bash +Usage: cargo ci [OPTIONS] [COMMAND] +``` + +**Options:** + +- `--skip`: Skip specified subcommands when running all + +When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. + +- `--help`: Print help (see a summary with '-h') + +### `test` + +Runs tests + +Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. + +**Usage:** +```bash +Usage: test +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `lint` + +Lints the codebase + +Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. + +**Usage:** +```bash +Usage: lint +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `wasm-bindings` + +Tests Wasm bindings + +Runs tests for the codegen crate and builds a test module with the wasm bindings. + +**Usage:** +```bash +Usage: wasm-bindings +``` + +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `smoketests` + +Runs smoketests + +Executes the smoketests suite with some default exclusions. + +**Usage:** +```bash +Usage: smoketests [ARGS]... +``` + +**Options:** + +- `args`: Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker` +- `--help`: Print help (see a summary with '-h') + +### `update-flow` + +Tests the update flow + +Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. + +**Usage:** +```bash +Usage: update-flow [OPTIONS] +``` + +**Options:** + +- `--target`: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. +- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. +- `--help`: Print help (see a summary with '-h') + +### `cli-docs` + +**Usage:** +```bash +Usage: cli-docs [OPTIONS] +``` + +**Options:** + +- `--spacetime-path`: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) +- `--help`: Print help (see a summary with '-h') + +### `self-docs` + +**Usage:** +```bash +Usage: self-docs [OPTIONS] +``` + +**Options:** + +- `--check`: Only check for changes, do not generate the docs +- `--help`: Print help (see a summary with '-h') + +### `help` + +**Usage:** +```bash +Usage: help [COMMAND]... +``` + +**Options:** + +- `subcommand`: + + +--- + +This document is auto-generated by running: + +```bash +cargo ci self-docs +``` \ No newline at end of file diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs new file mode 100644 index 00000000000..f09114e6787 --- /dev/null +++ b/tools/ci/src/ci_docs.rs @@ -0,0 +1,69 @@ +use clap::{Command, CommandFactory}; + +use crate::Cli; + +// TODO: use clap_markdown instead of this custom implementation in the future +pub fn generate_cli_docs() -> String { + let mut cli = Cli::command(); + let usage = generate_markdown(&mut cli, 2); + + format!( + "\ +# SpacetimeDB's cargo ci + +## Overview + +This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. + +{} +--- + +This document is auto-generated by running: + +```bash +cargo ci self-docs +```", + usage + ) +} + +fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { + let mut out = String::new(); + + let heading = "#".repeat(heading_level); + out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); + + if let Some(long_about) = cmd.get_long_about() { + out.push_str(&format!("{}\n\n", long_about)); + } + + out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); + + let mut options = String::new(); + for arg in cmd.get_arguments() { + let names = arg + .get_long() + .map(|l| format!("--{}", l)) + .or_else(|| arg.get_short().map(|s| format!("-{}", s))) + .unwrap_or_else(|| arg.get_id().to_string()); + let help = arg.get_long_help().unwrap_or_default(); + options.push_str(&format!( + "- `{}`: {}\n{}", + names, + help, + if help.to_string().lines().count() > 1 { "\n" } else { "" } + )); + } + + if !options.is_empty() { + out.push_str("**Options:**\n\n"); + out.push_str(&options); + out.push('\n'); + } + + for sub in cmd.get_subcommands_mut() { + out.push_str(&generate_markdown(sub, heading_level + 1)); + } + + out +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs new file mode 100644 index 00000000000..6bf1171a6e0 --- /dev/null +++ b/tools/ci/src/main.rs @@ -0,0 +1,260 @@ +use anyhow::{bail, Result}; +use clap::{CommandFactory, Parser, Subcommand}; +use duct::cmd; +use std::collections::HashMap; +use std::path::Path; +use std::{env, fs}; + +const README_PATH: &str = "tools/ci/README.md"; + +mod ci_docs; + +/// SpacetimeDB CI tasks +/// +/// This tool provides several subcommands for automating CI workflows in SpacetimeDB. +/// +/// It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in +/// sequence. It is mostly designed to be run in CI environments via the github workflows, but can +/// also be run locally +#[derive(Parser)] +#[command(name = "cargo ci", subcommand_required = false, arg_required_else_help = false)] +struct Cli { + #[command(subcommand)] + cmd: Option, + + /// Skip specified subcommands when running all + /// + /// When no subcommand is specified, all subcommands are run in sequence. This option allows + /// specifying subcommands to skip when running all. For example, to skip the `unreal-tests` + /// subcommand, use `--skip unreal-tests`. + #[arg(long)] + skip: Vec, +} + +#[derive(Subcommand)] +enum CiCmd { + /// Runs tests + /// + /// Runs rust tests, codegens csharp sdk and runs csharp tests. + /// This does not include Unreal tests. + /// This expects to run in a clean git state. + Test, + /// Lints the codebase + /// + /// Runs rustfmt, clippy, csharpier and generates rust docs to ensure there are no warnings. + Lint, + /// Tests Wasm bindings + /// + /// Runs tests for the codegen crate and builds a test module with the wasm bindings. + WasmBindings, + /// Runs smoketests + /// + /// Executes the smoketests suite with some default exclusions. + Smoketests { + #[arg( + trailing_var_arg = true, + long_help = "Additional arguments to pass to the smoketests runner. These are usually set by the CI environment, such as `-- --docker`" + )] + args: Vec, + }, + /// Tests the update flow + /// + /// Tests the self-update flow by building the spacetimedb-update binary for the specified + /// target, by default the current target, and performing a self-install into a temporary + /// directory. + UpdateFlow { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + target: Option, + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + github_token_auth: bool, + }, + /// Generates CLI documentation and checks for changes + CliDocs { + #[arg( + long, + long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" + )] + spacetime_path: Option, + }, + SelfDocs { + #[arg( + long, + default_value_t = false, + long_help = "Only check for changes, do not generate the docs" + )] + check: bool, + }, +} + +macro_rules! bash { + ($cmdline:expr) => { + run_bash($cmdline, &Vec::new()) + }; + ($cmdline:expr, $envs:expr) => { + run_bash($cmdline, $envs) + }; +} + +fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { + let subcmds = Cli::command() + .get_subcommands() + .map(|sc| sc.get_name().to_string()) + .collect::>(); + + for subcmd in subcmds { + if skips.contains(&subcmd) { + log::info!("skipping {subcmd} as requested"); + continue; + } + log::info!("executing cargo ci {subcmd}"); + bash!(&format!("cargo ci {subcmd}"))?; + } + + Ok(()) +} + +fn run_bash(cmdline: &str, additional_env: &[(&str, &str)]) -> Result<()> { + let mut env = env::vars().collect::>(); + env.extend(additional_env.iter().map(|(k, v)| (k.to_string(), v.to_string()))); + log::debug!("$ {cmdline}"); + let status = cmd!("bash", "-lc", cmdline).full_env(env).run()?; + if !status.status.success() { + let e = anyhow::anyhow!("command failed: {cmdline}"); + log::error!("{e}"); + return Err(e); + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.cmd { + Some(CiCmd::Test) => { + bash!("cargo test --all -- --skip unreal")?; + // The fallocate tests have been flakely when running in parallel + bash!("cargo test -p spacetimedb-durability --features fallocate -- --test-threads=1")?; + bash!("bash tools/check-diff.sh")?; + bash!("cargo run -p spacetimedb-codegen --example regen-csharp-moduledef && bash tools/check-diff.sh crates/bindings-csharp")?; + bash!("(cd crates/bindings-csharp && dotnet test -warnaserror)")?; + } + + Some(CiCmd::Lint) => { + bash!("cargo fmt --all -- --check")?; + bash!("cargo clippy --all --tests --benches -- -D warnings")?; + bash!("(cd crates/bindings-csharp && dotnet tool restore && dotnet csharpier --check .)")?; + // `bindings` is the only crate we care strongly about documenting, + // since we link to its docs.rs from our website. + // We won't pass `--no-deps`, though, + // since we want everything reachable through it to also work. + // This includes `sats` and `lib`. + bash!( + "cd crates/bindings && cargo doc", + // Make `cargo doc` exit with error on warnings, most notably broken links + &[("RUSTDOCFLAGS", "--deny warnings")] + )?; + } + + Some(CiCmd::WasmBindings) => { + bash!("cargo test -p spacetimedb-codegen")?; + // Make sure the `Cargo.lock` file reflects the latest available versions. + // This is what users would end up with on a fresh module, so we want to + // catch any compile errors arising from a different transitive closure + // of dependencies than what is in the workspace lock file. + // + // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 + bash!("cargo update")?; + bash!("cargo run -p spacetimedb-cli -- build --project-path modules/module-test")?; + } + + Some(CiCmd::Smoketests { args }) => { + // On some systems, there is no `python`, but there is `python3`. + let py3_available = bash!("command -v python3 >/dev/null 2>&1").is_ok(); + let python = if py3_available { "python3" } else { "python" }; + bash!(&format!("{python} -m smoketests {}", args.join(" ")))?; + } + + Some(CiCmd::UpdateFlow { + target, + github_token_auth, + }) => { + let target = target.map(|t| format!("--target {t}")).unwrap_or_default(); + let github_token_auth_flag = if github_token_auth { + "--features github-token-auth " + } else { + "" + }; + + bash!(&format!("echo 'checking update flow for target: {target}'"))?; + bash!(&format!( + "cargo build {github_token_auth_flag}{target} -p spacetimedb-update" + ))?; + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). + bash!(&format!( + r#" +ROOT_DIR="$(mktemp -d)" +cargo run {github_token_auth_flag}{target} -p spacetimedb-update -- self-install --root-dir="${{ROOT_DIR}}" --yes +"${{ROOT_DIR}}"/spacetime --root-dir="${{ROOT_DIR}}" help + "# + ))?; + } + + Some(CiCmd::CliDocs { spacetime_path }) => { + if let Some(path) = spacetime_path { + env::set_current_dir(path).ok(); + } + let current_dir = env::current_dir().expect("No current directory!"); + let dir_name = current_dir.file_name().expect("No current directory!"); + if dir_name != "SpacetimeDB" && dir_name != "public" { + anyhow::bail!( + "You must execute this binary from inside of the SpacetimeDB directory, or use --spacetime-path" + ); + } + + bash!("pnpm install --recursive")?; + bash!("cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md")?; + bash!("pnpm format")?; + bash!("git status")?; + bash!( + r#" +if git diff --exit-code HEAD; then + echo "No docs changes detected" +else + echo "It looks like the CLI docs have changed:" + exit 1 +fi + "# + )?; + } + + Some(CiCmd::SelfDocs { check }) => { + let readme_content = ci_docs::generate_cli_docs(); + let path = Path::new(README_PATH); + + if check { + let existing = fs::read_to_string(path).unwrap_or_default(); + if existing != readme_content { + bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); + } else { + log::info!("README.md is up to date."); + } + } else { + fs::write(path, readme_content)?; + log::info!("Wrote CLI docs to {}", path.display()); + } + } + + None => run_all_clap_subcommands(&cli.skip)?, + } + + Ok(()) +}