From 41ddefcdc9e16ec3f2618844855f5bffe2582568 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 15 Dec 2025 16:54:28 +0800 Subject: [PATCH 1/2] fix(test): speed up debug and codecov tests --- .github/workflows/coverage.yml | 10 +++------- Cargo.toml | 4 +++- Makefile | 3 +++ interop-tests/Cargo.toml | 16 ++++++++++++++++ src/tool/subcommands/api_cmd/test_snapshot.rs | 5 ----- src/utils/mod.rs | 2 ++ 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index af5cfa2c1e9a..b7f807ac2577 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -31,6 +31,8 @@ env: RUSTC_WRAPPER: sccache CC: sccache clang CXX: sccache clang++ + # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times + RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" jobs: codecov: @@ -50,13 +52,7 @@ jobs: - uses: taiki-e/install-action@cargo-llvm-cov - uses: taiki-e/install-action@nextest - name: Generate code coverage - run: | - cargo llvm-cov --workspace --codecov --output-path lcov.info nextest - env: - CC: "sccache clang" - CXX: "sccache clang++" - # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times - RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld" + run: make codecov # Save lcov.info as an artifact for debugging purposes - uses: actions/upload-artifact@v4 with: diff --git a/Cargo.toml b/Cargo.toml index f9d09ee32c50..d08d4d3a4f0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -268,9 +268,11 @@ rust2go = { workspace = true, features = ["build"] } debug = "line-tables-only" split-debuginfo = "unpacked" -# Avoid generating any debug information for dependencies [profile.dev.package."*"] +# Avoid generating any debug information for dependencies debug = false +# Speed up debug tests and codecov tests +opt-level = 1 # Profile for interactive debugging sessions, e.g., with LLDB. # Doesn't work with Go FFI - disable it with `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1`. diff --git a/Makefile b/Makefile index 1bcba1f4610c..35beaf3e627c 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,9 @@ test-release: test-all: test test-release +codecov: + cargo llvm-cov --workspace --codecov --output-path lcov.info --ignore-run-fail + # Checks if all headers are present and adds if not license: ./scripts/add_license.sh diff --git a/interop-tests/Cargo.toml b/interop-tests/Cargo.toml index c07527cc8277..83789cc4f781 100644 --- a/interop-tests/Cargo.toml +++ b/interop-tests/Cargo.toml @@ -41,3 +41,19 @@ tokio = { workspace = true, features = ['full'] } [build-dependencies] rust2go = { workspace = true, features = ["build"] } + +# https://doc.rust-lang.org/stable/cargo/guide/build-performance.html#reduce-amount-of-generated-debug-information +[profile.dev] +debug = "line-tables-only" +split-debuginfo = "unpacked" + +[profile.dev.package."*"] +# Avoid generating any debug information for dependencies +debug = false +# Speed up debug tests and codecov tests +opt-level = 1 + +[profile.dev.package."forest-filecoin"] +debug = "line-tables-only" +split-debuginfo = "unpacked" +opt-level = 0 diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index b47bb6448613..55d50e79ac2a 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -199,11 +199,6 @@ mod tests { include!(concat!(env!("OUT_DIR"), "/__rpc_regression_tests_gen.rs")); async fn rpc_regression_test_run(name: &str) { - // Skip for debug build on CI as the downloading is slow and flaky - if crate::utils::is_ci() && crate::utils::is_debug_build() { - return; - } - // Set proof parameter data dir and make sure the proofs are available { static PROOF_PARAMS_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 7b5653328e3e..49de3de44285 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -169,11 +169,13 @@ pub enum RetryError { RetriesExceeded, } +#[allow(dead_code)] #[cfg(test)] pub fn is_debug_build() -> bool { cfg!(debug_assertions) } +#[allow(dead_code)] #[cfg(test)] pub fn is_ci() -> bool { // https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables From a0faf61072915c58b49bac23c53359681e052b93 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 16 Dec 2025 19:38:50 +0800 Subject: [PATCH 2/2] test: unit test for state compute --- src/benchmark_private/tipset_validation.rs | 83 ++-------------- src/chain_sync/tipset_syncer.rs | 2 +- src/message_pool/msgpool/msg_pool.rs | 2 +- src/state_manager/mod.rs | 1 - src/state_manager/utils.rs | 110 ++++++++++++++++++++- 5 files changed, 118 insertions(+), 80 deletions(-) diff --git a/src/benchmark_private/tipset_validation.rs b/src/benchmark_private/tipset_validation.rs index e1957393bad0..1fa208a495f9 100644 --- a/src/benchmark_private/tipset_validation.rs +++ b/src/benchmark_private/tipset_validation.rs @@ -2,76 +2,13 @@ // SPDX-License-Identifier: Apache-2.0, MIT use crate::{ - blocks::Tipset, - chain::store::ChainStore, - db::{ - MemoryDB, - car::{AnyCar, ManyCar}, + networks::NetworkChain, + state_manager::utils::state_compute::{ + get_state_compute_snapshot, prepare_state_compute, state_compute, }, - genesis::read_genesis_header, - interpreter::VMTrace, - networks::{ChainConfig, NetworkChain}, - state_manager::StateManager, - utils::net::{DownloadFileOption, download_file_with_cache}, }; use criterion::Criterion; -use directories::ProjectDirs; -use std::{ - hint::black_box, - path::{Path, PathBuf}, - sync::{Arc, LazyLock}, -}; -use url::Url; - -pub static CACHE_DIR: LazyLock = LazyLock::new(|| { - let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest"); - project_dir - .map(|d| d.cache_dir().to_path_buf()) - .unwrap_or_else(std::env::temp_dir) - .join("state_compute_snapshots") -}); - -async fn get_snapshot(chain: &NetworkChain, epoch: i64) -> anyhow::Result { - let url = Url::parse(&format!( - "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/state_compute/{chain}_{epoch}.forest.car.zst" - ))?; - Ok( - download_file_with_cache(&url, &CACHE_DIR, DownloadFileOption::NonResumable) - .await? - .path, - ) -} - -async fn prepare_validation( - chain: &NetworkChain, - snapshot: &Path, -) -> anyhow::Result<(Arc>, Tipset)> { - let snap_car = AnyCar::try_from(snapshot)?; - let ts = snap_car.heaviest_tipset()?; - let db = Arc::new(ManyCar::new(MemoryDB::default()).with_read_only(snap_car)?); - let chain_config = Arc::new(ChainConfig::from_chain(chain)); - let genesis_header = - read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db).await?; - let chain_store = Arc::new(ChainStore::new( - db.clone(), - db.clone(), - db.clone(), - db.clone(), - chain_config, - genesis_header, - )?); - let state_manager = Arc::new(StateManager::new(chain_store.clone())?); - // warmup - validate(state_manager.clone(), ts.clone()).await; - Ok((state_manager, ts)) -} - -async fn validate(state_manager: Arc>, ts: Tipset) { - state_manager - .compute_tipset_state(ts, crate::state_manager::NO_CALLBACK, VMTrace::NotTraced) - .await - .unwrap(); -} +use std::hint::black_box; pub fn bench_tipset_validation(c: &mut Criterion) { let rt = tokio::runtime::Builder::new_multi_thread() @@ -87,24 +24,24 @@ pub fn bench_tipset_validation(c: &mut Criterion) { let epoch = 3111900; let (state_manager, ts) = rt .block_on(async { - let snapshot = get_snapshot(&chain, epoch).await?; - prepare_validation(&chain, &snapshot).await + let snapshot = get_state_compute_snapshot(&chain, epoch).await?; + prepare_state_compute(&chain, &snapshot, true).await }) .unwrap(); b.to_async(&rt) - .iter(|| validate(black_box(state_manager.clone()), black_box(ts.clone()))) + .iter(|| state_compute(black_box(state_manager.clone()), black_box(ts.clone()))) }) .bench_function("mainnet@5427431", |b| { let chain = NetworkChain::Mainnet; let epoch = 5427431; let (state_manager, ts) = rt .block_on(async { - let snapshot = get_snapshot(&chain, epoch).await?; - prepare_validation(&chain, &snapshot).await + let snapshot = get_state_compute_snapshot(&chain, epoch).await?; + prepare_state_compute(&chain, &snapshot, true).await }) .unwrap(); b.to_async(&rt) - .iter(|| validate(black_box(state_manager.clone()), black_box(ts.clone()))) + .iter(|| state_compute(black_box(state_manager.clone()), black_box(ts.clone()))) }); group.finish(); diff --git a/src/chain_sync/tipset_syncer.rs b/src/chain_sync/tipset_syncer.rs index 162d8b5376d6..ff7208ddc708 100644 --- a/src/chain_sync/tipset_syncer.rs +++ b/src/chain_sync/tipset_syncer.rs @@ -11,7 +11,7 @@ use crate::shim::{ address::Address, crypto::verify_bls_aggregate, econ::BLOCK_GAS_LIMIT, gas::price_list_by_network_version, message::Message, state_tree::StateTree, }; -use crate::state_manager::{Error as StateManagerError, StateManager, is_valid_for_sending}; +use crate::state_manager::{Error as StateManagerError, StateManager, utils::is_valid_for_sending}; use crate::{ blocks::{Block, CachingBlockHeader, Error as ForestBlockError, FullTipset, Tipset}, fil_cns::{self, FilecoinConsensus, FilecoinConsensusError}, diff --git a/src/message_pool/msgpool/msg_pool.rs b/src/message_pool/msgpool/msg_pool.rs index d4b8b3be4c5a..74ec38b9e816 100644 --- a/src/message_pool/msgpool/msg_pool.rs +++ b/src/message_pool/msgpool/msg_pool.rs @@ -22,7 +22,7 @@ use crate::shim::{ econ::TokenAmount, gas::{Gas, price_list_by_network_version}, }; -use crate::state_manager::is_valid_for_sending; +use crate::state_manager::utils::is_valid_for_sending; use crate::utils::cache::SizeTrackingLruCache; use crate::utils::get_size::CidWrapper; use ahash::{HashMap, HashMapExt, HashSet, HashSetExt}; diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index 1fa90d31a6a6..b6e6f6aa94f4 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -88,7 +88,6 @@ use std::time::Duration; use std::{num::NonZeroUsize, sync::Arc}; use tokio::sync::{RwLock, broadcast::error::RecvError}; use tracing::{error, info, instrument, trace, warn}; -pub use utils::is_valid_for_sending; const DEFAULT_TIPSET_CACHE_SIZE: NonZeroUsize = nonzero!(1024usize); diff --git a/src/state_manager/utils.rs b/src/state_manager/utils.rs index ea0681e5164b..9d5f50e619ff 100644 --- a/src/state_manager/utils.rs +++ b/src/state_manager/utils.rs @@ -1,6 +1,7 @@ // Copyright 2019-2025 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use super::MinerActorStateLoad as _; use crate::shim::actors::miner; use crate::shim::{ actors::{is_account_actor, is_ethaccount_actor, is_placeholder_actor}, @@ -10,6 +11,7 @@ use crate::shim::{ state_tree::ActorState, version::NetworkVersion, }; +use crate::state_manager::{StateManager, errors::*}; use crate::utils::encoding::prover_id_from_u64; use cid::Cid; use fil_actors_shared::filecoin_proofs_api::post; @@ -17,10 +19,6 @@ use fil_actors_shared::fvm_ipld_bitfield::BitField; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::bytes_32; -use crate::state_manager::{StateManager, errors::*}; - -use super::MinerActorStateLoad as _; - impl StateManager where DB: Blockstore, @@ -180,6 +178,110 @@ fn generate_winning_post_sector_challenge( ) } +#[cfg(any(test, feature = "benchmark-private"))] +pub mod state_compute { + use crate::{ + blocks::Tipset, + chain::store::ChainStore, + db::{ + MemoryDB, + car::{AnyCar, ManyCar}, + }, + genesis::read_genesis_header, + interpreter::VMTrace, + networks::{ChainConfig, NetworkChain}, + state_manager::StateManager, + utils::net::{DownloadFileOption, download_file_with_cache}, + }; + use directories::ProjectDirs; + use std::{ + path::{Path, PathBuf}, + sync::{Arc, LazyLock}, + }; + use url::Url; + + static SNAPSHOT_CACHE_DIR: LazyLock = LazyLock::new(|| { + let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest"); + project_dir + .map(|d| d.cache_dir().to_path_buf()) + .unwrap_or_else(std::env::temp_dir) + .join("state_compute_snapshots") + }); + + pub async fn get_state_compute_snapshot( + chain: &NetworkChain, + epoch: i64, + ) -> anyhow::Result { + let url = Url::parse(&format!( + "https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/state_compute/{chain}_{epoch}.forest.car.zst" + ))?; + Ok( + download_file_with_cache(&url, &SNAPSHOT_CACHE_DIR, DownloadFileOption::NonResumable) + .await? + .path, + ) + } + + pub async fn prepare_state_compute( + chain: &NetworkChain, + snapshot: &Path, + warmup: bool, + ) -> anyhow::Result<(Arc>, Tipset)> { + let snap_car = AnyCar::try_from(snapshot)?; + let ts = snap_car.heaviest_tipset()?; + let db = Arc::new(ManyCar::new(MemoryDB::default()).with_read_only(snap_car)?); + let chain_config = Arc::new(ChainConfig::from_chain(chain)); + let genesis_header = + read_genesis_header(None, chain_config.genesis_bytes(&db).await?.as_deref(), &db) + .await?; + let chain_store = Arc::new(ChainStore::new( + db.clone(), + db.clone(), + db.clone(), + db.clone(), + chain_config, + genesis_header, + )?); + let state_manager = Arc::new(StateManager::new(chain_store.clone())?); + if warmup { + state_compute(state_manager.clone(), ts.clone()).await; + } + Ok((state_manager, ts)) + } + + pub async fn state_compute(state_manager: Arc>, ts: Tipset) { + state_manager + .compute_tipset_state(ts, crate::state_manager::NO_CALLBACK, VMTrace::NotTraced) + .await + .unwrap(); + } + + #[cfg(test)] + mod tests { + use super::*; + + #[tokio::test(flavor = "multi_thread")] + async fn state_compute_calibnet() { + let chain = NetworkChain::Calibnet; + let snapshot = get_state_compute_snapshot(&chain, 3111900).await.unwrap(); + let (sm, ts) = prepare_state_compute(&chain, &snapshot, false) + .await + .unwrap(); + state_compute(sm, ts).await; + } + + #[tokio::test(flavor = "multi_thread")] + async fn state_compute_mainnet() { + let chain = NetworkChain::Mainnet; + let snapshot = get_state_compute_snapshot(&chain, 5427431).await.unwrap(); + let (sm, ts) = prepare_state_compute(&chain, &snapshot, false) + .await + .unwrap(); + state_compute(sm, ts).await; + } + } +} + #[cfg(test)] mod test { use crate::shim::{address::Address, econ::TokenAmount, state_tree::ActorState};