From 568d2ae636562c430a7191b67b3c12eecd267353 Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 13 Jan 2025 14:47:53 -0300 Subject: [PATCH 1/7] add error alerts --- alerts/sender_with_alert.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index 3b70d57d7c..a6f6d934f1 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -94,10 +94,20 @@ do --rpc_url $RPC_URL \ --batcher_url $BATCHER_URL \ --network $NETWORK \ - --max_fee 4000000000000000 \ + --max_fee 0.04ether \ 2>&1) echo "$submit" + + submit_errors=$(echo "$submit" | grep -oE 'ERROR[^]]*]([^[]*)' | sed 's/^[^]]*]//;s/[[:space:]]*$//') + + # Loop through each error found and print with the custom message + while IFS= read -r error; do + if [[ -n "$error" ]]; then + slack_error_message="Error submitting proof to $NETWORK: $error" + send_slack_message "$slack_error_message" + fi + done <<< "$submit_errors" echo "Waiting $VERIFICATION_WAIT_TIME seconds for verification" sleep $VERIFICATION_WAIT_TIME From e31e43599d133501c38b99283fa26db1d2b50a4a Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 13 Jan 2025 16:39:11 -0300 Subject: [PATCH 2/7] displays number of verified proofs + fix devnet aligned service manager address --- alerts/sender_with_alert.sh | 14 +++++--------- batcher/aligned-sdk/src/sdk.rs | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index a6f6d934f1..27b2b6943e 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -69,10 +69,6 @@ function send_slack_message() { while true do - ## Remove Proof Data - rm -rf ./scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs/* - rm -rf ./aligned_verification_data/* - mkdir -p ./scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs ## Generate Proof @@ -153,7 +149,7 @@ do spent_amount_usd=$(echo "$spent_amount * $eth_usd" | bc | awk '{printf "%.2f", $1}') slack_messsage="" - verified=1 + verified=0 ## Verify Proofs echo "Verifying $REPETITIONS proofs $x != 0" @@ -170,17 +166,17 @@ do message="Proof verification failed for $proof [ ${batch_explorer_urls[@]} ]" echo "$message" send_pagerduty_alert "$message" - verified=0 # Some proofs failed, so we should not send the success message break elif echo "$verification" | grep -q verified; then + ((verified++)) echo "Proof verification succeeded for $proof" fi done - if [ $verified -eq 1 ]; then - slack_message="$REPETITIONS Proofs submitted and verified. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]" - else + if [ $verified -eq 0 ]; then slack_message="$REPETITIONS Proofs submitted but not verified. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]" + else + slack_message="$verified / $REPETITIONS Proofs submitted and verified. Spent amount: $spent_amount ETH ($ $spent_amount_usd) [ ${batch_explorer_urls[@]} ]" fi ## Send Update to Slack diff --git a/batcher/aligned-sdk/src/sdk.rs b/batcher/aligned-sdk/src/sdk.rs index f57b2aed56..e3c9f1f175 100644 --- a/batcher/aligned-sdk/src/sdk.rs +++ b/batcher/aligned-sdk/src/sdk.rs @@ -284,7 +284,7 @@ pub fn get_payment_service_address(network: Network) -> ethers::types::H160 { pub fn get_aligned_service_manager_address(network: Network) -> ethers::types::H160 { match network { - Network::Devnet => H160::from_str("0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8").unwrap(), + Network::Devnet => H160::from_str("0x851356ae760d987E095750cCeb3bC6014560891C").unwrap(), Network::Holesky => H160::from_str("0x58F280BeBE9B34c9939C3C39e0890C81f163B623").unwrap(), Network::HoleskyStage => { H160::from_str("0x9C5231FC88059C086Ea95712d105A2026048c39B").unwrap() @@ -499,7 +499,9 @@ async fn _is_proof_verified( eth_rpc_provider: Provider, ) -> Result { let contract_address = get_aligned_service_manager_address(network); + println!("aligned service manager: {:?}", contract_address); let payment_service_addr = get_payment_service_address(network); + println!("batcher_payment_service: {:?}", payment_service_addr); // All the elements from the merkle proof have to be concatenated let merkle_proof: Vec = aligned_verification_data From 5ac38d9c046b38e5b5defd31e012c258f26eb56e Mon Sep 17 00:00:00 2001 From: PatStiles Date: Mon, 13 Jan 2025 16:41:41 -0300 Subject: [PATCH 3/7] nits --- batcher/aligned-sdk/src/sdk.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/batcher/aligned-sdk/src/sdk.rs b/batcher/aligned-sdk/src/sdk.rs index e3c9f1f175..e933ab6f8d 100644 --- a/batcher/aligned-sdk/src/sdk.rs +++ b/batcher/aligned-sdk/src/sdk.rs @@ -499,9 +499,7 @@ async fn _is_proof_verified( eth_rpc_provider: Provider, ) -> Result { let contract_address = get_aligned_service_manager_address(network); - println!("aligned service manager: {:?}", contract_address); let payment_service_addr = get_payment_service_address(network); - println!("batcher_payment_service: {:?}", payment_service_addr); // All the elements from the merkle proof have to be concatenated let merkle_proof: Vec = aligned_verification_data From 46a885a20f1a28bb67f9261ce5fc70619f1500bb Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:20:25 -0300 Subject: [PATCH 4/7] feat: dont run if failed connection --- alerts/sender_with_alert.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index c36437bbd5..6386db9cd6 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -89,8 +89,12 @@ do ## Generate Proof nonce=$(aligned get-user-nonce --batcher_url $BATCHER_URL --user_addr $SENDER_ADDRESS 2>&1 | awk '{print $9}') + if ! [[ "$nonce" =~ ^[0-9]+$ ]]; then + echo "Failed getting nonce value, exiting" + exit 1 + fi x=$((nonce + 1)) # So we don't have any issues with nonce = 0 - echo "Generating proof $x != 0" + echo "Generating proof $x != 0, nonce: $nonce" go run ./scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go $x ## Send Proof From ed538643ac562bdf159791129626b45835db72c4 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:20:52 -0300 Subject: [PATCH 5/7] chore: style --- alerts/sender_with_alert.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index 6386db9cd6..bebad54a44 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -90,9 +90,10 @@ do ## Generate Proof nonce=$(aligned get-user-nonce --batcher_url $BATCHER_URL --user_addr $SENDER_ADDRESS 2>&1 | awk '{print $9}') if ! [[ "$nonce" =~ ^[0-9]+$ ]]; then - echo "Failed getting nonce value, exiting" + echo "Failed getting user nonce, exiting" exit 1 fi + x=$((nonce + 1)) # So we don't have any issues with nonce = 0 echo "Generating proof $x != 0, nonce: $nonce" go run ./scripts/test_files/gnark_groth16_bn254_infinite_script/cmd/main.go $x From 6c6f74532f2170df998ea78234ecc4c4e15e91e9 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Fri, 17 Jan 2025 18:14:17 -0300 Subject: [PATCH 6/7] fix: revert removal of remove proof data --- alerts/sender_with_alert.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index bebad54a44..f8f7e07114 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -85,6 +85,10 @@ function send_slack_message() { while true do + ## Remove Proof Data + rm -rf ./scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs/* + rm -rf ./aligned_verification_data/* + mkdir -p ./scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs ## Generate Proof From da5aff607b11b6086c055a133a32985018da4d34 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:48:55 -0300 Subject: [PATCH 7/7] fix: retry send tasks in case of error and send notification to slack --- alerts/sender_with_alert.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/alerts/sender_with_alert.sh b/alerts/sender_with_alert.sh index f8f7e07114..d01ddddb78 100755 --- a/alerts/sender_with_alert.sh +++ b/alerts/sender_with_alert.sh @@ -93,9 +93,11 @@ do ## Generate Proof nonce=$(aligned get-user-nonce --batcher_url $BATCHER_URL --user_addr $SENDER_ADDRESS 2>&1 | awk '{print $9}') + echo $nonce if ! [[ "$nonce" =~ ^[0-9]+$ ]]; then - echo "Failed getting user nonce, exiting" - exit 1 + echo "Failed getting user nonce, retrying in 10 seconds" + sleep 10 + continue fi x=$((nonce + 1)) # So we don't have any issues with nonce = 0 @@ -123,12 +125,21 @@ do submit_errors=$(echo "$submit" | grep -oE 'ERROR[^]]*]([^[]*)' | sed 's/^[^]]*]//;s/[[:space:]]*$//') # Loop through each error found and print with the custom message + is_error=0 while IFS= read -r error; do if [[ -n "$error" ]]; then slack_error_message="Error submitting proof to $NETWORK: $error" send_slack_message "$slack_error_message" + is_error=1 fi done <<< "$submit_errors" + + if [ $is_error -eq 1 ]; then + echo "Error submitting proofs to $NETWORK, retrying in 60 seconds" + send_slack_message "Error submitting proofs to $NETWORK, retrying in 60 seconds" + sleep 60 + continue + fi echo "Waiting $VERIFICATION_WAIT_TIME seconds for verification" sleep $VERIFICATION_WAIT_TIME @@ -179,7 +190,7 @@ do spent_amount_usd=$(echo "$spent_amount * $eth_usd" | bc | awk '{printf "%.2f", $1}') slack_messsage="" - verified=0 + verified=1 ## Verify Proofs echo "Verifying $REPETITIONS proofs $x != 0" @@ -196,9 +207,9 @@ do message="Proof verification failed for $proof [ ${batch_explorer_urls[@]} ]" echo "$message" send_pagerduty_alert "$message" + verified=0 # Some proofs failed, so we should not send the success message break elif echo "$verification" | grep -q verified; then - ((verified++)) echo "Proof verification succeeded for $proof" fi done