multiple requests from all endpoints

Signed-off-by: Uncle Stinky <uncle.stinky@ghostchain.io>

bin/alice

@@ -0,0 +1,12 @@
+#!/bin/sh
+#--offchain-worker never \
+
+rm -rf /tmp/alice
+./ghost \
+    --base-path /tmp/alice \
+    --chain local \
+    --alice \
+    --port 30333 \
+    --rpc-port 9945 \
+    --node-key 0000000000000000000000000000000000000000000000000000000000000001 \
+    --validator

bin/bob

@@ -0,0 +1,13 @@
+#!/bin/sh
+#--offchain-worker never \
+
+rm -rf /tmp/bob
+./ghost \
+    --base-path /tmp/bob \
+    --chain local \
+    --bob \
+    --port 30334 \
+    --rpc-port 9934 \
+    --validator \
+    --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp \
+    --node-key 0000000000000000000000000000000000000000000000000000000000000002

bin/dave

@@ -0,0 +1,13 @@
+#!/bin/sh
+#--offchain-worker never \
+
+rm -rf /tmp/dave
+./ghost \
+    --base-path /tmp/dave \
+    --chain local \
+    --dave \
+    --port 30335 \
+    --rpc-port 9935 \
+    --validator \
+    --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp \
+    --node-key 0000000000000000000000000000000000000000000000000000000000000003

bin/dddd

@@ -0,0 +1,2 @@
+ sed -i '/#\[no_mangle\]/d' $(echo $CARGO_HOME)/git/checkouts/polkadot-sdk-dee0edd6eefa0594/b401690/substrate/primitives/io/src/lib.rs
+

pallets/slow-clap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ghost-slow-clap"
-version = "0.4.10"
+version = "0.4.11"
 description = "Applause protocol for the EVM bridge"
 license.workspace = true
 authors.workspace = true

pallets/slow-clap/src/lib.rs

@@ -1,8 +1,6 @@
 // Ensure we're `no_std` when compiling for Wasm.
 #![cfg_attr(not(feature = "std"), no_std)]
 
-use core::usize;
-
 use codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
 use serde::{Deserialize, Deserializer};
@@ -26,11 +24,11 @@ use sp_core::H256;
 use sp_runtime::{
     offchain::{
         self as rt_offchain,
+        http::PendingRequest,
         storage::StorageValueRef,
         storage_lock::{StorageLock, Time},
-        HttpError,
     },
-    traits::{BlockNumberProvider, Convert, Saturating, TrailingZeroInput},
+    traits::{BlockNumberProvider, Convert, Saturating},
     Perbill, RuntimeAppPublic, RuntimeDebug,
 };
 use sp_staking::{
@@ -162,11 +160,13 @@ pub struct PreparedApplause<AccountId, NetworkId, Balance> {
 enum OffchainErr<NetworkId> {
     HttpJsonParsingError,
     HttpBytesParsingError,
-    HttpRequestError(HttpError),
-    RequestUncompleted,
-    HttpResponseNotOk(u16),
     ErrorInEvmResponse,
     NoStoredNetworks,
+    NoRequestsSent,
+    EmptyResponses,
+    NotValidator,
+    DifferentEvmResponseTypes,
+    UnparsableRequestBody(Vec<u8>),
     NoEndpointAvailable(NetworkId),
     StorageRetrievalError(NetworkId),
     UtxoNotImplemented(NetworkId),
@@ -183,31 +183,20 @@ impl<NetworkId: core::fmt::Debug> core::fmt::Debug for OffchainErr<NetworkId> {
             OffchainErr::HttpBytesParsingError => {
                 write!(fmt, "Failed to parse evm response as bytes.")
             }
-            OffchainErr::HttpRequestError(http_error) => match http_error {
-                HttpError::DeadlineReached => write!(
-                    fmt,
-                    "Requested action couldn't been completed within a deadline."
-                ),
-                HttpError::IoError => {
-                    write!(fmt, "There was an IO error while processing the request.")
-                }
-                HttpError::Invalid => {
-                    write!(fmt, "The ID of the request is invalid in this context.")
-                }
-            },
             OffchainErr::StorageRetrievalError(ref network_id) => write!(
                 fmt,
                 "Storage value found for network #{:?} but it's undecodable.",
                 network_id
             ),
-            OffchainErr::RequestUncompleted => write!(fmt, "Failed to complete request."),
-            OffchainErr::HttpResponseNotOk(code) => {
-                write!(fmt, "Http response returned code {:?}.", code)
-            }
             OffchainErr::ErrorInEvmResponse => write!(fmt, "Error in evm reponse."),
             OffchainErr::NoStoredNetworks => {
                 write!(fmt, "No networks stored for the offchain slow claps.")
             }
+            OffchainErr::NoRequestsSent => write!(fmt, "Could not send a request to any available RPC ednpoint."),
+            OffchainErr::EmptyResponses => write!(fmt, "No responses to be used by the offchain worker."),
+            OffchainErr::NotValidator => write!(fmt, "Not a validator to broadcast slow claps"),
+            OffchainErr::DifferentEvmResponseTypes => write!(fmt, "Different endpoints returned conflicting response types."),
+            OffchainErr::UnparsableRequestBody(ref bytes) => write!(fmt, "Could not get valid utf8 request body from bytes: {:?}", bytes),
             OffchainErr::NoEndpointAvailable(ref network_id) => write!(
                 fmt,
                 "No RPC endpoint available for network #{:?}.",
@@ -853,55 +842,55 @@ impl<T: Config> Pallet<T> {
             network_data.default_endpoints.clone(),
         );
 
-        let random_seed = sp_io::offchain::random_seed();
+        let rpc_endpoints = if !stored_endpoints.is_empty() {
-        let random_number = <u32>::decode(&mut TrailingZeroInput::new(random_seed.as_ref()))
-            .expect("input is padded with zeroes; qed");
-
-        let random_index = (random_number as usize)
-            .checked_rem(stored_endpoints.len())
-            .unwrap_or_default();
-
-        let endpoints = if !stored_endpoints.is_empty() {
             &stored_endpoints
         } else {
             &network_data.default_endpoints
         };
 
-        let rpc_endpoint = endpoints
+        if rpc_endpoints.len() == 0 {
-            .get(random_index)
+            return Err(OffchainErr::NoEndpointAvailable(network_id));
-            .ok_or(OffchainErr::NoEndpointAvailable(network_id))?;
+        }
 
         let (from_block, to_block): (u64, u64) = StorageValueRef::persistent(&block_number_key)
             .get()
             .map_err(|_| OffchainErr::StorageRetrievalError(network_id))?
             .unwrap_or_default();
 
+        match network_data.network_type {
+            NetworkType::Evm => {
                 let request_body = if from_block < to_block.saturating_sub(1) {
-            Self::prepare_request_body_for_latest_transfers(
+                    Self::prepare_evm_request_body_for_latest_transfers(
                         from_block,
                         to_block.saturating_sub(1),
                         network_data,
                     )
                 } else {
-            Self::prepare_request_body_for_latest_block(network_data)
+                    Self::prepare_evm_request_body_for_latest_block(network_data)
                 };
 
-        let response_bytes = Self::fetch_from_remote(&rpc_endpoint, &request_body)?;
+                let pending_requests_metadata = Self::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
+                let parsed_evm_responses = Self::fetch_multiple_evm_from_remote(pending_requests_metadata)
+                    .iter()
+                    .filter_map(|response_bytes| {
+                        let parsed_evm_response = Self::parse_evm_response(response_bytes).ok()?;
+                        Some(parsed_evm_response)
+                    })
+                    .collect::<Vec<EvmResponseType>>();
+
+                Self::check_evm_responses_correctness(&parsed_evm_responses)?;
+                let parsed_evm_response = Self::get_balanced_evm_response(&parsed_evm_responses)?;
 
-        match network_data.network_type {
-            NetworkType::Evm => {
-                let parsed_evm_response = Self::parse_evm_response(&response_bytes)?;
                 let new_block_range = match parsed_evm_response {
                     EvmResponseType::BlockNumber(new_evm_block) if from_block.le(&to_block) => {
-                        let estimated_block =
+                        // stay in the range of block distance
-                            new_evm_block.saturating_sub(network_data.finality_delay);
+                        let estimated_block = new_evm_block.saturating_sub(network_data.finality_delay);
-                        let adjusted_block =
+                        match from_block {
-                            Self::adjust_to_block(estimated_block, from_block, max_block_distance);
+                            0 => (estimated_block, estimated_block),
-
+                            start_block => {
-                        if from_block == 0 {
+                                let block_deviation = start_block.saturating_add(max_block_distance);
-                            (estimated_block, estimated_block)
+                                (start_block, estimated_block.min(block_deviation))
-                        } else {
+                            }
-                            (from_block, adjusted_block)
                         }
                     }
                     _ => (to_block, to_block),
@@ -911,15 +900,14 @@ impl<T: Config> Pallet<T> {
 
                 log::info!(
                     target: LOG_TARGET,
-                    "👻 Slow Clap #{:?} stored block #{:?} for network {:?}",
+                    "👻 Offchain worker #{:?} stored block #{:?} for network {:?}",
                     block_number,
                     new_block_range.0,
                     network_id,
                 );
 
                 if !sp_io::offchain::is_validator() {
-                    log::info!(target: LOG_TARGET, "👻 Not a validator; no transactions available");
+                    return Err(OffchainErr::NotValidator);
-                    return Ok(());
                 }
 
                 for (authority_index, authority_key) in Self::local_authorities(&session_index) {
@@ -939,22 +927,6 @@ impl<T: Config> Pallet<T> {
         }
     }
 
-    fn adjust_to_block(estimated_block: u64, from_block: u64, max_block_distance: u64) -> u64 {
-        let fallback_value = from_block
-            .saturating_add(max_block_distance)
-            .min(estimated_block);
-
-        estimated_block
-            .checked_sub(from_block)
-            .map(|current_distance| {
-                current_distance
-                    .le(&max_block_distance)
-                    .then_some(estimated_block)
-            })
-            .flatten()
-            .unwrap_or(fallback_value)
-    }
-
     fn local_authorities(
         session_index: &SessionIndex,
     ) -> impl Iterator<Item = (u32, T::AuthorityId)> {
@@ -973,35 +945,151 @@ impl<T: Config> Pallet<T> {
             })
     }
 
-    fn fetch_from_remote(rpc_endpoint: &[u8], request_body: &[u8]) -> OffchainResult<T, Vec<u8>> {
+    fn prepare_pending_evm_requests(
-        let rpc_endpoint_str =
+        rpc_endpoints: &Vec<Vec<u8>>,
-            core::str::from_utf8(rpc_endpoint).expect("rpc endpoint valid str; qed");
+        request_body: &[u8],
-        let request_body_str =
+    ) -> OffchainResult<T, Vec<(PendingRequest, String)>> {
-            core::str::from_utf8(request_body).expect("request body valid str: qed");
+        let mut pending_requests_metadata = Vec::new();
+        let request_body_str = core::str::from_utf8(request_body)
+            .map_err(|_| OffchainErr::UnparsableRequestBody(request_body.to_vec()))?;
 
         let deadline = sp_io::offchain::timestamp()
             .add(rt_offchain::Duration::from_millis(FETCH_TIMEOUT_PERIOD));
 
-        let pending = rt_offchain::http::Request::post(&rpc_endpoint_str, vec![request_body_str])
+        for rpc_endpoint in rpc_endpoints.iter() {
+            let rpc_endpoint_str = match core::str::from_utf8(rpc_endpoint) {
+                Ok(rpc_endpoint_str) => rpc_endpoint_str,
+                Err(_) => {
+                    log::info!(target: LOG_TARGET, "👻 Could not get valid utf8 rpc endpoint from bytes: {:?}", rpc_endpoint);
+                    continue;
+                }
+            };
+
+            match rt_offchain::http::Request::post(&rpc_endpoint_str, vec![request_body_str])
                 .add_header("Accept", "application/json")
                 .add_header("Content-Type", "application/json")
                 .deadline(deadline)
                 .send() 
-            .map_err(|err| OffchainErr::HttpRequestError(err))?;
+            {
+                Ok(pending) => pending_requests_metadata.push((pending, rpc_endpoint_str.to_string())),
+                Err(_) => log::info!(target: LOG_TARGET, "👻 Request skipped: failed to send request to \"{}\"", rpc_endpoint_str),
+            }
+        }
 
-        let response = pending
+        if pending_requests_metadata.len() == 0 {
-            .try_wait(deadline)
+            return Err(OffchainErr::NoRequestsSent);
-            .map_err(|_| OffchainErr::RequestUncompleted)?
+        }
-            .map_err(|_| OffchainErr::RequestUncompleted)?;
+
+        log::info!(target: LOG_TARGET, "👻 Requests sent {} out of {}", pending_requests_metadata.len(), rpc_endpoints.len());
+
+        Ok(pending_requests_metadata)
+    }
+
+    fn get_balanced_evm_response(parsed_evm_responses: &Vec<EvmResponseType>) -> OffchainResult<T, EvmResponseType> {
+        let first_evm_response = parsed_evm_responses
+            .first()
+            .ok_or(OffchainErr::EmptyResponses)?;
+
+        let result = match first_evm_response {
+            EvmResponseType::BlockNumber(_) => {
+                let mut block_numbers = parsed_evm_responses
+                    .iter()
+                    .enumerate()
+                    .filter_map(|(index, response)| match response {
+                        EvmResponseType::BlockNumber(block) => Some((index as u32, *block)),
+                        EvmResponseType::TransactionLogs(_) => None,
+                    })
+                    .collect();
+                let median_value = Self::calculate_median_value(&mut block_numbers);
+                EvmResponseType::BlockNumber(median_value)
+            }
+            EvmResponseType::TransactionLogs(_) => {
+                let mut btree_map = BTreeMap::new();
+                parsed_evm_responses
+                    .iter()
+                    .for_each(|response| {
+                        if let EvmResponseType::TransactionLogs(logs) = response {
+                            let mut inner_logs = logs.clone();
+                            inner_logs.sort_by_key(|l| l.block_number);
+                            *btree_map.entry(inner_logs).or_insert(0) += 1;
+                        }
+                    });
+
+                let best_logs = btree_map.into_iter()
+                    .max_by_key(|&(_, count)| count)
+                    .map(|(v, _)| v.clone())
+                    .ok_or(OffchainErr::EmptyResponses)?;
+
+                EvmResponseType::TransactionLogs(best_logs)
+            }
+        };
+
+        Ok(result)
+    }
+
+    fn check_evm_responses_correctness(parsed_evm_responses: &Vec<EvmResponseType>) -> OffchainResult<T, ()> {
+        let first_evm_response = parsed_evm_responses
+            .first()
+            .ok_or(OffchainErr::EmptyResponses)?;
+        let first_evm_response_type = core::mem::discriminant(first_evm_response);
+
+        if !parsed_evm_responses
+            .iter()
+            .all(|parsed_evm_response| {
+                core::mem::discriminant(parsed_evm_response) == first_evm_response_type
+            }) 
+        {
+            return Err(OffchainErr::DifferentEvmResponseTypes);
+        }
+
+        Ok(())
+    }
+
+    fn fetch_multiple_evm_from_remote(pending_requests_metadata: Vec<(PendingRequest, String)>) -> Vec<Vec<u8>> {
+        let pending_requests = pending_requests_metadata.iter()
+            .map(|(pending_request, _)| PendingRequest { id: pending_request.id })
+            .collect::<Vec<PendingRequest>>();
+
+        let deadline = sp_io::offchain::timestamp()
+            .add(rt_offchain::Duration::from_millis(FETCH_TIMEOUT_PERIOD));
+
+        PendingRequest::try_wait_all(pending_requests, Some(deadline))
+            .into_iter()
+            .enumerate()
+            .filter_map(|(index, pending_request)| {
+                let url = pending_requests_metadata.get(index)
+                    .map(|(_, url)| url.clone())
+                    .unwrap_or_default();
+
+                // handle request-level errors (transport/connection failures)
+                let request_result = match pending_request {
+                    Ok(request) => request,
+                    Err(err) => {
+                        log::info!(target: LOG_TARGET, "👻 Request skipped; request to \"{}\" failed: {:?}", url, err);
+                        return None;
+                    }
+                };
+
+                // handle response-level errors (HTTP/protocol errors)
+                let response = match request_result {
+                    Ok(response) => response,
+                    Err(err) => {
+                        log::info!(target: LOG_TARGET, "👻 Response skipped from \"{}\" error: {:?}", url, err);
+                        return None;
+                    }
+                };
 
                 if response.code != 200 {
-            return Err(OffchainErr::HttpResponseNotOk(response.code));
+                    log::info!(target: LOG_TARGET, "👻 Response skipped from \"{}\": status {}", url, response.code);
+                    return None;
                 }
 
-        Ok(response.body().collect::<Vec<u8>>())
+                Some(response.body().collect::<Vec<u8>>())
+            })
+            .collect()
     }
 
-    fn prepare_request_body_for_latest_block(network_data: &NetworkData) -> Vec<u8> {
+    fn prepare_evm_request_body_for_latest_block(network_data: &NetworkData) -> Vec<u8> {
         match network_data.network_type {
             NetworkType::Evm => {
                 b"{\"id\":0,\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\"}".to_vec()
@@ -1010,7 +1098,7 @@ impl<T: Config> Pallet<T> {
         }
     }
 
-    fn prepare_request_body_for_latest_transfers(
+    fn prepare_evm_request_body_for_latest_transfers(
         from_block: u64,
         to_block: u64,
         network_data: &NetworkData,

pallets/slow-clap/src/tests.rs

@@ -137,16 +137,6 @@ fn bitmap_operations_correct() {
     assert_eq!(ones, 11);
 }
 
-#[test]
-fn should_correctly_adjust_to_block() {
-    assert_eq!(SlowClap::adjust_to_block(420, 69, 1337), 420);
-    assert_eq!(SlowClap::adjust_to_block(420, 1337, 69), 420);
-    assert_eq!(SlowClap::adjust_to_block(1337, 420, 69), 489);
-    assert_eq!(SlowClap::adjust_to_block(1337, 69, 420), 489);
-    assert_eq!(SlowClap::adjust_to_block(69, 1337, 420), 69);
-    assert_eq!(SlowClap::adjust_to_block(69, 420, 1337), 69);
-}
-
 #[test]
 fn request_body_is_correct_for_get_block_number() {
     let (offchain, _) = TestOffchainExt::new();
@@ -155,7 +145,7 @@ fn request_body_is_correct_for_get_block_number() {
 
     t.execute_with(|| {
         let network_data = prepare_evm_network(Some(1), None);
-        let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
         assert_eq!(
             core::str::from_utf8(&request_body).unwrap(),
             r#"{"id":0,"jsonrpc":"2.0","method":"eth_blockNumber"}"#
@@ -173,7 +163,7 @@ fn request_body_is_correct_for_get_logs() {
         let from_block: u64 = 420;
         let to_block: u64 = 1337;
         let network_data = prepare_evm_network(Some(1), None);
-        let request_body = SlowClap::prepare_request_body_for_latest_transfers(
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
             from_block, to_block, &network_data);
         assert_eq!(core::str::from_utf8(&request_body).unwrap(),
             r#"{"id":0,"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock":"0x1a4","toBlock":"0x539","address":"0x4d224452801ACEd8B2F0aebE155379bb5D594381","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"]}]}"#,
@@ -190,10 +180,14 @@ fn should_make_http_call_for_block_number() {
     evm_block_response(&mut state.write());
 
     let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
-        let rpc_endpoint = get_rpc_endpoint();
+        let rpc_endpoints = get_rpc_endpoints();
         let network_data = prepare_evm_network(Some(1), None);
-        let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
-        let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
+
+        let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
+        let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
+        let raw_response = raw_responses.first().unwrap();
+
         assert_eq!(raw_response.len(), 45usize); // precalculated
         Ok(())
     });
@@ -210,15 +204,19 @@ fn should_make_http_call_for_logs() {
     let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
         let from_block: u64 = 20335770;
         let to_block: u64 = 20335858;
-        let rpc_endpoint = get_rpc_endpoint();
+        let rpc_endpoints = get_rpc_endpoints();
 
         let network_data = prepare_evm_network(Some(1), None);
-        let request_body = SlowClap::prepare_request_body_for_latest_transfers(
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
             from_block,
             to_block,
             &network_data,
         );
-        let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
+
+        let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
+        let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
+        let raw_response = raw_responses.first().unwrap();
+
         assert_eq!(raw_response.len(), 1805); // precalculated
         Ok(())
     });
@@ -233,11 +231,14 @@ fn should_make_http_call_and_parse_block_number() {
     evm_block_response(&mut state.write());
 
     let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
-        let rpc_endpoint = get_rpc_endpoint();
+        let rpc_endpoints = get_rpc_endpoints();
         let network_data = prepare_evm_network(Some(1), None);
 
-        let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
-        let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
+        let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
+        let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
+        let raw_response = raw_responses.first().unwrap();
+
         let evm_block_number = SlowClap::parse_evm_response(&raw_response).map(
             |parsed_response| match parsed_response {
                 EvmResponseType::BlockNumber(block_number) => block_number,
@@ -263,19 +264,22 @@ fn should_make_http_call_and_parse_logs() {
     evm_logs_response(&mut state.write());
 
     let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
-        let rpc_endpoint = get_rpc_endpoint();
+        let rpc_endpoints = get_rpc_endpoints();
 
         let from_block: u64 = 20335770;
         let to_block: u64 = 20335858;
 
         let network_data = prepare_evm_network(None, None);
-        let request_body = SlowClap::prepare_request_body_for_latest_transfers(
+        let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
             from_block,
             to_block,
             &network_data,
         );
 
-        let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
+        let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
+        let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
+        let raw_response = raw_responses.first().unwrap();
+
         match SlowClap::parse_evm_response(&raw_response)? {
             EvmResponseType::BlockNumber(_) => assert_eq!(1, 0), // force break
             EvmResponseType::TransactionLogs(evm_logs) => assert_eq!(evm_logs.len(), 2),
@@ -1355,6 +1359,12 @@ fn should_split_commit_slash_between_active_validators() {
     });
 }
 
+// TODO: add tests
+// 1. prepare_pending_evm_requests fails fully and partially
+// 2. fetch_multiple_evm_from_remote when they give different results
+// 3. check_evm_responses_correctness
+// 4. get_balanced_evm_response
+
 fn assert_clapped_amount(
     session_index: &SessionIndex,
     unique_hash: &H256,
@@ -1571,6 +1581,7 @@ fn get_mocked_metadata() -> (H256, u64, u64, u64) {
 
 fn evm_block_response(state: &mut testing::OffchainState) {
     let expected_body = br#"{"id":0,"jsonrpc":"2.0","method":"eth_blockNumber"}"#.to_vec();
+
     state.expect_request(testing::PendingRequest {
         method: "POST".into(),
         uri: "https://rpc.endpoint.network.com".into(),
@@ -1579,6 +1590,19 @@ fn evm_block_response(state: &mut testing::OffchainState) {
             ("Content-Type".to_string(), "application/json".to_string()),
         ],
         response: Some(b"{\"id\":0,\"jsonrpc\":\"2.0\",\"result\":\"0x1364c81\"}".to_vec()),
+        body: expected_body.clone(),
+        sent: true,
+        ..Default::default()
+    });
+
+    state.expect_request(testing::PendingRequest {
+        method: "POST".into(),
+        uri: "https://other.endpoint.network.com".into(),
+        headers: vec![
+            ("Accept".to_string(), "application/json".to_string()),
+            ("Content-Type".to_string(), "application/json".to_string()),
+        ],
+        response: Some(b"{\"id\":0,\"jsonrpc\":\"2.0\",\"result\":\"0x1364c81\"}".to_vec()),
         body: expected_body,
         sent: true,
         ..Default::default()
@@ -1636,6 +1660,23 @@ fn evm_logs_response(state: &mut testing::OffchainState) {
             ("Accept".to_string(), "application/json".to_string()),
             ("Content-Type".to_string(), "application/json".to_string()),
         ],
+        body: expected_body.clone(),
+        response: Some(expected_response.clone()),
+        sent: true,
+        ..Default::default()
+    });
+
+    state.expect_request(testing::PendingRequest {
+        method: "POST".into(),
+        uri: "https://other.endpoint.network.com".into(),
+        headers: vec![
+            ("Accept".to_string(), "application/json".to_string()),
+            ("Content-Type".to_string(), "application/json".to_string()),
+        ],
+        response_headers: vec![
+            ("Accept".to_string(), "application/json".to_string()),
+            ("Content-Type".to_string(), "application/json".to_string()),
+        ],
         body: expected_body,
         response: Some(expected_response),
         sent: true,