ghostchain/ghost-node
30
0
Fork 3
Code Issues 5 Pull Requests 24 Actions Packages Projects Releases Wiki Activity

multiple requests from all endpoints

Browse Source 
Signed-off-by: Uncle Stinky <uncle.stinky@ghostchain.io>
This commit is contained in:
Uncle Stinky 2026-01-31 18:10:40 +03:00
parent 3c7e51c0f8
commit 90c82f73cb
Signed by: st1nky
GPG Key ID: 016064BD97603B40
9 changed files with 291 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
bin/alice Executable file
View File

@ -0,0 +1,12 @@
#!/bin/sh
#--offchain-worker never \
rm -rf /tmp/alice
./ghost \
--base-path /tmp/alice \
--chain local \
--alice \
--port 30333 \
--rpc-port 9945 \
--node-key 0000000000000000000000000000000000000000000000000000000000000001 \
--validator

13
bin/bob Executable file
View File

@ -0,0 +1,13 @@
#!/bin/sh
#--offchain-worker never \
rm -rf /tmp/bob
./ghost \
--base-path /tmp/bob \
--chain local \
--bob \
--port 30334 \
--rpc-port 9934 \
--validator \
--bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp \
--node-key 0000000000000000000000000000000000000000000000000000000000000002

13
bin/dave Executable file
View File

@ -0,0 +1,13 @@
#!/bin/sh
#--offchain-worker never \
rm -rf /tmp/dave
./ghost \
--base-path /tmp/dave \
--chain local \
--dave \
--port 30335 \
--rpc-port 9935 \
--validator \
--bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWEyoppNCUx8Yx66oV9fJnriXwCcXwDDUA2kj6vnc6iDEp \
--node-key 0000000000000000000000000000000000000000000000000000000000000003

2
bin/dddd Normal file
View File

@ -0,0 +1,2 @@
sed -i '/#\[no_mangle\]/d' $(echo $CARGO_HOME)/git/checkouts/polkadot-sdk-dee0edd6eefa0594/b401690/substrate/primitives/io/src/lib.rs

BIN
bin/ghost Executable file
View File

Binary file not shown.

BIN
bin/ghost_old Executable file
View File

Binary file not shown.

2
pallets/slow-clap/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "ghost-slow-clap"
version = "0.4.10"
version = "0.4.11"
description = "Applause protocol for the EVM bridge"
license.workspace = true
authors.workspace = true

256
pallets/slow-clap/src/lib.rs
View File

@ -1,8 +1,6 @@
// Ensure we're `no_std` when compiling for Wasm.
#![cfg_attr(not(feature = "std"), no_std)]
use core::usize;
use codec::{Decode, Encode, MaxEncodedLen};
use scale_info::TypeInfo;
use serde::{Deserialize, Deserializer};
@ -26,11 +24,11 @@ use sp_core::H256;
use sp_runtime::{
offchain::{
self as rt_offchain,
http::PendingRequest,
storage::StorageValueRef,
storage_lock::{StorageLock, Time},
HttpError,
},
traits::{BlockNumberProvider, Convert, Saturating, TrailingZeroInput},
traits::{BlockNumberProvider, Convert, Saturating},
Perbill, RuntimeAppPublic, RuntimeDebug,
};
use sp_staking::{
@ -162,11 +160,13 @@ pub struct PreparedApplause<AccountId, NetworkId, Balance> {
enum OffchainErr<NetworkId> {
HttpJsonParsingError,
HttpBytesParsingError,
HttpRequestError(HttpError),
RequestUncompleted,
HttpResponseNotOk(u16),
ErrorInEvmResponse,
NoStoredNetworks,
NoRequestsSent,
EmptyResponses,
NotValidator,
DifferentEvmResponseTypes,
UnparsableRequestBody(Vec<u8>),
NoEndpointAvailable(NetworkId),
StorageRetrievalError(NetworkId),
UtxoNotImplemented(NetworkId),
@ -183,31 +183,20 @@ impl<NetworkId: core::fmt::Debug> core::fmt::Debug for OffchainErr<NetworkId> {
OffchainErr::HttpBytesParsingError => {
write!(fmt, "Failed to parse evm response as bytes.")
}
OffchainErr::HttpRequestError(http_error) => match http_error {
HttpError::DeadlineReached => write!(
fmt,
"Requested action couldn't been completed within a deadline."
),
HttpError::IoError => {
write!(fmt, "There was an IO error while processing the request.")
}
HttpError::Invalid => {
write!(fmt, "The ID of the request is invalid in this context.")
}
},
OffchainErr::StorageRetrievalError(ref network_id) => write!(
fmt,
"Storage value found for network #{:?} but it's undecodable.",
network_id
),
OffchainErr::RequestUncompleted => write!(fmt, "Failed to complete request."),
OffchainErr::HttpResponseNotOk(code) => {
write!(fmt, "Http response returned code {:?}.", code)
}
OffchainErr::ErrorInEvmResponse => write!(fmt, "Error in evm reponse."),
OffchainErr::NoStoredNetworks => {
write!(fmt, "No networks stored for the offchain slow claps.")
}
OffchainErr::NoRequestsSent => write!(fmt, "Could not send a request to any available RPC ednpoint."),
OffchainErr::EmptyResponses => write!(fmt, "No responses to be used by the offchain worker."),
OffchainErr::NotValidator => write!(fmt, "Not a validator to broadcast slow claps"),
OffchainErr::DifferentEvmResponseTypes => write!(fmt, "Different endpoints returned conflicting response types."),
OffchainErr::UnparsableRequestBody(ref bytes) => write!(fmt, "Could not get valid utf8 request body from bytes: {:?}", bytes),
OffchainErr::NoEndpointAvailable(ref network_id) => write!(
fmt,
"No RPC endpoint available for network #{:?}.",
@ -853,55 +842,55 @@ impl<T: Config> Pallet<T> {
network_data.default_endpoints.clone(),
);
let random_seed = sp_io::offchain::random_seed();
let random_number = <u32>::decode(&mut TrailingZeroInput::new(random_seed.as_ref()))
.expect("input is padded with zeroes; qed");
let random_index = (random_number as usize)
.checked_rem(stored_endpoints.len())
.unwrap_or_default();
let endpoints = if !stored_endpoints.is_empty() {
let rpc_endpoints = if !stored_endpoints.is_empty() {
&stored_endpoints
} else {
&network_data.default_endpoints
};
let rpc_endpoint = endpoints
.get(random_index)
.ok_or(OffchainErr::NoEndpointAvailable(network_id))?;
if rpc_endpoints.len() == 0 {
return Err(OffchainErr::NoEndpointAvailable(network_id));
}
let (from_block, to_block): (u64, u64) = StorageValueRef::persistent(&block_number_key)
.get()
.map_err(|_| OffchainErr::StorageRetrievalError(network_id))?
.unwrap_or_default();
match network_data.network_type {
NetworkType::Evm => {
let request_body = if from_block < to_block.saturating_sub(1) {
Self::prepare_request_body_for_latest_transfers(
Self::prepare_evm_request_body_for_latest_transfers(
from_block,
to_block.saturating_sub(1),
network_data,
)
} else {
Self::prepare_request_body_for_latest_block(network_data)
Self::prepare_evm_request_body_for_latest_block(network_data)
};
let response_bytes = Self::fetch_from_remote(&rpc_endpoint, &request_body)?;
let pending_requests_metadata = Self::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
let parsed_evm_responses = Self::fetch_multiple_evm_from_remote(pending_requests_metadata)
.iter()
.filter_map(|response_bytes| {
let parsed_evm_response = Self::parse_evm_response(response_bytes).ok()?;
Some(parsed_evm_response)
})
.collect::<Vec<EvmResponseType>>();
Self::check_evm_responses_correctness(&parsed_evm_responses)?;
let parsed_evm_response = Self::get_balanced_evm_response(&parsed_evm_responses)?;
match network_data.network_type {
NetworkType::Evm => {
let parsed_evm_response = Self::parse_evm_response(&response_bytes)?;
let new_block_range = match parsed_evm_response {
EvmResponseType::BlockNumber(new_evm_block) if from_block.le(&to_block) => {
let estimated_block =
new_evm_block.saturating_sub(network_data.finality_delay);
let adjusted_block =
Self::adjust_to_block(estimated_block, from_block, max_block_distance);
if from_block == 0 {
(estimated_block, estimated_block)
} else {
(from_block, adjusted_block)
// stay in the range of block distance
let estimated_block = new_evm_block.saturating_sub(network_data.finality_delay);
match from_block {
0 => (estimated_block, estimated_block),
start_block => {
let block_deviation = start_block.saturating_add(max_block_distance);
(start_block, estimated_block.min(block_deviation))
}
}
}
_ => (to_block, to_block),
@ -911,15 +900,14 @@ impl<T: Config> Pallet<T> {
log::info!(
target: LOG_TARGET,
"👻 Slow Clap #{:?} stored block #{:?} for network {:?}",
"👻 Offchain worker #{:?} stored block #{:?} for network {:?}",
block_number,
new_block_range.0,
network_id,
);
if !sp_io::offchain::is_validator() {
log::info!(target: LOG_TARGET, "👻 Not a validator; no transactions available");
return Ok(());
return Err(OffchainErr::NotValidator);
}
for (authority_index, authority_key) in Self::local_authorities(&session_index) {
@ -939,22 +927,6 @@ impl<T: Config> Pallet<T> {
}
}
fn adjust_to_block(estimated_block: u64, from_block: u64, max_block_distance: u64) -> u64 {
let fallback_value = from_block
.saturating_add(max_block_distance)
.min(estimated_block);
estimated_block
.checked_sub(from_block)
.map(|current_distance| {
current_distance
.le(&max_block_distance)
.then_some(estimated_block)
})
.flatten()
.unwrap_or(fallback_value)
}
fn local_authorities(
session_index: &SessionIndex,
) -> impl Iterator<Item = (u32, T::AuthorityId)> {
@ -973,35 +945,151 @@ impl<T: Config> Pallet<T> {
})
}
fn fetch_from_remote(rpc_endpoint: &[u8], request_body: &[u8]) -> OffchainResult<T, Vec<u8>> {
let rpc_endpoint_str =
core::str::from_utf8(rpc_endpoint).expect("rpc endpoint valid str; qed");
let request_body_str =
core::str::from_utf8(request_body).expect("request body valid str: qed");
fn prepare_pending_evm_requests(
rpc_endpoints: &Vec<Vec<u8>>,
request_body: &[u8],
) -> OffchainResult<T, Vec<(PendingRequest, String)>> {
let mut pending_requests_metadata = Vec::new();
let request_body_str = core::str::from_utf8(request_body)
.map_err(|_| OffchainErr::UnparsableRequestBody(request_body.to_vec()))?;
let deadline = sp_io::offchain::timestamp()
.add(rt_offchain::Duration::from_millis(FETCH_TIMEOUT_PERIOD));
let pending = rt_offchain::http::Request::post(&rpc_endpoint_str, vec![request_body_str])
for rpc_endpoint in rpc_endpoints.iter() {
let rpc_endpoint_str = match core::str::from_utf8(rpc_endpoint) {
Ok(rpc_endpoint_str) => rpc_endpoint_str,
Err(_) => {
log::info!(target: LOG_TARGET, "👻 Could not get valid utf8 rpc endpoint from bytes: {:?}", rpc_endpoint);
continue;
}
};
match rt_offchain::http::Request::post(&rpc_endpoint_str, vec![request_body_str])
.add_header("Accept", "application/json")
.add_header("Content-Type", "application/json")
.deadline(deadline)
.send()
.map_err(|err| OffchainErr::HttpRequestError(err))?;
{
Ok(pending) => pending_requests_metadata.push((pending, rpc_endpoint_str.to_string())),
Err(_) => log::info!(target: LOG_TARGET, "👻 Request skipped: failed to send request to \"{}\"", rpc_endpoint_str),
}
}
let response = pending
.try_wait(deadline)
.map_err(|_| OffchainErr::RequestUncompleted)?
.map_err(|_| OffchainErr::RequestUncompleted)?;
if pending_requests_metadata.len() == 0 {
return Err(OffchainErr::NoRequestsSent);
}
log::info!(target: LOG_TARGET, "👻 Requests sent {} out of {}", pending_requests_metadata.len(), rpc_endpoints.len());
Ok(pending_requests_metadata)
}
fn get_balanced_evm_response(parsed_evm_responses: &Vec<EvmResponseType>) -> OffchainResult<T, EvmResponseType> {
let first_evm_response = parsed_evm_responses
.first()
.ok_or(OffchainErr::EmptyResponses)?;
let result = match first_evm_response {
EvmResponseType::BlockNumber(_) => {
let mut block_numbers = parsed_evm_responses
.iter()
.enumerate()
.filter_map(|(index, response)| match response {
EvmResponseType::BlockNumber(block) => Some((index as u32, *block)),
EvmResponseType::TransactionLogs(_) => None,
})
.collect();
let median_value = Self::calculate_median_value(&mut block_numbers);
EvmResponseType::BlockNumber(median_value)
}
EvmResponseType::TransactionLogs(_) => {
let mut btree_map = BTreeMap::new();
parsed_evm_responses
.iter()
.for_each(|response| {
if let EvmResponseType::TransactionLogs(logs) = response {
let mut inner_logs = logs.clone();
inner_logs.sort_by_key(|l| l.block_number);
*btree_map.entry(inner_logs).or_insert(0) += 1;
}
});
let best_logs = btree_map.into_iter()
.max_by_key(|&(_, count)| count)
.map(|(v, _)| v.clone())
.ok_or(OffchainErr::EmptyResponses)?;
EvmResponseType::TransactionLogs(best_logs)
}
};
Ok(result)
}
fn check_evm_responses_correctness(parsed_evm_responses: &Vec<EvmResponseType>) -> OffchainResult<T, ()> {
let first_evm_response = parsed_evm_responses
.first()
.ok_or(OffchainErr::EmptyResponses)?;
let first_evm_response_type = core::mem::discriminant(first_evm_response);
if !parsed_evm_responses
.iter()
.all(|parsed_evm_response| {
core::mem::discriminant(parsed_evm_response) == first_evm_response_type
})
{
return Err(OffchainErr::DifferentEvmResponseTypes);
}
Ok(())
}
fn fetch_multiple_evm_from_remote(pending_requests_metadata: Vec<(PendingRequest, String)>) -> Vec<Vec<u8>> {
let pending_requests = pending_requests_metadata.iter()
.map(|(pending_request, _)| PendingRequest { id: pending_request.id })
.collect::<Vec<PendingRequest>>();
let deadline = sp_io::offchain::timestamp()
.add(rt_offchain::Duration::from_millis(FETCH_TIMEOUT_PERIOD));
PendingRequest::try_wait_all(pending_requests, Some(deadline))
.into_iter()
.enumerate()
.filter_map(|(index, pending_request)| {
let url = pending_requests_metadata.get(index)
.map(|(_, url)| url.clone())
.unwrap_or_default();
// handle request-level errors (transport/connection failures)
let request_result = match pending_request {
Ok(request) => request,
Err(err) => {
log::info!(target: LOG_TARGET, "👻 Request skipped; request to \"{}\" failed: {:?}", url, err);
return None;
}
};
// handle response-level errors (HTTP/protocol errors)
let response = match request_result {
Ok(response) => response,
Err(err) => {
log::info!(target: LOG_TARGET, "👻 Response skipped from \"{}\" error: {:?}", url, err);
return None;
}
};
if response.code != 200 {
return Err(OffchainErr::HttpResponseNotOk(response.code));
log::info!(target: LOG_TARGET, "👻 Response skipped from \"{}\": status {}", url, response.code);
return None;
}
Ok(response.body().collect::<Vec<u8>>())
Some(response.body().collect::<Vec<u8>>())
})
.collect()
}
fn prepare_request_body_for_latest_block(network_data: &NetworkData) -> Vec<u8> {
fn prepare_evm_request_body_for_latest_block(network_data: &NetworkData) -> Vec<u8> {
match network_data.network_type {
NetworkType::Evm => {
b"{\"id\":0,\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\"}".to_vec()
@ -1010,7 +1098,7 @@ impl<T: Config> Pallet<T> {
}
}
fn prepare_request_body_for_latest_transfers(
fn prepare_evm_request_body_for_latest_transfers(
from_block: u64,
to_block: u64,
network_data: &NetworkData,

89
pallets/slow-clap/src/tests.rs
View File

@ -137,16 +137,6 @@ fn bitmap_operations_correct() {
assert_eq!(ones, 11);
}
#[test]
fn should_correctly_adjust_to_block() {
assert_eq!(SlowClap::adjust_to_block(420, 69, 1337), 420);
assert_eq!(SlowClap::adjust_to_block(420, 1337, 69), 420);
assert_eq!(SlowClap::adjust_to_block(1337, 420, 69), 489);
assert_eq!(SlowClap::adjust_to_block(1337, 69, 420), 489);
assert_eq!(SlowClap::adjust_to_block(69, 1337, 420), 69);
assert_eq!(SlowClap::adjust_to_block(69, 420, 1337), 69);
}
#[test]
fn request_body_is_correct_for_get_block_number() {
let (offchain, _) = TestOffchainExt::new();
@ -155,7 +145,7 @@ fn request_body_is_correct_for_get_block_number() {
t.execute_with(|| {
let network_data = prepare_evm_network(Some(1), None);
let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
assert_eq!(
core::str::from_utf8(&request_body).unwrap(),
r#"{"id":0,"jsonrpc":"2.0","method":"eth_blockNumber"}"#
@ -173,7 +163,7 @@ fn request_body_is_correct_for_get_logs() {
let from_block: u64 = 420;
let to_block: u64 = 1337;
let network_data = prepare_evm_network(Some(1), None);
let request_body = SlowClap::prepare_request_body_for_latest_transfers(
let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
from_block, to_block, &network_data);
assert_eq!(core::str::from_utf8(&request_body).unwrap(),
r#"{"id":0,"jsonrpc":"2.0","method":"eth_getLogs","params":[{"fromBlock":"0x1a4","toBlock":"0x539","address":"0x4d224452801ACEd8B2F0aebE155379bb5D594381","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"]}]}"#,
@ -190,10 +180,14 @@ fn should_make_http_call_for_block_number() {
evm_block_response(&mut state.write());
let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
let rpc_endpoint = get_rpc_endpoint();
let rpc_endpoints = get_rpc_endpoints();
let network_data = prepare_evm_network(Some(1), None);
let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
let raw_response = raw_responses.first().unwrap();
assert_eq!(raw_response.len(), 45usize); // precalculated
Ok(())
});
@ -210,15 +204,19 @@ fn should_make_http_call_for_logs() {
let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
let from_block: u64 = 20335770;
let to_block: u64 = 20335858;
let rpc_endpoint = get_rpc_endpoint();
let rpc_endpoints = get_rpc_endpoints();
let network_data = prepare_evm_network(Some(1), None);
let request_body = SlowClap::prepare_request_body_for_latest_transfers(
let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
from_block,
to_block,
&network_data,
);
let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
let raw_response = raw_responses.first().unwrap();
assert_eq!(raw_response.len(), 1805); // precalculated
Ok(())
});
@ -233,11 +231,14 @@ fn should_make_http_call_and_parse_block_number() {
evm_block_response(&mut state.write());
let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
let rpc_endpoint = get_rpc_endpoint();
let rpc_endpoints = get_rpc_endpoints();
let network_data = prepare_evm_network(Some(1), None);
let request_body = SlowClap::prepare_request_body_for_latest_block(&network_data);
let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
let request_body = SlowClap::prepare_evm_request_body_for_latest_block(&network_data);
let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
let raw_response = raw_responses.first().unwrap();
let evm_block_number = SlowClap::parse_evm_response(&raw_response).map(
|parsed_response| match parsed_response {
EvmResponseType::BlockNumber(block_number) => block_number,
@ -263,19 +264,22 @@ fn should_make_http_call_and_parse_logs() {
evm_logs_response(&mut state.write());
let _: Result<(), OffchainErr<u32>> = t.execute_with(|| {
let rpc_endpoint = get_rpc_endpoint();
let rpc_endpoints = get_rpc_endpoints();
let from_block: u64 = 20335770;
let to_block: u64 = 20335858;
let network_data = prepare_evm_network(None, None);
let request_body = SlowClap::prepare_request_body_for_latest_transfers(
let request_body = SlowClap::prepare_evm_request_body_for_latest_transfers(
from_block,
to_block,
&network_data,
);
let raw_response = SlowClap::fetch_from_remote(&rpc_endpoint, &request_body)?;
let pending_requests = SlowClap::prepare_pending_evm_requests(&rpc_endpoints, &request_body)?;
let raw_responses = SlowClap::fetch_multiple_evm_from_remote(pending_requests);
let raw_response = raw_responses.first().unwrap();
match SlowClap::parse_evm_response(&raw_response)? {
EvmResponseType::BlockNumber(_) => assert_eq!(1, 0), // force break
EvmResponseType::TransactionLogs(evm_logs) => assert_eq!(evm_logs.len(), 2),
@ -1355,6 +1359,12 @@ fn should_split_commit_slash_between_active_validators() {
});
}
// TODO: add tests
// 1. prepare_pending_evm_requests fails fully and partially
// 2. fetch_multiple_evm_from_remote when they give different results
// 3. check_evm_responses_correctness
// 4. get_balanced_evm_response
fn assert_clapped_amount(
session_index: &SessionIndex,
unique_hash: &H256,
@ -1571,6 +1581,7 @@ fn get_mocked_metadata() -> (H256, u64, u64, u64) {
fn evm_block_response(state: &mut testing::OffchainState) {
let expected_body = br#"{"id":0,"jsonrpc":"2.0","method":"eth_blockNumber"}"#.to_vec();
state.expect_request(testing::PendingRequest {
method: "POST".into(),
uri: "https://rpc.endpoint.network.com".into(),
@ -1579,6 +1590,19 @@ fn evm_block_response(state: &mut testing::OffchainState) {
("Content-Type".to_string(), "application/json".to_string()),
],
response: Some(b"{\"id\":0,\"jsonrpc\":\"2.0\",\"result\":\"0x1364c81\"}".to_vec()),
body: expected_body.clone(),
sent: true,
..Default::default()
});
state.expect_request(testing::PendingRequest {
method: "POST".into(),
uri: "https://other.endpoint.network.com".into(),
headers: vec![
("Accept".to_string(), "application/json".to_string()),
("Content-Type".to_string(), "application/json".to_string()),
],
response: Some(b"{\"id\":0,\"jsonrpc\":\"2.0\",\"result\":\"0x1364c81\"}".to_vec()),
body: expected_body,
sent: true,
..Default::default()
@ -1636,6 +1660,23 @@ fn evm_logs_response(state: &mut testing::OffchainState) {
("Accept".to_string(), "application/json".to_string()),
("Content-Type".to_string(), "application/json".to_string()),
],
body: expected_body.clone(),
response: Some(expected_response.clone()),
sent: true,
..Default::default()
});
state.expect_request(testing::PendingRequest {
method: "POST".into(),
uri: "https://other.endpoint.network.com".into(),
headers: vec![
("Accept".to_string(), "application/json".to_string()),
("Content-Type".to_string(), "application/json".to_string()),
],
response_headers: vec![
("Accept".to_string(), "application/json".to_string()),
("Content-Type".to_string(), "application/json".to_string()),
],
body: expected_body,
response: Some(expected_response),
sent: true,