optimize merkle tree prefixed computations

Signed-off-by: Uncle Stinky <uncle.stinky@ghostchain.io>

pallets/exodus/src/benchmarking.rs

@@ -415,10 +415,16 @@ mod benchmarks {
                     dummy_values.insert(identifier, dummy_value.clone());
                 }
 
+                let identifier_size = network_curve.scalar_bytes_len();
+
                 let merkle_tree =
                 	NetworkCurve::build_merkle_tree(
-                	    &dummy_values,
+                	    identifier_size + dummy_value.len(),
-                	    |value: &Vec<u8>| Ok(value.clone()),
+                        &dummy_values,
+                	    |value, buffer| {
+                            buffer.extend_from_slice(value.as_ref());
+                            Ok(())
+                        }
                     ).expect("Merkle tree for verifying shares should be valid");
 
                 let merkle_root = merkle_tree.last().copied().unwrap();
@@ -697,10 +703,16 @@ mod benchmarks {
                     dummy_values.insert(identifier, dummy_binding_factor.clone());
                 }
 
+                let identifier_size = network_curve.scalar_bytes_len();
+
                 let merkle_tree =
                 	NetworkCurve::build_merkle_tree(
-                	    &dummy_values,
+                	    identifier_size + dummy_scalar.len(),
-                	    |value: &Vec<u8>| Ok(value.clone()),
+                        &dummy_values,
+                	    |value, buffer| {
+                            buffer.extend_from_slice(value.as_ref());
+                            Ok(())
+                        }
                     ).expect("Merkle tree for binding factor should be valid");
 
                 let merkle_root = merkle_tree.last().copied().unwrap();

pallets/exodus/src/error.rs

@@ -43,6 +43,7 @@ pub enum ExodusError {
     InvalidMerkleProof,
     InvalidIdentityElement,
     NoActiveAuthorities,
+    TreeGenerationFailed,
     Unknown,
 }
 
@@ -88,6 +89,7 @@ impl core::fmt::Debug for ExodusError {
             ExodusError::InvalidMerkleProof => write!(fmt, "Invalid Merkle proof provided."),
             ExodusError::InvalidIdentityElement => write!(fmt, "Invalid identity element provided."),
             ExodusError::NoActiveAuthorities => write!(fmt, "No active authorities for signing EXODUS transactions."),
+            ExodusError::TreeGenerationFailed => write!(fmt, "Tree generation failed because of inconsistent indexation."),
             ExodusError::Unknown => write!(fmt, "Unknown error."),
         }
     }

pallets/exodus/src/impls/dkg.rs

@@ -3,6 +3,7 @@ use rand_chacha::rand_core::{CryptoRng, RngCore};
 
 use ghost_traits::exodus::{
     DistributedKeyGeneration, IdentifierConverter, MerkleTreeBuilder,
+    CiphersuiteSizes,
 };
 
 use crate::{AuthIndex, ExodusError, NetworkCurve};
@@ -241,6 +242,8 @@ impl DistributedKeyGeneration<AuthIndex, ExodusError> for NetworkCurve {
         round2_packages_bytes: &Self::RoundPackages,
     ) -> Self::Part3Result {
         with_ciphersuite!(self, |f| {
+            type InnerGroup = <Ciphersuite as frost_core::Ciphersuite>::Group;
+
             let round2_secret_package = f::keys::dkg::round2::SecretPackage::deserialize(round2_secret_package_bytes)
                 .map_err(|_| ExodusError::DeserializationError)?;
 
@@ -291,13 +294,24 @@ impl DistributedKeyGeneration<AuthIndex, ExodusError> for NetworkCurve {
             let key_package_bytes = key_package.serialize()
                 .map_err(|_| ExodusError::SerializationError)?;
 
+            let identifier_size = self.scalar_bytes_len();
+            let verifying_share_size = self.element_bytes_len();
+
             let verifying_shares_merkle_tree =
                 Self::build_merkle_tree(
+                    identifier_size + verifying_share_size,
                     pubkey_package.verifying_shares(),
-                    |verifying_share| verifying_share.serialize()
+                    |verifying_share, buffer| {
-                        .map_err(|_| ExodusError::SerializationError)
+                        let serialized = <InnerGroup as frost_core::Group>::serialize(
+                            &verifying_share.to_element()
+                        ).map_err(|_| ExodusError::SerializationError)?;
+
+                        buffer.extend_from_slice(serialized.as_ref());
+                        Ok(())
+                    }
                 )?;
 
+            // TODO: make it better
             let (_, merkle_proof) =
                 Self::generate_merkle_proof_from_tree(
                     pubkey_package.verifying_shares(),

pallets/exodus/src/impls/frost.rs

@@ -1,5 +1,5 @@
 use ghost_traits::exodus::{
-    IdentifierConverter, MerkleTreeBuilder,
+    CiphersuiteSizes, IdentifierConverter, MerkleTreeBuilder,
     FlexibleRoundOptimizedSchnorrThresholdSignature,
 };
 use sp_std::{
@@ -15,29 +15,6 @@ impl FlexibleRoundOptimizedSchnorrThresholdSignature<AuthIndex, ExodusError> for
     type Packages<'a> = BTreeMap<AuthIndex, &'a [u8]>;
     type NoncePackages<'a> = BTreeMap<AuthIndex, (&'a [u8], &'a [u8])>;
 
-    fn header_bytes_len(&self) -> usize { 5 }
-
-    fn signature_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 64,
-            NetworkCurve::Ed25519 => 64,
-        }
-    }
-
-    fn element_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 33,
-            NetworkCurve::Ed25519 => 32,
-        }
-    }
-
-    fn scalar_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 32,
-            NetworkCurve::Ed25519 => 32,
-        }
-    }
-
     fn verify_signature_share(
         &self,
         index: AuthIndex,
@@ -330,11 +307,16 @@ impl FlexibleRoundOptimizedSchnorrThresholdSignature<AuthIndex, ExodusError> for
                 return Err(ExodusError::InvalidParticipantId);
             }
 
+            let identifier_size = self.scalar_bytes_len();
+            let binding_factor_size = self.scalar_bytes_len();
+
             let binding_factor_tree = Self::build_merkle_tree(
+                identifier_size + binding_factor_size,
                 &binding_factors_list,
-                |binding_factor| {
+                |binding_factor, buffer| {
                     let serialized = <InnerField as frost_core::Field>::serialize(binding_factor);
-	                Ok(serialized.as_ref().to_vec())
+                    buffer.extend_from_slice(serialized.as_ref());
+                    Ok(())
                 }
             )?;
 

pallets/exodus/src/impls/merkle.rs

@@ -13,15 +13,16 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
     type Hash = <SubstrateBlake2Hasher as GhostHasher>::Hash;
 
     fn build_merkle_tree<C, V, F>(
+        size_hint: usize,
         values: &BTreeMap<Identifier<C>, V>,
-        serialize_fn: F,
+        update_preimage_buffer: F,
     ) -> Result<Vec<Self::Hash>, ExodusError>
 	where
 		C: frost_core::Ciphersuite,
 		<<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
-		F: Fn(&V) -> Result<Vec<u8>, ExodusError>
+		F: Fn(&V, &mut Vec<u8>) -> Result<(), ExodusError>
 	{
-        use ghost_helpers::merkle_tree::generate_tree;
+        use ghost_helpers::merkle_tree::{generate_tree, MerkleTreeError};
 
         type CField<C> = <<C as Ciphersuite>::Group as Group>::Field;
 
@@ -34,25 +35,28 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             })
             .ok_or(ExodusError::IncorrectNumberOfIdentifiers)?;
 
-        generate_tree::<SubstrateBlake2Hasher, _, _, _, _>(
+        let maybe_tree = generate_tree::<SubstrateBlake2Hasher, _, _, _, _>(
             max_index,
+            size_hint,
             values.iter(),
-            |(identifier, value)| {
+            |(identifier, value), buffer| {
                 let id_scalar = identifier.to_scalar();
                 let id_bytes = <CField<C> as frost_core::Field>::serialize(&id_scalar);
 
                 let index = Self::convert_identifier_to_index(id_bytes.as_ref())?;
-                let value_bytes = serialize_fn(value).map_err(|_| ExodusError::SerializationError)?;
 
-                // NOTE: revisit
+                buffer.extend_from_slice(id_bytes.as_ref());
-                // Is it possible to use [u8; CONSTANT] where constant is
+                update_preimage_buffer(value, buffer)?;
-                // purely dependant on provided generic.
-                let mut preimage = Vec::<u8>::with_capacity(id_bytes.as_ref().len() + value_bytes.len());
-                preimage.extend_from_slice(id_bytes.as_ref());
-                preimage.extend_from_slice(value_bytes.as_ref());
 
-                Ok((index as usize, preimage))
+                Ok(index as usize)
-            })
+            });
+
+        match maybe_tree {
+            Ok(tree) => Ok(tree),
+            Err(MerkleTreeError::Generate(exodus_err)) => Err(exodus_err),
+            Err(MerkleTreeError::InvalidIndex) => Err(ExodusError::TreeGenerationFailed),
+            Err(MerkleTreeError::Empty) => Err(ExodusError::IncorrectNumberOfIdentifiers),
+        }
     }
 
     fn generate_merkle_proof_from_tree<C, V, F>(
@@ -66,7 +70,7 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
 		<<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
 		F: Fn(&V) -> Result<Vec<u8>, ExodusError>
 	{
-        use ghost_helpers::merkle_tree::generate_proof;
+        use ghost_helpers::merkle_tree::{generate_proof, MerkleTreeError};
 
         type CField<C> = <<C as Ciphersuite>::Group as Group>::Field;
 
@@ -75,7 +79,7 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             .map_err(|_| ExodusError::InvalidParticipantId)?;
 
         let value = values.get(&identifier).ok_or(ExodusError::InvalidParticipantId)?;
-        let value_bytes = serialize_fn(&value).map_err(|_| ExodusError::SerializationError)?;
+        let value_bytes = serialize_fn(&value)?;
 
         let max_index = values.keys()
             .next_back()
@@ -86,11 +90,15 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             })
             .ok_or(ExodusError::IncorrectNumberOfIdentifiers)?;
 
-        let proof = generate_proof::<SubstrateBlake2Hasher, _>(
+        let proof = generate_proof::<SubstrateBlake2Hasher, _, _>(
             merkle_tree,
             max_index,
             authority_index,
-        );
+        ).map_err(|err| match err {
+            MerkleTreeError::Generate(exodus_err) => exodus_err,
+            MerkleTreeError::InvalidIndex => ExodusError::TreeGenerationFailed,
+            MerkleTreeError::Empty => ExodusError::IncorrectNumberOfIdentifiers,
+        })?;
 
         Ok((value_bytes, proof))
     }

pallets/exodus/src/impls/mod.rs

@@ -72,3 +72,6 @@ mod ecdh;
 
 #[macro_use]
 mod frost;
+
+#[macro_use]
+mod sizes;

pallets/exodus/src/impls/sizes.rs

@@ -0,0 +1,29 @@
+use ghost_helpers::networks::NetworkCurve;
+use ghost_traits::exodus::CiphersuiteSizes;
+
+use crate::{AuthIndex, ExodusError};
+
+impl CiphersuiteSizes<AuthIndex, ExodusError> for NetworkCurve {
+    fn header_bytes_len(&self) -> usize { 5 }
+
+    fn signature_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 64,
+            NetworkCurve::Ed25519 => 64,
+        }
+    }
+
+    fn element_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 33,
+            NetworkCurve::Ed25519 => 32,
+        }
+    }
+
+    fn scalar_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 32,
+            NetworkCurve::Ed25519 => 32,
+        }
+    }
+}

pallets/exodus/src/lib.rs

@@ -48,6 +48,7 @@ use ghost_traits::{
     exodus::{
         MerkleTreeBuilder, DistributedKeyGeneration, EllipticCurveDiffieHellman,
         FlexibleRoundOptimizedSchnorrThresholdSignature, EvmGovernanceRegistrar,
+        CiphersuiteSizes,
     },
     networks::{
         NetworkDataBasicHandler, NetworkDataInspectHandler,

pallets/exodus/src/tests.rs

@@ -1571,9 +1571,17 @@ fn test_merkle_tree_success_power_of_two_for_shares() {
     let indices = (1..=32).collect::<Vec<AuthIndex>>();
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1608,11 +1616,16 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
     let indices = (1..=32).collect::<Vec<AuthIndex>>();
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1623,7 +1636,7 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
             &identifier.serialize(),
         ).unwrap();
 
-        let (shares_bytes, proof) =
+        let (scalar_bytes, proof) =
             NetworkCurve::generate_merkle_proof_from_tree::<Secp256K1Sha256TR, _, _>(
                 &scalars,
                 &tree,
@@ -1635,7 +1648,7 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
             ).unwrap();
 
         let result = network_curve.verify_merkle_proof(
-            &shares_bytes,
+            &scalar_bytes,
             &proof,
             root,
             auth_index,
@@ -1657,9 +1670,17 @@ fn test_random_sparse_combinations_stress_for_shares() {
     let indices: Vec<AuthIndex> = unique_indices.into_iter().collect();
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1695,11 +1716,16 @@ fn test_random_sparse_combinations_stress_for_scalars() {
     let indices: Vec<AuthIndex> = unique_indices.into_iter().collect();
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1733,9 +1759,17 @@ fn test_participants_with_high_index_gap_for_shares() {
     let indices = vec![1, 1023];
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1765,11 +1799,16 @@ fn test_participants_with_high_index_gap_for_scalars() {
     let indices = vec![1, 1023];
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1803,9 +1842,17 @@ fn test_regression_consecutive_vs_position_for_shares() {
     let custom_indices = vec![2, 3];
     let shares = generate_mock_secp256k1_shares(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |scalar| scalar.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1828,11 +1875,16 @@ fn test_regression_consecutive_vs_position_for_scalars() {
     let custom_indices = vec![2, 3];
     let scalars = generate_mock_secp256k1_scalars(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1859,9 +1911,17 @@ fn test_attack_with_unknown_authority_index_for_shares() {
     let custom_indices = vec![1, 2, 3];
     let shares = generate_mock_secp256k1_shares(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |scalar| scalar.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1884,11 +1944,16 @@ fn test_attack_with_unknown_authority_index_for_scalars() {
     let custom_indices = vec![1, 2, 3];
     let scalars = generate_mock_secp256k1_scalars(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 

pallets/governance/src/tests.rs

@@ -79,14 +79,21 @@ fn claim_happy_path() {
         let users = vec![user_1.clone(), user_2.clone(), user_3.clone()];
         let max_index = 2u8;
 
+        let tmp_package_probe = ClaimPackage::<Test> {
+            shares: user_1.allowed_shares,
+            merkle_proof: Default::default(),
+            token_id: user_1.token_id,
+            index: 0u32,
+        };
+        let size_hint = tmp_package_probe.get_preimage(&user_1.evm_address, network_id).as_ref().len();
+
         let raw_values_complex = vec![(0u8, &user_1), (1u8, &user_2), (2u8, &user_3)];
-        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, _>(
+        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, ()>(
             max_index,
+            size_hint,
             raw_values_complex,
 
-            |item: (u8, &NewTestUser)| -> Result<(usize, Vec<u8>), ()> {
+            |(index, u), buffer| {
-                let (index, u) = item;
-
                 let tmp_package = ClaimPackage::<Test> {
                     shares: u.allowed_shares,
                     merkle_proof: Default::default(),
@@ -94,9 +101,12 @@ fn claim_happy_path() {
                     index: index as u32
                 };
 
-                Ok((index as usize, tmp_package.get_preimage(&u.evm_address, network_id).to_vec()))
+                let preimage_bytes = tmp_package.get_preimage(&u.evm_address, network_id);
+                buffer.extend_from_slice(preimage_bytes.as_ref());
+
+                Ok(index as usize)
             }
-        ).unwrap();
+        ).ok().unwrap();
 
         let merkle_root = *merkle_tree.last().unwrap();
 
@@ -116,7 +126,9 @@ fn claim_happy_path() {
                 activated_shares: 0,
             });
 
-            let proof = generate_proof::<SubstrateKeccakHasher, u8>(&merkle_tree, max_index, slot_index as u8);
+            let proof = generate_proof::<SubstrateKeccakHasher, u8, ()>(&merkle_tree, max_index, slot_index as u8)
+                .ok()
+                .unwrap();
 
             let claim_package = ClaimPackage::<Test> {
                 shares: current_user.allowed_shares,
@@ -205,21 +217,33 @@ fn partial_claims_happy_path() {
         let max_index = 0u8;
         let raw_values_complex = vec![(0u8, &current_user)];
 
-        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, _>(
+        let tmp_package_probe = ClaimPackage::<Test> {
+            shares: current_user.allowed_shares,
+            merkle_proof: Default::default(),
+            token_id: current_user.token_id,
+            index: 0u32,
+        };
+        let size_hint = tmp_package_probe.get_preimage(&current_user.evm_address, network_id).as_ref().len();
+
+        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, ()>(
             max_index,
+            size_hint,
             raw_values_complex,
 
-            |item: (u8, &NewTestUser)| -> Result<(usize, Vec<u8>), ()> {
+            |(index, u), buffer| {
-                let (index, u) = item;
                 let tmp_package = ClaimPackage::<Test> {
                     shares: u.allowed_shares,
                     merkle_proof: Default::default(),
                     token_id: u.token_id,
                     index: index as u32
                 };
-                Ok((index as usize, tmp_package.get_preimage(&u.evm_address, network_id).to_vec()))
+
+                let preimage_bytes = tmp_package.get_preimage(&u.evm_address, network_id);
+                buffer.extend_from_slice(preimage_bytes.as_ref());
+
+                Ok(index as usize)
             }
-        ).unwrap();
+        ).ok().unwrap();
 
         let total_shares = 1_000_000;
         let merkle_root = *merkle_tree.last().unwrap();
@@ -240,7 +264,10 @@ fn partial_claims_happy_path() {
         setup_governor(&substrate_account_b);
         setup_governor(&substrate_account_c);
 
-        let proof = generate_proof::<SubstrateKeccakHasher, u8>(&merkle_tree, max_index, 0u8);
+        let proof = generate_proof::<SubstrateKeccakHasher, u8, ()>(&merkle_tree, max_index, 0u8)
+            .ok()
+            .unwrap();
+
         let claim_package = ClaimPackage {
             shares: current_user.allowed_shares,
             merkle_proof: BoundedVec::try_from(proof).unwrap(),

pallets/helpers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ghost-helpers"
-version = "0.0.11"
+version = "0.0.12"
 description = "Cryptographic utility suite for custom runtimes: optimized Bitmaps, UTXO parsing, Merkle Tree proofs, and Hash Chain components."
 license.workspace = true
 authors.workspace = true

pallets/helpers/src/hash_chain.rs

@@ -1,19 +1,5 @@
 use ghost_traits::hashing::GhostHasher;
 
-fn combine_values<H, V>(left: V, right: V) -> H::Hash
-where
-    H: GhostHasher,
-    V: AsRef<[u8]>,
-{
-    let hash_len = H::hash_len();
-    let mut combined = sp_std::vec![0u8; hash_len * 2];
-
-    combined[..hash_len].copy_from_slice(left.as_ref());
-    combined[hash_len..].copy_from_slice(right.as_ref());
-
-    H::hash(&combined)
-}
-
 pub fn cumulative_hash<H, S>(raw_values: S) -> H::Hash
 where
     H: GhostHasher,
@@ -24,7 +10,7 @@ where
         .into_iter()
         .fold(H::empty(), |acc_hash, raw_value| {
             let hashed_value = H::hash(raw_value.as_ref());
-            combine_values::<H, _>(acc_hash, hashed_value)
+            H::combine_hashes(acc_hash, hashed_value)
         })
 }
 
@@ -47,7 +33,7 @@ where
 
     iter.fold(current_hash, |acc_hash, raw_value| {
         let next_hash = H::from_slice(raw_value.as_ref());
-        combine_values::<H, _>(acc_hash, next_hash)
+        H::combine_hashes(acc_hash, next_hash)
     })
 }
 
@@ -94,45 +80,14 @@ where
 mod tests {
     use super::*;
     use sp_core::H256;
-    use sp_io::hashing::{blake2_256, sha2_256};
     use sp_std::vec::Vec;
 
-    pub struct TestBlake2Hasher;
+    use crate::{
-    impl GhostHasher for TestBlake2Hasher {
+        SubstrateBlake2Hasher, SubstrateKeccakHasher, UtxoSha2Hasher,
-        type Hash = H256;
+    };
-        type HashBytes = [u8; 32];
-
-        fn hash(data: &[u8]) -> Self::Hash {
-            H256::from(blake2_256(data))
-        }
-        fn empty() -> Self::Hash {
-            H256::zero()
-        }
-        fn hash_len() -> usize {
-            H256::len_bytes()
-        }
-    }
-
-    pub struct TestSha2Hasher;
-    impl GhostHasher for TestSha2Hasher {
-        type Hash = H256;
-        type HashBytes = [u8; 32];
-        fn hash(data: &[u8]) -> Self::Hash {
-            let first_pass: [u8; 32] = sha2_256(data);
-            let second_pass: [u8; 32] = sha2_256(&first_pass);
-
-            H256::from(second_pass)
-        }
-        fn hash_len() -> usize {
-            H256::len_bytes()
-        }
-        fn empty() -> Self::Hash {
-            H256::zero()
-        }
-    }
 
     fn mock_sequential_hash(header: &[u8; 80]) -> H256 {
-        sequential_hash::<TestSha2Hasher, _>(core::iter::once(header))
+        sequential_hash::<UtxoSha2Hasher, _>(core::iter::once(header))
     }
 
     fn create_header(prev_hash: [u8; 32]) -> [u8; 80] {
@@ -141,89 +96,101 @@ mod tests {
         header
     }
 
-    #[test]
+    macro_rules! run_hasher_tests {
-    fn test_generate_empty_values() {
+        ($hasher_type:ty, $mod_name:ident) => {
-        let empty_list: Vec<Vec<u8>> = sp_std::vec![];
+            mod $mod_name {
-        let result = cumulative_hash::<TestBlake2Hasher, _>(empty_list);
+                use super::*;
-        assert_eq!(H256::zero(), result);
+
+                #[test]
+                fn test_generate_empty_values() {
+                    let empty_list: Vec<Vec<u8>> = sp_std::vec![];
+                    let result = cumulative_hash::<$hasher_type, _>(empty_list);
+                    assert_eq!(H256::zero(), result);
+                }
+
+                #[test]
+                fn test_cumulative_hash_determinism_and_avalanche() {
+                    let dataset_1 = sp_std::vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
+                    let dataset_2 = sp_std::vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
+                    let dataset_3 = sp_std::vec![vec![9, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
+
+                    let hash_1 = cumulative_hash::<$hasher_type, _>(dataset_1);
+                    let hash_2 = cumulative_hash::<$hasher_type, _>(dataset_2);
+                    let hash_3 = cumulative_hash::<$hasher_type, _>(dataset_3);
+
+                    assert_eq!(hash_1, hash_2);
+                    assert_ne!(hash_1, hash_3);
+                    assert_ne!(hash_2, hash_3);
+                }
+
+                #[test]
+                fn test_cumulative_hash_order_sensitive() {
+                    let dataset_forward = sp_std::vec![sp_std::vec![1, 1, 1], sp_std::vec![2, 2, 2]];
+                    let dataset_reversed = sp_std::vec![sp_std::vec![2, 2, 2], sp_std::vec![1, 1, 1]];
+
+                    let hash_forward = cumulative_hash::<$hasher_type, _>(dataset_forward);
+                    let hash_reversed = cumulative_hash::<$hasher_type, _>(dataset_reversed);
+
+                    assert_ne!(hash_forward, hash_reversed);
+                }
+
+                #[test]
+                fn test_cumulative_hash_with_slices() {
+                    let slice_data: &[&[u8]] = &[&[1, 2, 3], &[4, 5, 6]];
+                    let hash_from_slice = cumulative_hash::<$hasher_type, _>(slice_data.iter());
+                    assert_ne!(hash_from_slice, H256::zero());
+                }
+
+                #[test]
+                fn test_sequential_hash_empty() {
+                    let empty_list: Vec<Vec<u8>> = sp_std::vec![];
+                    let result = sequential_hash::<$hasher_type, _>(empty_list);
+                    assert_eq!(H256::zero(), result);
+                }
+
+                #[test]
+                fn test_sequential_hash_single_element() {
+                    let single_item = sp_std::vec![sp_std::vec![1, 2, 3]];
+                    let result = sequential_hash::<$hasher_type, _>(single_item);
+                    let expected = <$hasher_type>::hash(&[1, 2, 3]);
+                    assert_eq!(result, expected);
+                }
+
+                #[test]
+                fn test_sequential_hash_determinism_and_order() {
+                    let data_1 = sp_std::vec![sp_std::vec![1], sp_std::vec![2], sp_std::vec![3]];
+                    let data_2 = sp_std::vec![sp_std::vec![1], sp_std::vec![2], sp_std::vec![3]];
+                    let data_reversed = sp_std::vec![sp_std::vec![3], sp_std::vec![2], sp_std::vec![1]];
+
+                    let hash_1 = sequential_hash::<$hasher_type, _>(data_1);
+                    let hash_2 = sequential_hash::<$hasher_type, _>(data_2);
+                    let hash_reversed = sequential_hash::<$hasher_type, _>(data_reversed);
+
+                    assert_eq!(hash_1, hash_2);
+                    assert_ne!(hash_1, hash_reversed);
+                }
+
+                #[test]
+                fn test_sequential_vs_cumulative_hash() {
+                    let dataset = sp_std::vec![sp_std::vec![1, 2, 3], sp_std::vec![4, 5, 6]];
+                    let seq_hash = sequential_hash::<$hasher_type, _>(dataset.clone());
+                    let cum_hash = cumulative_hash::<$hasher_type, _>(dataset);
+                    assert_ne!(seq_hash, cum_hash);
+                }
+
+                #[test]
+                fn test_sequential_hash_with_slices() {
+                    let slice_data: &[&[u8]] = &[&[10, 20], &[30, 40]];
+                    let result = sequential_hash::<$hasher_type, _>(slice_data.iter());
+                    assert_ne!(result, H256::zero());
+                }
+            }
+        }
     }
 
-    #[test]
+    run_hasher_tests!(SubstrateBlake2Hasher, blake2_hashing_tests);
-    fn test_cumulative_hash_determinism_and_avalanche() {
+    run_hasher_tests!(SubstrateKeccakHasher, keccak_hashing_tests);
-        let dataset_1 = sp_std::vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
+    run_hasher_tests!(UtxoSha2Hasher, sha2_hashing_tests);
-        let dataset_2 = sp_std::vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
-        let dataset_3 = sp_std::vec![vec![9, 2, 3], vec![4, 5, 6], vec![7, 8, 9]];
-
-        let hash_1 = cumulative_hash::<TestBlake2Hasher, _>(dataset_1);
-        let hash_2 = cumulative_hash::<TestBlake2Hasher, _>(dataset_2);
-        let hash_3 = cumulative_hash::<TestBlake2Hasher, _>(dataset_3);
-
-        assert_eq!(hash_1, hash_2);
-        assert_ne!(hash_1, hash_3);
-        assert_ne!(hash_2, hash_3);
-    }
-
-    #[test]
-    fn test_cumulative_hash_order_sensitive() {
-        let dataset_forward = sp_std::vec![sp_std::vec![1, 1, 1], sp_std::vec![2, 2, 2]];
-        let dataset_reversed = sp_std::vec![sp_std::vec![2, 2, 2], sp_std::vec![1, 1, 1]];
-
-        let hash_forward = cumulative_hash::<TestBlake2Hasher, _>(dataset_forward);
-        let hash_reversed = cumulative_hash::<TestBlake2Hasher, _>(dataset_reversed);
-
-        assert_ne!(hash_forward, hash_reversed);
-    }
-
-    #[test]
-    fn test_cumulative_hash_with_slices() {
-        let slice_data: &[&[u8]] = &[&[1, 2, 3], &[4, 5, 6]];
-        let hash_from_slice = cumulative_hash::<TestBlake2Hasher, _>(slice_data.iter());
-        assert_ne!(hash_from_slice, H256::zero());
-    }
-
-    #[test]
-    fn test_sequential_hash_empty() {
-        let empty_list: Vec<Vec<u8>> = sp_std::vec![];
-        let result = sequential_hash::<TestBlake2Hasher, _>(empty_list);
-        assert_eq!(H256::zero(), result);
-    }
-
-    #[test]
-    fn test_sequential_hash_single_element() {
-        let single_item = sp_std::vec![sp_std::vec![1, 2, 3]];
-        let result = sequential_hash::<TestBlake2Hasher, _>(single_item);
-        let expected = TestBlake2Hasher::hash(&[1, 2, 3]);
-        assert_eq!(result, expected);
-    }
-
-    #[test]
-    fn test_sequential_hash_determinism_and_order() {
-        let data_1 = sp_std::vec![sp_std::vec![1], sp_std::vec![2], sp_std::vec![3]];
-        let data_2 = sp_std::vec![sp_std::vec![1], sp_std::vec![2], sp_std::vec![3]];
-        let data_reversed = sp_std::vec![sp_std::vec![3], sp_std::vec![2], sp_std::vec![1]];
-
-        let hash_1 = sequential_hash::<TestBlake2Hasher, _>(data_1);
-        let hash_2 = sequential_hash::<TestBlake2Hasher, _>(data_2);
-        let hash_reversed = sequential_hash::<TestBlake2Hasher, _>(data_reversed);
-
-        assert_eq!(hash_1, hash_2);
-        assert_ne!(hash_1, hash_reversed);
-    }
-
-    #[test]
-    fn test_sequential_vs_cumulative_hash() {
-        let dataset = sp_std::vec![sp_std::vec![1, 2, 3], sp_std::vec![4, 5, 6]];
-        let seq_hash = sequential_hash::<TestBlake2Hasher, _>(dataset.clone());
-        let cum_hash = cumulative_hash::<TestBlake2Hasher, _>(dataset);
-        assert_ne!(seq_hash, cum_hash);
-    }
-
-    #[test]
-    fn test_sequential_hash_with_slices() {
-        let slice_data: &[&[u8]] = &[&[10, 20], &[30, 40]];
-        let result = sequential_hash::<TestBlake2Hasher, _>(slice_data.iter());
-        assert_ne!(result, H256::zero());
-    }
 
     #[test]
     fn test_verify_in_current_block_with_empty_ancestry() {
@@ -234,7 +201,7 @@ mod tests {
         trusted_bytes.reverse();
         let trusted_top_block_hash = H256::from(trusted_bytes);
 
-        let result = verify_hash_ancestry::<TestSha2Hasher, &[[u8; 80]]>(
+        let result = verify_hash_ancestry::<UtxoSha2Hasher, &[[u8; 80]]>(
             &tx_header,
             &[],
             trusted_top_block_hash,
@@ -256,7 +223,7 @@ mod tests {
         let trusted_top_block_hash = H256::from(trusted_bytes);
 
         let ancestry = [header_101];
-        let result = verify_hash_ancestry::<TestSha2Hasher, &[[u8; 80]]>(
+        let result = verify_hash_ancestry::<UtxoSha2Hasher, &[[u8; 80]]>(
             &tx_header_100,
             &ancestry,
             trusted_top_block_hash,
@@ -284,7 +251,7 @@ mod tests {
         let trusted_top_block_hash = H256::from(trusted_bytes);
 
         let ancestry = [header_101, header_102, header_103];
-        let result = verify_hash_ancestry::<TestSha2Hasher, &[[u8; 80]]>(
+        let result = verify_hash_ancestry::<UtxoSha2Hasher, &[[u8; 80]]>(
             &header_100,
             &ancestry,
             trusted_top_block_hash,
@@ -305,7 +272,7 @@ mod tests {
         let trusted_top_block_hash = H256::from(trusted_bytes);
 
         let ancestry = [broken_header_101];
-        let result = verify_hash_ancestry::<TestSha2Hasher, &[[u8; 80]]>(
+        let result = verify_hash_ancestry::<UtxoSha2Hasher, &[[u8; 80]]>(
             &header_100,
             &ancestry,
             trusted_top_block_hash,

pallets/helpers/src/lib.rs

@@ -48,53 +48,73 @@ where
     value - f
 }
 
-pub struct SubstrateBlake2Hasher;
+macro_rules! impl_ghost_hasher {
-impl GhostHasher for SubstrateBlake2Hasher {
+    (
-    type Hash = H256;
+        struct $name:ident;
-    type HashBytes = [u8; 32];
+        hash_type: $hash_type:ty;
-    fn hash(data: &[u8]) -> Self::Hash {
+        hash_bytes_len: $hash_bytes_len:expr;
-        H256::from(blake2_256(data))
+        fn hash($data:ident) -> $hash_res:expr
-    }
+    ) => {
-    fn hash_len() -> usize {
+        pub struct $name;
-        H256::len_bytes()
+
-    }
+        impl GhostHasher for $name {
-    fn empty() -> Self::Hash {
+            type Hash = $hash_type;
-        H256::zero()
+            type HashBytes = [u8; $hash_bytes_len];
-    }
+
+            fn hash($data: &[u8]) -> Self::Hash {
+                $hash_res
+            }
+
+            fn hash_len() -> usize {
+                $hash_bytes_len
+            }
+
+            fn empty() -> Self::Hash {
+                <$hash_type>::zero()
+            }
+
+            fn combine_hashes(left: Self::Hash, right: Self::Hash) -> Self::Hash {
+                let mut combined = [0u8; $hash_bytes_len * 2];
+                combined[..$hash_bytes_len].copy_from_slice(left.as_ref());
+                combined[$hash_bytes_len..].copy_from_slice(right.as_ref());
+                Self::hash(&combined)
+            }
+
+            fn combine_hashes_prefixed(p: u8, l: Self::Hash, r: Self::Hash) -> Self::Hash {
+                let mut combined = [p; $hash_bytes_len * 2 + 1];
+                combined[1..1 + $hash_bytes_len].copy_from_slice(l.as_ref());
+                combined[1 + $hash_bytes_len..].copy_from_slice(r.as_ref());
+                Self::hash(&combined)
+            }
+        }
+    };
 }
 
-pub struct SubstrateKeccakHasher;
+impl_ghost_hasher!(
-impl GhostHasher for SubstrateKeccakHasher {
+    struct SubstrateBlake2Hasher;
-    type Hash = H256;
+    hash_type: H256;
-    type HashBytes = [u8; 32];
+    hash_bytes_len: 32;
-    fn hash(data: &[u8]) -> Self::Hash {
+    fn hash(data) -> H256::from(blake2_256(data))
-        H256::from(keccak_256(data))
+);
-    }
-    fn hash_len() -> usize {
-        H256::len_bytes()
-    }
-    fn empty() -> Self::Hash {
-        H256::zero()
-    }
-}
 
-pub struct UtxoSha2Hasher;
+impl_ghost_hasher!(
-impl GhostHasher for UtxoSha2Hasher {
+    struct SubstrateKeccakHasher;
-    type Hash = H256;
+    hash_type: H256;
-    type HashBytes = [u8; 32];
+    hash_bytes_len: 32;
-    fn hash(data: &[u8]) -> Self::Hash {
+    fn hash(data) -> H256::from(keccak_256(data))
+);
+
+impl_ghost_hasher!(
+    struct UtxoSha2Hasher;
+    hash_type: H256;
+    hash_bytes_len: 32;
+    fn hash(data) -> {
         let first_pass: [u8; 32] = sha2_256(data);
         let second_pass: [u8; 32] = sha2_256(&first_pass);
 
         H256::from(second_pass)
     }
-    fn hash_len() -> usize {
+);
-        H256::len_bytes()
-    }
-    fn empty() -> Self::Hash {
-        H256::zero()
-    }
-}
 
 #[cfg(test)]
 mod tests {

pallets/traits/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ghost-traits"
-version = "0.4.6"
+version = "0.4.7"
 description = "Shared traits including `GhostHasher`, `NetworkDataBasicHandler`, `BoundedBTreeMap`, `MerkleTree` and more."
 license.workspace = true
 authors.workspace = true

pallets/traits/src/exodus.rs

@@ -6,7 +6,14 @@ use sp_std::{
     vec::Vec,
 };
 
-pub trait IdentifierConverter<A, E> {
+pub trait CiphersuiteSizes<A, E> {
+    fn header_bytes_len(&self) -> usize;
+    fn element_bytes_len(&self) -> usize;
+    fn signature_bytes_len(&self) -> usize;
+    fn scalar_bytes_len(&self) -> usize;
+}
+
+pub trait IdentifierConverter<A, E>: CiphersuiteSizes<A, E> {
     fn non_zero_index(index: A) -> Result<A, E>;
     fn convert_identifier_to_index(bytes: &[u8]) -> Result<A, E>;
 }
@@ -15,11 +22,6 @@ pub trait FlexibleRoundOptimizedSchnorrThresholdSignature<A, E>: MerkleTreeBuild
     type Packages<'a>;
     type NoncePackages<'a>;
 
-    fn header_bytes_len(&self) -> usize;
-    fn element_bytes_len(&self) -> usize;
-    fn signature_bytes_len(&self) -> usize;
-    fn scalar_bytes_len(&self) -> usize;
-
     fn verify_signature_share(
         &self,
         i: A,
@@ -91,13 +93,14 @@ pub trait MerkleTreeBuilder<A, E>: IdentifierConverter<A, E> {
     type Hash;
 
     fn build_merkle_tree<C, V, F>(
+        size_hint: usize,
         verifying_shares: &BTreeMap<Identifier<C>, V>,
         serialize_fn: F,
     ) -> Result<Vec<Self::Hash>, E>
     where
         C: frost_core::Ciphersuite,
         <<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
-		F: Fn(&V) -> Result<Vec<u8>, E>;
+		F: Fn(&V, &mut Vec<u8>) -> Result<(), E>;
 
     fn generate_merkle_proof_from_tree<C, V, F>(
         verifying_shares: &BTreeMap<Identifier<C>, V>,

pallets/traits/src/hashing.rs

@@ -21,6 +21,13 @@ pub trait GhostHasher {
         bytes
     }
 
+    fn combine_hashes(left: Self::Hash, right: Self::Hash) -> Self::Hash;
+    fn combine_hashes_prefixed(
+        prefix: u8,
+        left: Self::Hash,
+        right: Self::Hash,
+    ) -> Self::Hash;
+
     fn hash(data: &[u8]) -> Self::Hash;
     fn empty() -> Self::Hash;
     fn hash_len() -> usize;