optimize merkle tree prefixed computations

Signed-off-by: Uncle Stinky <uncle.stinky@ghostchain.io>

pallets/exodus/src/benchmarking.rs

@@ -415,10 +415,16 @@ mod benchmarks {
                     dummy_values.insert(identifier, dummy_value.clone());
                 }
 
+                let identifier_size = network_curve.scalar_bytes_len();
+
                 let merkle_tree =
                 	NetworkCurve::build_merkle_tree(
-                	    &dummy_values,
+                	    identifier_size + dummy_value.len(),
-                	    |value: &Vec<u8>| Ok(value.clone()),
+                        &dummy_values,
+                	    |value, buffer| {
+                            buffer.extend_from_slice(value.as_ref());
+                            Ok(())
+                        }
                     ).expect("Merkle tree for verifying shares should be valid");
 
                 let merkle_root = merkle_tree.last().copied().unwrap();
@@ -697,10 +703,16 @@ mod benchmarks {
                     dummy_values.insert(identifier, dummy_binding_factor.clone());
                 }
 
+                let identifier_size = network_curve.scalar_bytes_len();
+
                 let merkle_tree =
                 	NetworkCurve::build_merkle_tree(
-                	    &dummy_values,
+                	    identifier_size + dummy_scalar.len(),
-                	    |value: &Vec<u8>| Ok(value.clone()),
+                        &dummy_values,
+                	    |value, buffer| {
+                            buffer.extend_from_slice(value.as_ref());
+                            Ok(())
+                        }
                     ).expect("Merkle tree for binding factor should be valid");
 
                 let merkle_root = merkle_tree.last().copied().unwrap();

pallets/exodus/src/error.rs

@@ -43,6 +43,7 @@ pub enum ExodusError {
     InvalidMerkleProof,
     InvalidIdentityElement,
     NoActiveAuthorities,
+    TreeGenerationFailed,
     Unknown,
 }
 
@@ -88,6 +89,7 @@ impl core::fmt::Debug for ExodusError {
             ExodusError::InvalidMerkleProof => write!(fmt, "Invalid Merkle proof provided."),
             ExodusError::InvalidIdentityElement => write!(fmt, "Invalid identity element provided."),
             ExodusError::NoActiveAuthorities => write!(fmt, "No active authorities for signing EXODUS transactions."),
+            ExodusError::TreeGenerationFailed => write!(fmt, "Tree generation failed because of inconsistent indexation."),
             ExodusError::Unknown => write!(fmt, "Unknown error."),
         }
     }

pallets/exodus/src/impls/dkg.rs

@@ -3,6 +3,7 @@ use rand_chacha::rand_core::{CryptoRng, RngCore};
 
 use ghost_traits::exodus::{
     DistributedKeyGeneration, IdentifierConverter, MerkleTreeBuilder,
+    CiphersuiteSizes,
 };
 
 use crate::{AuthIndex, ExodusError, NetworkCurve};
@@ -241,6 +242,8 @@ impl DistributedKeyGeneration<AuthIndex, ExodusError> for NetworkCurve {
         round2_packages_bytes: &Self::RoundPackages,
     ) -> Self::Part3Result {
         with_ciphersuite!(self, |f| {
+            type InnerGroup = <Ciphersuite as frost_core::Ciphersuite>::Group;
+
             let round2_secret_package = f::keys::dkg::round2::SecretPackage::deserialize(round2_secret_package_bytes)
                 .map_err(|_| ExodusError::DeserializationError)?;
 
@@ -291,13 +294,24 @@ impl DistributedKeyGeneration<AuthIndex, ExodusError> for NetworkCurve {
             let key_package_bytes = key_package.serialize()
                 .map_err(|_| ExodusError::SerializationError)?;
 
+            let identifier_size = self.scalar_bytes_len();
+            let verifying_share_size = self.element_bytes_len();
+
             let verifying_shares_merkle_tree =
                 Self::build_merkle_tree(
+                    identifier_size + verifying_share_size,
                     pubkey_package.verifying_shares(),
-                    |verifying_share| verifying_share.serialize()
+                    |verifying_share, buffer| {
-                        .map_err(|_| ExodusError::SerializationError)
+                        let serialized = <InnerGroup as frost_core::Group>::serialize(
+                            &verifying_share.to_element()
+                        ).map_err(|_| ExodusError::SerializationError)?;
+
+                        buffer.extend_from_slice(serialized.as_ref());
+                        Ok(())
+                    }
                 )?;
 
+            // TODO: make it better
             let (_, merkle_proof) =
                 Self::generate_merkle_proof_from_tree(
                     pubkey_package.verifying_shares(),

pallets/exodus/src/impls/frost.rs

@@ -1,5 +1,5 @@
 use ghost_traits::exodus::{
-    IdentifierConverter, MerkleTreeBuilder,
+    CiphersuiteSizes, IdentifierConverter, MerkleTreeBuilder,
     FlexibleRoundOptimizedSchnorrThresholdSignature,
 };
 use sp_std::{
@@ -15,29 +15,6 @@ impl FlexibleRoundOptimizedSchnorrThresholdSignature<AuthIndex, ExodusError> for
     type Packages<'a> = BTreeMap<AuthIndex, &'a [u8]>;
     type NoncePackages<'a> = BTreeMap<AuthIndex, (&'a [u8], &'a [u8])>;
 
-    fn header_bytes_len(&self) -> usize { 5 }
-
-    fn signature_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 64,
-            NetworkCurve::Ed25519 => 64,
-        }
-    }
-
-    fn element_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 33,
-            NetworkCurve::Ed25519 => 32,
-        }
-    }
-
-    fn scalar_bytes_len(&self) -> usize {
-        match self {
-            NetworkCurve::Secp256k1 => 32,
-            NetworkCurve::Ed25519 => 32,
-        }
-    }
-
     fn verify_signature_share(
         &self,
         index: AuthIndex,
@@ -330,11 +307,16 @@ impl FlexibleRoundOptimizedSchnorrThresholdSignature<AuthIndex, ExodusError> for
                 return Err(ExodusError::InvalidParticipantId);
             }
 
+            let identifier_size = self.scalar_bytes_len();
+            let binding_factor_size = self.scalar_bytes_len();
+
             let binding_factor_tree = Self::build_merkle_tree(
+                identifier_size + binding_factor_size,
                 &binding_factors_list,
-                |binding_factor| {
+                |binding_factor, buffer| {
                     let serialized = <InnerField as frost_core::Field>::serialize(binding_factor);
-	                Ok(serialized.as_ref().to_vec())
+                    buffer.extend_from_slice(serialized.as_ref());
+                    Ok(())
                 }
             )?;
 

pallets/exodus/src/impls/merkle.rs

@@ -13,15 +13,16 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
     type Hash = <SubstrateBlake2Hasher as GhostHasher>::Hash;
 
     fn build_merkle_tree<C, V, F>(
+        size_hint: usize,
         values: &BTreeMap<Identifier<C>, V>,
-        serialize_fn: F,
+        update_preimage_buffer: F,
     ) -> Result<Vec<Self::Hash>, ExodusError>
 	where
 		C: frost_core::Ciphersuite,
 		<<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
-		F: Fn(&V) -> Result<Vec<u8>, ExodusError>
+		F: Fn(&V, &mut Vec<u8>) -> Result<(), ExodusError>
 	{
-        use ghost_helpers::merkle_tree::generate_tree;
+        use ghost_helpers::merkle_tree::{generate_tree, MerkleTreeError};
 
         type CField<C> = <<C as Ciphersuite>::Group as Group>::Field;
 
@@ -34,25 +35,28 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             })
             .ok_or(ExodusError::IncorrectNumberOfIdentifiers)?;
 
-        generate_tree::<SubstrateBlake2Hasher, _, _, _, _>(
+        let maybe_tree = generate_tree::<SubstrateBlake2Hasher, _, _, _, _>(
             max_index,
+            size_hint,
             values.iter(),
-            |(identifier, value)| {
+            |(identifier, value), buffer| {
                 let id_scalar = identifier.to_scalar();
                 let id_bytes = <CField<C> as frost_core::Field>::serialize(&id_scalar);
 
                 let index = Self::convert_identifier_to_index(id_bytes.as_ref())?;
-                let value_bytes = serialize_fn(value).map_err(|_| ExodusError::SerializationError)?;
 
-                // NOTE: revisit
+                buffer.extend_from_slice(id_bytes.as_ref());
-                // Is it possible to use [u8; CONSTANT] where constant is
+                update_preimage_buffer(value, buffer)?;
-                // purely dependant on provided generic.
-                let mut preimage = Vec::<u8>::with_capacity(id_bytes.as_ref().len() + value_bytes.len());
-                preimage.extend_from_slice(id_bytes.as_ref());
-                preimage.extend_from_slice(value_bytes.as_ref());
 
-                Ok((index as usize, preimage))
+                Ok(index as usize)
-            })
+            });
+
+        match maybe_tree {
+            Ok(tree) => Ok(tree),
+            Err(MerkleTreeError::Generate(exodus_err)) => Err(exodus_err),
+            Err(MerkleTreeError::InvalidIndex) => Err(ExodusError::TreeGenerationFailed),
+            Err(MerkleTreeError::Empty) => Err(ExodusError::IncorrectNumberOfIdentifiers),
+        }
     }
 
     fn generate_merkle_proof_from_tree<C, V, F>(
@@ -66,7 +70,7 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
 		<<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
 		F: Fn(&V) -> Result<Vec<u8>, ExodusError>
 	{
-        use ghost_helpers::merkle_tree::generate_proof;
+        use ghost_helpers::merkle_tree::{generate_proof, MerkleTreeError};
 
         type CField<C> = <<C as Ciphersuite>::Group as Group>::Field;
 
@@ -75,7 +79,7 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             .map_err(|_| ExodusError::InvalidParticipantId)?;
 
         let value = values.get(&identifier).ok_or(ExodusError::InvalidParticipantId)?;
-        let value_bytes = serialize_fn(&value).map_err(|_| ExodusError::SerializationError)?;
+        let value_bytes = serialize_fn(&value)?;
 
         let max_index = values.keys()
             .next_back()
@@ -86,11 +90,15 @@ impl MerkleTreeBuilder<AuthIndex, ExodusError> for NetworkCurve {
             })
             .ok_or(ExodusError::IncorrectNumberOfIdentifiers)?;
 
-        let proof = generate_proof::<SubstrateBlake2Hasher, _>(
+        let proof = generate_proof::<SubstrateBlake2Hasher, _, _>(
             merkle_tree,
             max_index,
             authority_index,
-        );
+        ).map_err(|err| match err {
+            MerkleTreeError::Generate(exodus_err) => exodus_err,
+            MerkleTreeError::InvalidIndex => ExodusError::TreeGenerationFailed,
+            MerkleTreeError::Empty => ExodusError::IncorrectNumberOfIdentifiers,
+        })?;
 
         Ok((value_bytes, proof))
     }

pallets/exodus/src/impls/mod.rs

@@ -72,3 +72,6 @@ mod ecdh;
 
 #[macro_use]
 mod frost;
+
+#[macro_use]
+mod sizes;

pallets/exodus/src/impls/sizes.rs

@@ -0,0 +1,29 @@
+use ghost_helpers::networks::NetworkCurve;
+use ghost_traits::exodus::CiphersuiteSizes;
+
+use crate::{AuthIndex, ExodusError};
+
+impl CiphersuiteSizes<AuthIndex, ExodusError> for NetworkCurve {
+    fn header_bytes_len(&self) -> usize { 5 }
+
+    fn signature_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 64,
+            NetworkCurve::Ed25519 => 64,
+        }
+    }
+
+    fn element_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 33,
+            NetworkCurve::Ed25519 => 32,
+        }
+    }
+
+    fn scalar_bytes_len(&self) -> usize {
+        match self {
+            NetworkCurve::Secp256k1 => 32,
+            NetworkCurve::Ed25519 => 32,
+        }
+    }
+}

pallets/exodus/src/lib.rs

@@ -48,6 +48,7 @@ use ghost_traits::{
     exodus::{
         MerkleTreeBuilder, DistributedKeyGeneration, EllipticCurveDiffieHellman,
         FlexibleRoundOptimizedSchnorrThresholdSignature, EvmGovernanceRegistrar,
+        CiphersuiteSizes,
     },
     networks::{
         NetworkDataBasicHandler, NetworkDataInspectHandler,

pallets/exodus/src/tests.rs

@@ -1571,9 +1571,17 @@ fn test_merkle_tree_success_power_of_two_for_shares() {
     let indices = (1..=32).collect::<Vec<AuthIndex>>();
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1608,11 +1616,16 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
     let indices = (1..=32).collect::<Vec<AuthIndex>>();
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1623,7 +1636,7 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
             &identifier.serialize(),
         ).unwrap();
 
-        let (shares_bytes, proof) =
+        let (scalar_bytes, proof) =
             NetworkCurve::generate_merkle_proof_from_tree::<Secp256K1Sha256TR, _, _>(
                 &scalars,
                 &tree,
@@ -1635,7 +1648,7 @@ fn test_merkle_tree_success_power_of_two_for_scalar() {
             ).unwrap();
 
         let result = network_curve.verify_merkle_proof(
-            &shares_bytes,
+            &scalar_bytes,
             &proof,
             root,
             auth_index,
@@ -1657,9 +1670,17 @@ fn test_random_sparse_combinations_stress_for_shares() {
     let indices: Vec<AuthIndex> = unique_indices.into_iter().collect();
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1695,11 +1716,16 @@ fn test_random_sparse_combinations_stress_for_scalars() {
     let indices: Vec<AuthIndex> = unique_indices.into_iter().collect();
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1733,9 +1759,17 @@ fn test_participants_with_high_index_gap_for_shares() {
     let indices = vec![1, 1023];
     let shares = generate_mock_secp256k1_shares(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |share| share.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1765,11 +1799,16 @@ fn test_participants_with_high_index_gap_for_scalars() {
     let indices = vec![1, 1023];
     let scalars = generate_mock_secp256k1_scalars(&indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1803,9 +1842,17 @@ fn test_regression_consecutive_vs_position_for_shares() {
     let custom_indices = vec![2, 3];
     let shares = generate_mock_secp256k1_shares(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |scalar| scalar.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1828,11 +1875,16 @@ fn test_regression_consecutive_vs_position_for_scalars() {
     let custom_indices = vec![2, 3];
     let scalars = generate_mock_secp256k1_scalars(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 
@@ -1859,9 +1911,17 @@ fn test_attack_with_unknown_authority_index_for_shares() {
     let custom_indices = vec![1, 2, 3];
     let shares = generate_mock_secp256k1_shares(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_share_len = network_curve.element_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_share_len,
         &shares,
-        |scalar| scalar.serialize().map_err(|_| ExodusError::SerializationError),
+        |share, buffer| {
+            let serialized = share.serialize().map_err(|_| ExodusError::SerializationError)?;
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
+        }
     ).unwrap();
 
     let root = *tree.last().unwrap();
@@ -1884,11 +1944,16 @@ fn test_attack_with_unknown_authority_index_for_scalars() {
     let custom_indices = vec![1, 2, 3];
     let scalars = generate_mock_secp256k1_scalars(&custom_indices);
 
+    let identifier_len = network_curve.scalar_bytes_len();
+    let secret_scalar_len = network_curve.scalar_bytes_len();
+
     let tree = NetworkCurve::build_merkle_tree::<Secp256K1Sha256TR, _, _>(
+        identifier_len + secret_scalar_len,
         &scalars,
-        |scalar| {
+        |scalar, buffer| {
             let serialized = <Secp256K1ScalarField as Field>::serialize(scalar);
-            Ok(serialized.as_ref().to_vec())
+            buffer.extend_from_slice(serialized.as_ref());
+            Ok(())
         }
     ).unwrap();
 

pallets/governance/src/tests.rs

@@ -79,14 +79,21 @@ fn claim_happy_path() {
         let users = vec![user_1.clone(), user_2.clone(), user_3.clone()];
         let max_index = 2u8;
 
+        let tmp_package_probe = ClaimPackage::<Test> {
+            shares: user_1.allowed_shares,
+            merkle_proof: Default::default(),
+            token_id: user_1.token_id,
+            index: 0u32,
+        };
+        let size_hint = tmp_package_probe.get_preimage(&user_1.evm_address, network_id).as_ref().len();
+
         let raw_values_complex = vec![(0u8, &user_1), (1u8, &user_2), (2u8, &user_3)];
-        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, _>(
+        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, ()>(
             max_index,
+            size_hint,
             raw_values_complex,
 
-            |item: (u8, &NewTestUser)| -> Result<(usize, Vec<u8>), ()> {
+            |(index, u), buffer| {
-                let (index, u) = item;
-
                 let tmp_package = ClaimPackage::<Test> {
                     shares: u.allowed_shares,
                     merkle_proof: Default::default(),
@@ -94,9 +101,12 @@ fn claim_happy_path() {
                     index: index as u32
                 };
 
-                Ok((index as usize, tmp_package.get_preimage(&u.evm_address, network_id).to_vec()))
+                let preimage_bytes = tmp_package.get_preimage(&u.evm_address, network_id);
+                buffer.extend_from_slice(preimage_bytes.as_ref());
+
+                Ok(index as usize)
             }
-        ).unwrap();
+        ).ok().unwrap();
 
         let merkle_root = *merkle_tree.last().unwrap();
 
@@ -116,7 +126,9 @@ fn claim_happy_path() {
                 activated_shares: 0,
             });
 
-            let proof = generate_proof::<SubstrateKeccakHasher, u8>(&merkle_tree, max_index, slot_index as u8);
+            let proof = generate_proof::<SubstrateKeccakHasher, u8, ()>(&merkle_tree, max_index, slot_index as u8)
+                .ok()
+                .unwrap();
 
             let claim_package = ClaimPackage::<Test> {
                 shares: current_user.allowed_shares,
@@ -205,21 +217,33 @@ fn partial_claims_happy_path() {
         let max_index = 0u8;
         let raw_values_complex = vec![(0u8, &current_user)];
 
-        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, _>(
+        let tmp_package_probe = ClaimPackage::<Test> {
+            shares: current_user.allowed_shares,
+            merkle_proof: Default::default(),
+            token_id: current_user.token_id,
+            index: 0u32,
+        };
+        let size_hint = tmp_package_probe.get_preimage(&current_user.evm_address, network_id).as_ref().len();
+
+        let merkle_tree = generate_tree::<SubstrateKeccakHasher, u8, _, _, ()>(
             max_index,
+            size_hint,
             raw_values_complex,
 
-            |item: (u8, &NewTestUser)| -> Result<(usize, Vec<u8>), ()> {
+            |(index, u), buffer| {
-                let (index, u) = item;
                 let tmp_package = ClaimPackage::<Test> {
                     shares: u.allowed_shares,
                     merkle_proof: Default::default(),
                     token_id: u.token_id,
                     index: index as u32
                 };
-                Ok((index as usize, tmp_package.get_preimage(&u.evm_address, network_id).to_vec()))
+
+                let preimage_bytes = tmp_package.get_preimage(&u.evm_address, network_id);
+                buffer.extend_from_slice(preimage_bytes.as_ref());
+
+                Ok(index as usize)
             }
-        ).unwrap();
+        ).ok().unwrap();
 
         let total_shares = 1_000_000;
         let merkle_root = *merkle_tree.last().unwrap();
@@ -240,7 +264,10 @@ fn partial_claims_happy_path() {
         setup_governor(&substrate_account_b);
         setup_governor(&substrate_account_c);
 
-        let proof = generate_proof::<SubstrateKeccakHasher, u8>(&merkle_tree, max_index, 0u8);
+        let proof = generate_proof::<SubstrateKeccakHasher, u8, ()>(&merkle_tree, max_index, 0u8)
+            .ok()
+            .unwrap();
+
         let claim_package = ClaimPackage {
             shares: current_user.allowed_shares,
             merkle_proof: BoundedVec::try_from(proof).unwrap(),

pallets/helpers/src/hash_chain.rs

@@ -188,9 +188,9 @@ mod tests {
         }
     }
 
-    run_hasher_tests!(SubstrateBlake2Hasher, blake2_tests);
+    run_hasher_tests!(SubstrateBlake2Hasher, blake2_hashing_tests);
-    run_hasher_tests!(SubstrateKeccakHasher, keccak_tests);
+    run_hasher_tests!(SubstrateKeccakHasher, keccak_hashing_tests);
-    run_hasher_tests!(UtxoSha2Hasher, sha2_tests);
+    run_hasher_tests!(UtxoSha2Hasher, sha2_hashing_tests);
 
     #[test]
     fn test_verify_in_current_block_with_empty_ancestry() {

pallets/helpers/src/lib.rs

@@ -79,6 +79,13 @@ macro_rules! impl_ghost_hasher {
                 combined[$hash_bytes_len..].copy_from_slice(right.as_ref());
                 Self::hash(&combined)
             }
+
+            fn combine_hashes_prefixed(p: u8, l: Self::Hash, r: Self::Hash) -> Self::Hash {
+                let mut combined = [p; $hash_bytes_len * 2 + 1];
+                combined[1..1 + $hash_bytes_len].copy_from_slice(l.as_ref());
+                combined[1 + $hash_bytes_len..].copy_from_slice(r.as_ref());
+                Self::hash(&combined)
+            }
         }
     };
 }

pallets/traits/src/exodus.rs

@@ -6,7 +6,14 @@ use sp_std::{
     vec::Vec,
 };
 
-pub trait IdentifierConverter<A, E> {
+pub trait CiphersuiteSizes<A, E> {
+    fn header_bytes_len(&self) -> usize;
+    fn element_bytes_len(&self) -> usize;
+    fn signature_bytes_len(&self) -> usize;
+    fn scalar_bytes_len(&self) -> usize;
+}
+
+pub trait IdentifierConverter<A, E>: CiphersuiteSizes<A, E> {
     fn non_zero_index(index: A) -> Result<A, E>;
     fn convert_identifier_to_index(bytes: &[u8]) -> Result<A, E>;
 }
@@ -15,11 +22,6 @@ pub trait FlexibleRoundOptimizedSchnorrThresholdSignature<A, E>: MerkleTreeBuild
     type Packages<'a>;
     type NoncePackages<'a>;
 
-    fn header_bytes_len(&self) -> usize;
-    fn element_bytes_len(&self) -> usize;
-    fn signature_bytes_len(&self) -> usize;
-    fn scalar_bytes_len(&self) -> usize;
-
     fn verify_signature_share(
         &self,
         i: A,
@@ -91,13 +93,14 @@ pub trait MerkleTreeBuilder<A, E>: IdentifierConverter<A, E> {
     type Hash;
 
     fn build_merkle_tree<C, V, F>(
+        size_hint: usize,
         verifying_shares: &BTreeMap<Identifier<C>, V>,
         serialize_fn: F,
     ) -> Result<Vec<Self::Hash>, E>
     where
         C: frost_core::Ciphersuite,
         <<C as Ciphersuite>::Group as Group>::Field: frost_core::Field,
-		F: Fn(&V) -> Result<Vec<u8>, E>;
+		F: Fn(&V, &mut Vec<u8>) -> Result<(), E>;
 
     fn generate_merkle_proof_from_tree<C, V, F>(
         verifying_shares: &BTreeMap<Identifier<C>, V>,

pallets/traits/src/hashing.rs

@@ -22,6 +22,12 @@ pub trait GhostHasher {
     }
 
     fn combine_hashes(left: Self::Hash, right: Self::Hash) -> Self::Hash;
+    fn combine_hashes_prefixed(
+        prefix: u8,
+        left: Self::Hash,
+        right: Self::Hash,
+    ) -> Self::Hash;
+
     fn hash(data: &[u8]) -> Self::Hash;
     fn empty() -> Self::Hash;
     fn hash_len() -> usize;