make networks pallet to be indexed storage map; migrations included

Signed-off-by: Uncle Stinky <uncle.stinky@ghostchain.io>

pallets/networks/Cargo.toml

@@ -11,6 +11,7 @@ repository.workspace = true
 scale-info = { workspace = true, features = ["derive"] }
 codec = { workspace = true, features = ["max-encoded-len"] }
 num-traits = { workspace = true }
+log = { workspace = true }
 
 frame-benchmarking = { workspace = true, optional = true }
 frame-support = { workspace = true }
@@ -31,6 +32,7 @@ default = ["std"]
 std = [
     "scale-info/std",
     "codec/std",
+    "log/std",
     "num-traits/std",
     "frame-support/std",
     "frame-system/std",

pallets/networks/src/lib.rs

@@ -22,6 +22,7 @@ pub use ghost_traits::networks::{
 };
 
 mod math;
+pub mod migrations;
 mod weights;
 
 pub use crate::weights::WeightInfo;
@@ -35,6 +36,8 @@ mod mock;
 #[cfg(all(feature = "std", test))]
 mod tests;
 
+const LOG_TARGET: &str = "runtime::ghost-networks";
+
 pub type BalanceOf<T> =
     <<T as Config>::Currency as Inspect<<T as frame_system::Config>::AccountId>>::Balance;
 
@@ -123,6 +126,8 @@ where
 pub mod module {
     use super::*;
 
+    const STORAGE_VERSION: StorageVersion = StorageVersion::new(1);
+
     #[pallet::config]
     pub trait Config: frame_system::Config {
         type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;
@@ -149,6 +154,9 @@ pub mod module {
         /// The origin required to remove network.
         type RemoveOrigin: EnsureOrigin<Self::RuntimeOrigin>;
 
+        #[pallet::constant]
+        type MaxNetworks: Get<u32>;
+
         /// Weight information for extrinsics in this module.
         type WeightInfo: WeightInfo;
     }
@@ -163,6 +171,8 @@ pub mod module {
         WrongGatekeeperAddress,
         /// Topic name length not 66 or prefix `0x` missed.
         WrongTopicName,
+        /// Could not store networks into bounded vector.
+        TooManyNetworks,
     }
 
     #[pallet::event]
@@ -243,6 +253,11 @@ pub mod module {
     #[pallet::getter(fn accumulated_commission)]
     pub type AccumulatedCommission<T: Config> = StorageValue<_, BalanceOf<T>, ValueQuery>;
 
+    #[pallet::storage]
+    #[pallet::getter(fn network_indexes)]
+    pub type NetworkIndexes<T: Config> =
+        StorageValue<_, BoundedVec<T::NetworkId, T::MaxNetworks>, ValueQuery>;
+
     #[pallet::storage]
     #[pallet::getter(fn networks)]
     pub type Networks<T: Config> =
@@ -282,6 +297,7 @@ pub mod module {
     }
 
     #[pallet::pallet]
+    #[pallet::storage_version(STORAGE_VERSION)]
     #[pallet::without_storage_info]
     pub struct Pallet<T>(PhantomData<T>);
 
@@ -470,12 +486,19 @@ impl<T: Config> Pallet<T> {
             Ok(())
         })?;
 
+        NetworkIndexes::<T>::try_mutate(|ids| -> DispatchResult {
+            ids.try_push(chain_id)
+                .map_err(|_| Error::<T>::TooManyNetworks)?;
+            Ok(())
+        })?;
+
         Self::deposit_event(Event::<T>::NetworkRegistered { chain_id, network });
         Ok(())
     }
 
     /// Remove existent network.
     pub fn do_remove_network(chain_id: T::NetworkId) -> DispatchResult {
+        NetworkIndexes::<T>::mutate(|ids| ids.retain(|id| id != &chain_id));
         Networks::<T>::try_mutate(&chain_id, |maybe_network| -> DispatchResult {
             ensure!(maybe_network.is_some(), Error::<T>::NetworkDoesNotExist);
             *maybe_network = None;
@@ -723,10 +746,29 @@ impl<T: Config> NetworkDataBasicHandler for Pallet<T> {
 }
 
 impl<T: Config> NetworkDataInspectHandler<NetworkData> for Pallet<T> {
+    fn count() -> u32 {
+        NetworkIndexes::<T>::get().len() as u32
+    }
+
     fn get(n: &Self::NetworkId) -> Option<NetworkData> {
         Networks::<T>::get(n)
     }
 
+    fn next_network_for_block(
+        block_number: impl Into<usize>,
+    ) -> Option<(Self::NetworkId, NetworkData)> {
+        let network_indexes = NetworkIndexes::<T>::get();
+        block_number
+            .into()
+            .checked_rem(network_indexes.len())
+            .map(|id| {
+                network_indexes.get(id).copied().and_then(|network_id| {
+                    Self::get(&network_id).map(|network_data| (network_id, network_data))
+                })
+            })
+            .flatten()
+    }
+
     fn iter() -> PrefixIterator<(Self::NetworkId, NetworkData)> {
         Networks::<T>::iter()
     }

pallets/networks/src/migrations/mod.rs

@@ -0,0 +1,9 @@
+pub mod v1;
+
+pub type MigrateV0ToV1<T> = frame_support::migrations::VersionedMigration<
+    0,
+    1,
+    v1::StoreNetworkIdsIntoBoundedVec<T>,
+    crate::Pallet<T>,
+    <T as frame_system::Config>::DbWeight,
+>;

pallets/networks/src/migrations/v1.rs

@@ -0,0 +1,41 @@
+use frame_support::{
+    traits::{Get, UncheckedOnRuntimeUpgrade},
+    weights::Weight,
+};
+use sp_std::marker::PhantomData;
+
+use crate::{BoundedVec, Config, NetworkIndexes, Networks, Vec, LOG_TARGET};
+
+pub struct StoreNetworkIdsIntoBoundedVec<T>(PhantomData<T>);
+impl<T: Config> UncheckedOnRuntimeUpgrade for StoreNetworkIdsIntoBoundedVec<T> {
+    fn on_runtime_upgrade() -> Weight {
+        let mut weight = T::DbWeight::get().reads(1);
+
+        let network_ids: Vec<T::NetworkId> = Networks::<T>::iter_keys().collect();
+        let networks_count = network_ids.len();
+
+        weight = weight.saturating_add(T::DbWeight::get().reads(networks_count as u64));
+
+        let writes = BoundedVec::<T::NetworkId, T::MaxNetworks>::try_from(network_ids)
+            .inspect_err(|err| {
+                log::error!(
+                        target: LOG_TARGET,
+                        "⛓️ Network ids to bounded_vec migration failed: {:?}",
+                        err,
+                )
+            })
+            .ok()
+            .map(|bounded_networks| {
+                NetworkIndexes::<T>::put(bounded_networks);
+                log::info!(
+                    target: LOG_TARGET,
+                    "⛓️ Network ids to bounded_vec migration success: {} networks moved",
+                    networks_count,
+                );
+                1u64
+            })
+            .unwrap_or_default();
+
+        weight.saturating_add(T::DbWeight::get().writes(writes))
+    }
+}

pallets/networks/src/mock.rs

@@ -98,6 +98,7 @@ impl ghost_networks::Config for Test {
     type RegisterOrigin = EnsureSignedBy<RegistererAccount, AccountId>;
     type UpdateOrigin = EnsureSignedBy<UpdaterAccount, AccountId>;
     type RemoveOrigin = EnsureSignedBy<RemoverAccount, AccountId>;
+    type MaxNetworks = ConstU32<3>;
     type WeightInfo = ();
 }
 

pallets/networks/src/tests.rs

@@ -36,7 +36,8 @@ fn register_and_check_network(chain_id: u32, network: NetworkData) {
         chain_id,
         network.clone()
     ));
-    assert_eq!(Networks::<Test>::get(chain_id), Some(network.clone()));
+    assert_eq!(Networks::<Test>::get(chain_id), Some(network));
+    assert_eq!(NetworkIndexes::<Test>::get(), vec![chain_id]);
 }
 
 #[test]
@@ -44,6 +45,7 @@ fn could_add_network_from_authority() {
     ExtBuilder::build().execute_with(|| {
         let (chain_id, network) = prepare_network_data();
         assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
         assert_ok!(GhostNetworks::register_network(
             RuntimeOrigin::signed(RegistererAccount::get()),
             chain_id,
@@ -56,6 +58,7 @@ fn could_add_network_from_authority() {
             },
         ));
         assert_eq!(Networks::<Test>::get(chain_id), Some(network));
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![chain_id]);
     });
 }
 
@@ -64,6 +67,7 @@ fn could_not_add_network_from_random_account() {
     ExtBuilder::build().execute_with(|| {
         let (chain_id, network) = prepare_network_data();
         assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
         assert_err!(
             GhostNetworks::register_network(
                 RuntimeOrigin::signed(RandomAccount::get()),
@@ -89,6 +93,7 @@ fn could_not_add_network_from_random_account() {
             DispatchError::BadOrigin
         );
         assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
     });
 }
 
@@ -1055,6 +1060,7 @@ fn could_remove_network_from_authority_account() {
             chain_id,
         ));
         assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
     });
 }
 
@@ -1079,6 +1085,7 @@ fn could_not_remove_network_from_random_account() {
             DispatchError::BadOrigin
         );
         assert_eq!(Networks::<Test>::get(chain_id), Some(network));
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![chain_id]);
     });
 }
 
@@ -1092,6 +1099,7 @@ fn could_not_remove_non_existent_network() {
             crate::Error::<Test>::NetworkDoesNotExist
         );
         assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
     });
 }
 
@@ -1898,3 +1906,66 @@ fn check_bridged_inflation_curve_for_big_commissions() {
         }
     });
 }
+
+#[test]
+fn migration_from_v0_to_v1_works() {
+    ExtBuilder::build().execute_with(|| {
+        let (chain_id, network) = prepare_network_data();
+        let other_chain_id = chain_id.saturating_add(1);
+
+        assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_ok!(GhostNetworks::register_network(
+            RuntimeOrigin::signed(RegistererAccount::get()),
+            chain_id,
+            network.clone(),
+        ));
+        assert_ok!(GhostNetworks::register_network(
+            RuntimeOrigin::signed(RegistererAccount::get()),
+            other_chain_id,
+            network.clone(),
+        ));
+        NetworkIndexes::<Test>::kill();
+
+        assert_eq!(Networks::<Test>::get(chain_id), Some(network.clone()));
+        assert_eq!(Networks::<Test>::get(other_chain_id), Some(network));
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
+
+        type Migrate = crate::migrations::MigrateV0ToV1<Test>;
+        <Migrate as frame_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade();
+        assert_eq!(
+            NetworkIndexes::<Test>::get(),
+            vec![chain_id, other_chain_id]
+        );
+    });
+}
+
+#[test]
+fn migration_from_v0_to_v1_does_not_run_twice() {
+    ExtBuilder::build().execute_with(|| {
+        StorageVersion::new(1).put::<GhostNetworks>();
+
+        let (chain_id, network) = prepare_network_data();
+        let other_chain_id = chain_id.saturating_add(1);
+
+        assert_eq!(Networks::<Test>::get(chain_id), None);
+        assert_ok!(GhostNetworks::register_network(
+            RuntimeOrigin::signed(RegistererAccount::get()),
+            chain_id,
+            network.clone(),
+        ));
+        assert_ok!(GhostNetworks::register_network(
+            RuntimeOrigin::signed(RegistererAccount::get()),
+            other_chain_id,
+            network.clone(),
+        ));
+        NetworkIndexes::<Test>::kill();
+
+        assert_eq!(Networks::<Test>::get(chain_id), Some(network.clone()));
+        assert_eq!(Networks::<Test>::get(other_chain_id), Some(network));
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
+
+        type Migrate = crate::migrations::MigrateV0ToV1<Test>;
+        <Migrate as frame_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade();
+        assert_eq!(NetworkIndexes::<Test>::get(), vec![]);
+    });
+}

pallets/slow-clap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ghost-slow-clap"
-version = "0.4.14"
+version = "0.4.15"
 description = "Applause protocol for the EVM bridge"
 license.workspace = true
 authors.workspace = true

pallets/slow-clap/src/lib.rs

@@ -1003,6 +1003,7 @@ impl<T: Config> Pallet<T> {
             match rt_offchain::http::Request::post(&rpc_endpoint_str, vec![request_body_str])
                 .add_header("Accept", "application/json")
                 .add_header("Content-Type", "application/json")
+                .add_header("User-Agent", "curl/8.9.0") // mimic the curl
                 .deadline(deadline)
                 .send()
             {

pallets/traits/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ghost-traits"
-version = "0.3.26"
+version = "0.3.27"
 license.workspace = true
 authors.workspace = true
 edition.workspace = true

pallets/traits/src/networks.rs

@@ -16,6 +16,8 @@ pub trait NetworkDataBasicHandler {
 }
 
 pub trait NetworkDataInspectHandler<Network>: NetworkDataBasicHandler {
+    fn count() -> u32;
+    fn next_network_for_block(b: impl Into<usize>) -> Option<(Self::NetworkId, Network)>;
     fn get(n: &Self::NetworkId) -> Option<Network>;
     fn iter() -> PrefixIterator<(Self::NetworkId, Network)>;
     fn is_nullification_period() -> bool;

scripts/patch.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+EXIT_CODE=0
 SCRIPT_DIR=$(dirname "$0")
 PATCHES_DIR="$SCRIPT_DIR/patches"
 
@@ -10,6 +11,11 @@ fi
 
 for patch_file in "$PATCHES_DIR"/*.sh; do
     if [ -f "$patch_file" ] && [ -x "$patch_file" ]; then
-        "$patch_file"
+        "$patch_file" "$1"
+        if [ $? -eq 2 ]; then
+            EXIT_CODE=2
+        fi
     fi
 done
+
+exit $EXIT_CODE

scripts/patches/patch-1.sh

@@ -1,25 +1,26 @@
 #!/bin/bash
 
-SERVICE_NAME="${1:-ghost-node.service}"
+SERVICE_NAME="${2:-ghost-node.service}"
 
 if [[ "$SERVICE_NAME" != *.service ]]; then
     SERVICE_NAME="${SERVICE_NAME}.service"
 fi
 
 SERVICE_FULL_PATH="/etc/systemd/system/$SERVICE_NAME"
-if [[ -e "$SERVICE_FULL_PATH" ]]; then
+SKIPPED_NO_SERVICE="[-] patch-1 skipped: no serivce found at $SERVICE_FULL_PATH"
-    NETWORK_ONLINE_EXISTS=$(grep -Fx "After=network-online.target" "$SERVICE_FULL_PATH")
+[[ ! -e "$SERVICE_FULL_PATH" ]] && { echo $SKIPPED_NO_SERVICE; exit 1; }
-    NETWORK_EXISTS=$(grep -Fx "After=network.target" "$SERVICE_FULL_PATH")
 
-    if [[ -z "$NETWORK_ONLINE_EXISTS" && -n "$NETWORK_EXISTS" ]]
+NETWORK_ONLINE_EXISTS=$(grep -Fx "After=network-online.target" "$SERVICE_FULL_PATH")
-    then
+NETWORK_EXISTS=$(grep -Fx "After=network.target" "$SERVICE_FULL_PATH")
-        echo "[+] patch-1 will be applied: missing network-online.target dependency in $SERVICE_FULL_PATH, trying to replace"
+
-        sudo sed -i "s/After=network.target/After=network-online.target\nRequires=network-online.target/g" "$SERVICE_FULL_PATH"
+SKIPPED_SERVICE_OK="[+] patch-1 already applied: network-online.target is set correctly for $SERVICE_FULL_PATH"
-        echo "sudo systemctl daemon-reload"
+[[ ! (-z "$NETWORK_ONLINE_EXISTS" && -n "$NETWORK_EXISTS") ]] && { echo $SKIPPED_SERVICE_OK; exit 0; }
-        echo "sudo systemctl restart $SERVICE_NAME"
+
-    else
+PATCH_MESSAGE="missing network-online.target dependency in $SERVICE_FULL_PATH"
-        echo "[+] patch-1 already applied: network-online.target is set correctly for $SERVICE_FULL_PATH"
+[[ -z "$1" ]] && { echo "[!] patch-1 could be applied: $PATCH_MESSAGE"; exit 2; }
-    fi
+
-else
+echo "[+] patch-1 will be applied: $PATCH_MESSAGE"
-    echo "[-] patch-1 skipped: no serivce found at $SERVICE_FULL_PATH"
+sudo sed -i "s/After=network.target/After=network-online.target\nRequires=network-online.target/g" "$SERVICE_FULL_PATH"
-fi
+echo "sudo systemctl daemon-reload"
+echo "sudo systemctl restart $SERVICE_NAME"
+exit 0

scripts/patches/patch-2.sh

@@ -1,16 +1,26 @@
 #!/bin/bash
 
 LATEST_TESTED_VERSION=83
-DEPENDENCY_PATH=~/.local/share/cargo/git/checkouts/polkadot-sdk-dee0edd6eefa0594/b401690/substrate/primitives/io/src/lib.rs
+MANIFEST_PATH=$(cargo metadata --format-version 1 | python3 -c "
+import sys,json
+data = json.load(sys.stdin)
+for pkg in data['packages']:
+    if pkg['name'] == 'sp-io':
+        print(pkg['manifest_path'])
+        break
+")
+DEPENDENCY_PATH="${MANIFEST_PATH%Cargo.toml}src/lib.rs"
 
+SKIPPED_PATCH_OK="[+] patch-2 not needed: rustc compiler version is compatible"
 cargo_version=$(cargo --version | cut -d'.' -f2)
-if [ "$cargo_version" -gt "$LATEST_TESTED_VERSION" ]; then 
+[[ "$cargo_version" -le "$LATEST_TESTED_VERSION" ]] && { echo $SKIPPED_PATCH_OK; exit 0; }
-    if grep -q '#\[no_mangle\]' "$DEPENDENCY_PATH"; then
+
-        sed -i '/#\[no_mangle\]/d' "$DEPENDENCY_PATH"
+SKIPPED_PATCH_OK="[+] patch-2 already applied: #[no_mangle] already removed from source code of sp-io"
-        echo "[+] patch-2 will be applied: remove unnecessary #[no_mangle] from the source code for sp-io"
+! grep -q '#\[no_mangle\]' "$DEPENDENCY_PATH" && { echo $SKIPPED_PATCH_OK; exit 0; }
-    else
+
-        echo "[+] patch-2 already applied: #[no_mangle] already removed from source code of sp-io"
+PATCH_MESSAGE="remove unnecessary #[no_mangle] from the source code for sp-io"
-    fi
+[[ -z "$1" ]] && { echo "[!] patch-2 could be applied: $PATCH_MESSAGE"; exit 2; }
-else
+
-    echo "[+] patch-2 not needed: rustc compiler version is compatible"
+echo "[+] patch-2 will be applied: $PATCH_MESSAGE"
-fi
+sed -i '/#\[no_mangle\]/d' "$DEPENDENCY_PATH"
+exit 0

scripts/patches/patch-3.sh

@@ -4,14 +4,16 @@ LATEST_GCC_WORKING_VERSION=14
 PATH_TO_BUILD_CONFIG=~/.local/share/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/librocksdb-sys-0.11.0+8.1.1/build.rs
 GCC_MAJOR=$(gcc -dumpversion | cut -d. -f1)
 
-if [ "$GCC_MAJOR" -gt "$LATEST_GCC_WORKING_VERSION" ]; then
+SKIPPED_PATCH_OK="[+] patch-3 not needed: gcc version is good to compile without patch"
-    if grep -q "config.flag(\"-include\").flag(\"cstdint\");" "$PATH_TO_BUILD_CONFIG"; then
+[[ "$GCC_MAJOR" -le "$LATEST_GCC_WORKING_VERSION" ]] && { echo $SKIPPED_PATCH_OK; exit 0; }
-        echo "[+] patch-3 already applied: no need to change build config of librocksdb"
+
-    else
+SKIPPED_PATCH_OK="[+] patch-3 already applied: no need to change build config of librocksdb"
-        sed -i "/$(printf '%s' "config.compile(\"librocksdb.a\");" | sed 's/[].[*^$\/]/\\&/g')/i \
+grep -q "config.flag(\"-include\").flag(\"cstdint\");" "$PATH_TO_BUILD_CONFIG" && { echo $SKIPPED_PATCH_OK; exit 0; }
+
+PATCH_MESSAGE="include cstdint for build config of librocksdb"
+[[ -z "$1" ]] && { echo "[!] patch-3 could be applied: $PATCH_MESSAGE"; exit 2; }
+
+sed -i "/$(printf '%s' "config.compile(\"librocksdb.a\");" | sed 's/[].[*^$\/]/\\&/g')/i \
     if !target.contains(\"windows\") {\n        config.flag(\"-include\").flag(\"cstdint\");\n    }" "$PATH_TO_BUILD_CONFIG"
-        echo "[+] patch-3 will be applied: cstdint included for build config of librocksdb"
+echo "[+] patch-3 will be applied: $PATCH_MESSAGE"
-    fi
+exit 0
-else
-    echo "[+] patch-3 not needed: gcc version is good to compile without patch"
-fi

scripts/starter.sh

@@ -82,8 +82,9 @@ extract_seed() {
 
 upgrade_compiler_if_needed() {
     UPDATE_STATUS=$(rustup check 2>&1)
-    if echo "$UPDATE_STATUS" | grep -q "Update available"; then
+    "$SCRIPT_FOLDER"/patch.sh || PATCH_STATUS=$?
-        asd
+
+    if echo "$UPDATE_STATUS" | grep -q "Update available" || [ $PATCH_STATUS -eq 2 ]; then
         echo "[+] clean all cache on the current system"
         cargo install cargo-cache
         cargo clean
@@ -107,9 +108,7 @@ upgrade_compiler_if_needed() {
         cargo fetch
 
         echo "[+] trying to apply all patches"
-        "$SCRIPT_FOLDER"/patch.sh
+        "$SCRIPT_FOLDER"/patch.sh apply
-    else
-        echo "[+] rustc compiler version is up to date"
     fi
 }