From 8ca8ae7080fcb77389fac12e48e5521843ae1f41 Mon Sep 17 00:00:00 2001 From: Francisco Aguirre Date: Thu, 1 Feb 2024 15:34:46 +0100 Subject: [PATCH 1/5] Backport DepositReserveAsset fix --- Cargo.lock | 5 +- .../tests/assets/asset-hub-rococo/Cargo.toml | 1 + .../tests/assets/asset-hub-rococo/src/lib.rs | 5 +- .../src/tests/reserve_transfer.rs | 176 ++++++++++++++++++ .../runtimes/testing/penpal/src/xcm_config.rs | 49 +++-- polkadot/xcm/xcm-executor/Cargo.toml | 2 +- polkadot/xcm/xcm-executor/src/lib.rs | 17 ++ 7 files changed, 224 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95efb3f7ad14..20e9f026ab72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -868,6 +868,7 @@ dependencies = [ "assert_matches", "asset-hub-rococo-runtime", "asset-test-utils", + "cumulus-pallet-parachain-system", "emulated-integration-tests-common", "frame-support", "pallet-asset-conversion", @@ -10828,7 +10829,7 @@ dependencies = [ [[package]] name = "pallet-staking" -version = "26.0.0" +version = "26.0.1" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -18252,7 +18253,7 @@ dependencies = [ [[package]] name = "staging-xcm-executor" -version = "5.0.0" +version = "5.0.1" dependencies = [ "environmental", "frame-benchmarking", diff --git a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/Cargo.toml b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/Cargo.toml index 80391e2dcc4a..49a507473476 100644 --- a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/Cargo.toml @@ -30,3 +30,4 @@ asset-hub-rococo-runtime = { version = "0.10.0", path = "../../../../../runtimes emulated-integration-tests-common = { path = "../../../common", default-features = false, version = "1.0.0" } penpal-runtime = { version = "0.12.0", path = "../../../../../runtimes/testing/penpal" } rococo-system-emulated-network = { version = "0.1.0", path = "../../../networks/rococo-system" } +cumulus-pallet-parachain-system = { path = "../../../../../../pallets/parachain-system", default-features = false } diff --git a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/lib.rs b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/lib.rs index 3ff8c37c6465..bc29ad8a5277 100644 --- a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/lib.rs @@ -35,7 +35,7 @@ pub use emulated_integration_tests_common::{ test_parachain_is_trusted_teleporter, xcm_emulator::{ assert_expected_events, bx, helpers::weight_within_threshold, Chain, Parachain as Para, - RelayChain as Relay, Test, TestArgs, TestContext, TestExt, + RelayChain as Relay, Test, TestArgs, TestContext, TestExt, Parachain as ParachainTrait, }, xcm_helpers::{xcm_transact_paid_execution, xcm_transact_unpaid_execution}, PROOF_SIZE_THRESHOLD, REF_TIME_THRESHOLD, XCM_V3, @@ -53,6 +53,8 @@ pub use rococo_system_emulated_network::{ PenpalAParaReceiver as PenpalAReceiver, PenpalAParaSender as PenpalASender, RococoRelay as Rococo, RococoRelayReceiver as RococoReceiver, RococoRelaySender as RococoSender, + PenpalBParaReceiver as PenpalBReceiver, PenpalBParaSender as PenpalBSender, + PenpalBPara as PenpalB, }; pub const ASSET_ID: u32 = 1; @@ -65,6 +67,7 @@ pub type RelayToParaTest = Test; pub type SystemParaToRelayTest = Test; pub type SystemParaToParaTest = Test; pub type ParaToSystemParaTest = Test; +pub type ParaToParaTest = Test; /// Returns a `TestArgs` instance to be used for the Relay Chain across integration tests pub fn relay_test_args( diff --git a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/tests/reserve_transfer.rs b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/tests/reserve_transfer.rs index d0e9b72176bc..2a8315f7a047 100644 --- a/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/tests/reserve_transfer.rs +++ b/cumulus/parachains/integration-tests/emulated/tests/assets/asset-hub-rococo/src/tests/reserve_transfer.rs @@ -14,6 +14,7 @@ // limitations under the License. use crate::*; +use frame_support::traits::Get; use asset_hub_rococo_runtime::xcm_config::XcmConfig as AssetHubRococoXcmConfig; use penpal_runtime::xcm_config::XcmConfig as PenpalRococoXcmConfig; use rococo_runtime::xcm_config::XcmConfig as RococoXcmConfig; @@ -110,6 +111,66 @@ fn para_to_system_para_sender_assertions(t: ParaToSystemParaTest) { ); } +fn para_to_para_sender_assertions(t: ParaToParaTest) { + type RuntimeEvent = ::RuntimeEvent; + PenpalA::assert_xcm_pallet_attempted_complete(None); + assert_expected_events!( + PenpalA, + vec![ + // Amount to reserve transfer is transferred to Parachain's Sovereign account + RuntimeEvent::Balances( + pallet_balances::Event::Withdraw { who, amount } + ) => { + who: *who == t.sender.account_id, + amount: *amount == t.args.amount, + }, + // XCM sent to relay reserve + RuntimeEvent::ParachainSystem( + cumulus_pallet_parachain_system::Event::UpwardMessageSent { .. } + ) => {}, + ] + ); +} + +fn para_to_para_relay_hop_assertions(t: ParaToParaTest) { + type RuntimeEvent = ::RuntimeEvent; + let sov_penpal_a_on_rococo = + Rococo::sovereign_account_id_of(Rococo::child_location_of(PenpalA::para_id())); + let sov_penpal_b_on_rococo = + Rococo::sovereign_account_id_of(Rococo::child_location_of(PenpalB::para_id())); + assert_expected_events!( + Rococo, + vec![ + // Withdrawn from sender parachain SA + RuntimeEvent::Balances( + pallet_balances::Event::Withdraw { who, amount } + ) => { + who: *who == sov_penpal_a_on_rococo, + amount: *amount == t.args.amount, + }, + // Deposited to receiver parachain SA + RuntimeEvent::Balances( + pallet_balances::Event::Deposit { who, .. } + ) => { + who: *who == sov_penpal_b_on_rococo, + }, + RuntimeEvent::MessageQueue( + pallet_message_queue::Event::Processed { success: true, .. } + ) => {}, + ] + ); +} + +fn para_to_para_receiver_assertions(_: ParaToParaTest) { + type RuntimeEvent = ::RuntimeEvent; + assert_expected_events!( + PenpalB, + vec![ + RuntimeEvent::Balances(pallet_balances::Event::Deposit { .. }) => {}, + ] + ); +} + fn para_to_system_para_receiver_assertions(t: ParaToSystemParaTest) { type RuntimeEvent = ::RuntimeEvent; @@ -175,6 +236,121 @@ fn system_para_to_para_assets_receiver_assertions(_: Test) { ); } +// function assumes fees and assets have the same remote reserve +fn remote_reserve_transfer_program( + reserve: MultiLocation, + dest: MultiLocation, + beneficiary: MultiLocation, + assets: Vec, + fees: MultiAsset, + weight_limit: WeightLimit, +) -> Xcm { + let max_assets = assets.len() as u32; + let context = X1(Parachain(::ParachainInfo::get().into())); + // we spend up to half of fees for execution on reserve and other half for execution on + // destination + let (fees_half_1, fees_half_2) = match fees.fun { + Fungible(amount) => { + let fee1 = amount.saturating_div(2); + let fee2 = amount.saturating_sub(fee1); + assert!(fee1 > 0); + assert!(fee2 > 0); + (MultiAsset::from((fees.id.clone(), fee1)), MultiAsset::from((fees.id.clone(), fee2))) + }, + NonFungible(_) => unreachable!(), + }; + // identifies fee item as seen by `reserve` - to be used at reserve chain + let reserve_fees = fees_half_1.reanchored(&reserve, context).unwrap(); + // identifies fee item as seen by `dest` - to be used at destination chain + let dest_fees = fees_half_2.reanchored(&dest, context).unwrap(); + // identifies `dest` as seen by `reserve` + let dest = dest.reanchored(&reserve, context).unwrap(); + // xcm to be executed at dest + let xcm_on_dest = Xcm(vec![ + BuyExecution { fees: dest_fees, weight_limit: weight_limit.clone() }, + DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, + ]); + // xcm to be executed on reserve + let xcm_on_reserve = Xcm(vec![ + BuyExecution { fees: reserve_fees, weight_limit }, + DepositReserveAsset { assets: Wild(AllCounted(max_assets)), dest, xcm: xcm_on_dest }, + ]); + Xcm(vec![ + WithdrawAsset(assets.into()), + InitiateReserveWithdraw { + assets: Wild(AllCounted(max_assets)), + reserve, + xcm: xcm_on_reserve, + }, + ]) +} + +fn para_to_para_remote_reserve_transfer_native_assets(t: ParaToParaTest) -> DispatchResult { + let xcm = remote_reserve_transfer_program( + Parent.into(), + t.args.dest, + t.args.beneficiary, + t.args.assets.clone().into_inner(), + t.args.assets.into_inner().pop().unwrap(), + t.args.weight_limit, + ); + ::PolkadotXcm::execute( + t.signed_origin, + bx!(xcm::VersionedXcm::V3(xcm)), + Weight::MAX, + ) + .unwrap(); + Ok(()) +} + +/// Reserve Transfers of native asset from Parachain to Parachain (through Relay reserve) should +/// work +#[test] +fn reserve_transfer_native_asset_from_para_to_para() { + // Init values for Penpal Parachain + let destination = PenpalA::sibling_location_of(PenpalB::para_id()); + let beneficiary_id = PenpalBReceiver::get(); + let amount_to_send: Balance = ASSET_HUB_ROCOCO_ED * 10000; + let assets = (Parent, amount_to_send).into(); + + let test_args = TestContext { + sender: PenpalASender::get(), + receiver: PenpalBReceiver::get(), + args: para_test_args(destination, beneficiary_id, amount_to_send, assets, None, 0), + }; + + let mut test = ParaToParaTest::new(test_args); + + let sender_balance_before = test.sender.balance; + let receiver_balance_before = test.receiver.balance; + + let sender_as_seen_by_relay = Rococo::child_location_of(PenpalA::para_id()); + let sov_of_sender_on_relay = Rococo::sovereign_account_id_of(sender_as_seen_by_relay); + + // fund the PenpalA's SA on Rococo with the native tokens held in reserve + Rococo::fund_accounts(vec![(sov_of_sender_on_relay.into(), amount_to_send * 2)]); + + test.set_assertion::(para_to_para_sender_assertions); + test.set_assertion::(para_to_para_relay_hop_assertions); + test.set_assertion::(para_to_para_receiver_assertions); + test.set_dispatchable::(para_to_para_remote_reserve_transfer_native_assets); + test.assert(); + + let sender_balance_after = test.sender.balance; + let receiver_balance_after = test.receiver.balance; + + let delivery_fees = PenpalA::execute_with(|| { + xcm_helpers::transfer_assets_delivery_fees::< + ::XcmSender, + >(test.args.assets.clone(), 0, test.args.weight_limit, test.args.beneficiary, test.args.dest) + }); + + // Sender's balance is reduced + assert_eq!(sender_balance_before - amount_to_send - delivery_fees, sender_balance_after); + // Receiver's balance is increased + assert!(receiver_balance_after > receiver_balance_before); +} + fn relay_to_para_limited_reserve_transfer_assets(t: RelayToParaTest) -> DispatchResult { ::XcmPallet::limited_reserve_transfer_assets( t.signed_origin, diff --git a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs index 74d9a0b071d8..8cdc120b5edf 100644 --- a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs @@ -155,34 +155,29 @@ match_types! { }; } -pub type Barrier = TrailingSetTopicAsId< - DenyThenTry< - DenyReserveTransferToRelayChain, +pub type Barrier = TrailingSetTopicAsId<( + TakeWeightCredit, + // Expected responses are OK. + AllowKnownQueryResponses, + // Allow XCMs with some computed origins to pass through. + WithComputedOrigin< ( - TakeWeightCredit, - // Expected responses are OK. - AllowKnownQueryResponses, - // Allow XCMs with some computed origins to pass through. - WithComputedOrigin< - ( - // If the message is one that immediately attempts to pay for execution, then - // allow it. - AllowTopLevelPaidExecutionFrom, - // System Assets parachain, parent and its exec plurality get free - // execution - AllowExplicitUnpaidExecutionFrom<( - CommonGoodAssetsParachain, - ParentOrParentsExecutivePlurality, - )>, - // Subscriptions for version tracking are OK. - AllowSubscriptionsFrom, - ), - UniversalLocation, - ConstU32<8>, - >, + // If the message is one that immediately attempts to pay for execution, then + // allow it. + AllowTopLevelPaidExecutionFrom, + // System Assets parachain, parent and its exec plurality get free + // execution + AllowExplicitUnpaidExecutionFrom<( + CommonGoodAssetsParachain, + ParentOrParentsExecutivePlurality, + )>, + // Subscriptions for version tracking are OK. + AllowSubscriptionsFrom, ), + UniversalLocation, + ConstU32<8>, >, ->; +)>; /// Type alias to conveniently refer to `frame_system`'s `Config::AccountId`. pub type AccountIdOf = ::AccountId; @@ -259,7 +254,7 @@ impl xcm_executor::Config for XcmConfig { type OriginConverter = XcmOriginToTransactDispatchOrigin; type IsReserve = Reserves; // no teleport trust established with other chains - type IsTeleporter = NativeAsset; + type IsTeleporter = (); type UniversalLocation = UniversalLocation; type Barrier = Barrier; type Weigher = FixedWeightBounds; @@ -298,7 +293,7 @@ impl pallet_xcm::Config for Runtime { type SendXcmOrigin = EnsureXcmOrigin; type XcmRouter = XcmRouter; type ExecuteXcmOrigin = EnsureXcmOrigin; - type XcmExecuteFilter = Nothing; + type XcmExecuteFilter = Everything; // ^ Disable dispatchable execute on the XCM pallet. // Needs to be `Everything` for local testing. type XcmExecutor = XcmExecutor; diff --git a/polkadot/xcm/xcm-executor/Cargo.toml b/polkadot/xcm/xcm-executor/Cargo.toml index 489905db3a01..e937cd80ac1c 100644 --- a/polkadot/xcm/xcm-executor/Cargo.toml +++ b/polkadot/xcm/xcm-executor/Cargo.toml @@ -4,7 +4,7 @@ description = "An abstract and configurable XCM message executor." authors.workspace = true edition.workspace = true license.workspace = true -version = "5.0.0" +version = "5.0.1" [dependencies] impl-trait-for-tuples = "0.2.2" diff --git a/polkadot/xcm/xcm-executor/src/lib.rs b/polkadot/xcm/xcm-executor/src/lib.rs index ac256ea14899..368c6f1855e4 100644 --- a/polkadot/xcm/xcm-executor/src/lib.rs +++ b/polkadot/xcm/xcm-executor/src/lib.rs @@ -630,6 +630,20 @@ impl XcmExecutor { Ok(()) }, DepositReserveAsset { assets, dest, xcm } => { + // we need to do this take/put cycle to solve wildcards and get exact assets to be + // weighed + let to_weigh = self.holding.saturating_take(assets.clone()); + self.holding.subsume_assets(to_weigh.clone()); + + let mut message_to_weigh = + vec![ReserveAssetDeposited(to_weigh.into()), ClearOrigin]; + message_to_weigh.extend(xcm.0.clone().into_iter()); + let (_, fee) = + validate_send::(dest.clone(), Xcm(message_to_weigh))?; + // set aside fee to be charged by XcmSender + let parked_fee = self.holding.saturating_take(fee.into()); + + // now take assets to deposit (excluding parked_fee) let deposited = self.holding.saturating_take(assets); for asset in deposited.assets_iter() { Config::AssetTransactor::deposit_asset(&asset, &dest, Some(&self.context))?; @@ -639,6 +653,9 @@ impl XcmExecutor { let assets = Self::reanchored(deposited, &dest, None); let mut message = vec![ReserveAssetDeposited(assets), ClearOrigin]; message.extend(xcm.0.into_iter()); + + // put back parked_fee in holding register to be charged by XcmSender + self.holding.subsume_assets(parked_fee); self.send(dest, Xcm(message), FeeReason::DepositReserveAsset)?; Ok(()) }, From e043aac295d0ac6c38bbc076c82a74521f55b494 Mon Sep 17 00:00:00 2001 From: Branislav Kontur Date: Tue, 12 Dec 2023 21:01:36 +0100 Subject: [PATCH 2/5] [ci] Add `-D warnings` for `cargo-check-each-crate` job to fail on warnings (#2670) This PR turns on `-D warnings` for `cargo-check-each-crate job` job to fail on warnings e.g. like this: https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673130 Before this PR, there was a warning and `cargo-check-each-crate` job did not fail: https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4641444 ``` warning: unused import: `ToTokens` --> substrate/primitives/api/proc-macro/src/utils.rs:22:34 | 22 | use quote::{format_ident, quote, ToTokens}; | ^^^^^^^^ | = note: `#[warn(unused_imports)]` on by default warning: `sp-api-proc-macro` (lib) generated 1 warning (run `cargo fix --lib -p sp-api-proc-macro` to apply 1 suggestion) ``` Fixes on the way: https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4641444 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673265 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673410 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673681 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673836 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4673941 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4674256 https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4679328 - [ ] why does this check triggers only `cargo check --locked`? `--all-features` or `--all-targets` are not needed? Or aren't they avoided intentionally? --------- Co-authored-by: command-bot <> --- .gitlab/pipeline/test.yml | 3 ++- substrate/client/network/src/behaviour.rs | 2 +- substrate/client/network/src/protocol/message.rs | 4 ---- .../client/network/src/protocol/notifications/upgrade.rs | 8 +++++--- substrate/client/offchain/src/api.rs | 1 - substrate/client/transaction-pool/src/graph/mod.rs | 5 +---- substrate/frame/contracts/proc-macro/src/lib.rs | 2 +- substrate/primitives/api/proc-macro/src/utils.rs | 3 ++- substrate/primitives/state-machine/src/lib.rs | 1 - 9 files changed, 12 insertions(+), 17 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 4ed3ec19c48a..cf29776f01aa 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -439,7 +439,8 @@ cargo-check-each-crate: - .run-immediately # - .collect-artifacts variables: - # $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all + RUSTFLAGS: "-D warnings" + # $CI_JOB_NAME is set manually so that cache could be shared for all jobs # "cargo-check-each-crate I/N" jobs CI_JOB_NAME: cargo-check-each-crate timeout: 2h diff --git a/substrate/client/network/src/behaviour.rs b/substrate/client/network/src/behaviour.rs index 0aa724818e02..b486a8eab79f 100644 --- a/substrate/client/network/src/behaviour.rs +++ b/substrate/client/network/src/behaviour.rs @@ -39,7 +39,7 @@ use sc_network_common::role::{ObservedRole, Roles}; use sp_runtime::traits::Block as BlockT; use std::{collections::HashSet, sync::Arc, time::Duration}; -pub use crate::request_responses::{InboundFailure, OutboundFailure, RequestId, ResponseFailure}; +pub use crate::request_responses::{InboundFailure, OutboundFailure, ResponseFailure}; /// General behaviour of the network. Combines all protocols together. #[derive(NetworkBehaviour)] diff --git a/substrate/client/network/src/protocol/message.rs b/substrate/client/network/src/protocol/message.rs index 66dca2975375..d4a05c041ef7 100644 --- a/substrate/client/network/src/protocol/message.rs +++ b/substrate/client/network/src/protocol/message.rs @@ -19,10 +19,6 @@ //! Network packet message types. These get serialized and put into the lower level protocol //! payload. -pub use self::generic::{ - RemoteCallRequest, RemoteChangesRequest, RemoteChangesResponse, RemoteHeaderRequest, - RemoteHeaderResponse, RemoteReadChildRequest, RemoteReadRequest, -}; use codec::{Decode, Encode}; use sc_client_api::StorageProof; use sc_network_common::message::RequestId; diff --git a/substrate/client/network/src/protocol/notifications/upgrade.rs b/substrate/client/network/src/protocol/notifications/upgrade.rs index 70c6023623f5..8fd837f949d8 100644 --- a/substrate/client/network/src/protocol/notifications/upgrade.rs +++ b/substrate/client/network/src/protocol/notifications/upgrade.rs @@ -16,12 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +#[cfg(test)] +pub(crate) use self::notifications::{ + NotificationsInOpen, NotificationsInSubstreamHandshake, NotificationsOutOpen, +}; pub use self::{ collec::UpgradeCollec, notifications::{ - NotificationsHandshakeError, NotificationsIn, NotificationsInOpen, - NotificationsInSubstream, NotificationsInSubstreamHandshake, NotificationsOut, - NotificationsOutError, NotificationsOutOpen, NotificationsOutSubstream, + NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, }, }; diff --git a/substrate/client/offchain/src/api.rs b/substrate/client/offchain/src/api.rs index c7df5784d329..7d33f8dc9225 100644 --- a/substrate/client/offchain/src/api.rs +++ b/substrate/client/offchain/src/api.rs @@ -30,7 +30,6 @@ use sp_core::{ }, OpaquePeerId, }; -pub use sp_offchain::STORAGE_PREFIX; mod http; diff --git a/substrate/client/transaction-pool/src/graph/mod.rs b/substrate/client/transaction-pool/src/graph/mod.rs index 5afdddb7402d..484a6d6cf9f0 100644 --- a/substrate/client/transaction-pool/src/graph/mod.rs +++ b/substrate/client/transaction-pool/src/graph/mod.rs @@ -39,9 +39,6 @@ pub mod watcher; pub use self::{ base_pool::Transaction, - pool::{ - BlockHash, ChainApi, EventStream, ExtrinsicFor, ExtrinsicHash, NumberFor, Options, Pool, - TransactionFor, - }, + pool::{BlockHash, ChainApi, ExtrinsicFor, ExtrinsicHash, NumberFor, Options, Pool}, }; pub use validated_pool::{IsValidator, ValidatedTransaction}; diff --git a/substrate/frame/contracts/proc-macro/src/lib.rs b/substrate/frame/contracts/proc-macro/src/lib.rs index ad9cd2dadecf..2d23a997edd4 100644 --- a/substrate/frame/contracts/proc-macro/src/lib.rs +++ b/substrate/frame/contracts/proc-macro/src/lib.rs @@ -74,7 +74,7 @@ fn derive_debug(input: TokenStream, fmt: impl Fn(&Ident) -> TokenStream2) -> Tok #[cfg(not(feature = "full"))] let fields = { drop(fmt); - drop(data); + let _ = data; TokenStream2::new() }; diff --git a/substrate/primitives/api/proc-macro/src/utils.rs b/substrate/primitives/api/proc-macro/src/utils.rs index e261b162b5aa..8c87f0d154ae 100644 --- a/substrate/primitives/api/proc-macro/src/utils.rs +++ b/substrate/primitives/api/proc-macro/src/utils.rs @@ -19,7 +19,7 @@ use crate::common::API_VERSION_ATTRIBUTE; use inflector::Inflector; use proc_macro2::{Span, TokenStream}; use proc_macro_crate::{crate_name, FoundCrate}; -use quote::{format_ident, quote, ToTokens}; +use quote::{format_ident, quote}; use syn::{ parse_quote, spanned::Spanned, token::And, Attribute, Error, FnArg, GenericArgument, Ident, ImplItem, ItemImpl, Pat, Path, PathArguments, Result, ReturnType, Signature, Type, TypePath, @@ -261,6 +261,7 @@ pub fn versioned_trait_name(trait_ident: &Ident, version: u64) -> Ident { /// Extract the documentation from the provided attributes. #[cfg(feature = "frame-metadata")] pub fn get_doc_literals(attrs: &[syn::Attribute]) -> Vec { + use quote::ToTokens; attrs .iter() .filter_map(|attr| { diff --git a/substrate/primitives/state-machine/src/lib.rs b/substrate/primitives/state-machine/src/lib.rs index 0e2b9bfdfffc..5cc796054fc2 100644 --- a/substrate/primitives/state-machine/src/lib.rs +++ b/substrate/primitives/state-machine/src/lib.rs @@ -142,7 +142,6 @@ pub use crate::{ mod std_reexport { pub use crate::{ basic::BasicExternalities, - error::{Error, ExecutionError}, in_memory_backend::new_in_mem, read_only::{InspectState, ReadOnlyExternalities}, testing::TestExternalities, From 9c951fe56cf85836b7df032f52b4634dad8de0f6 Mon Sep 17 00:00:00 2001 From: Francisco Aguirre Date: Sat, 3 Feb 2024 09:46:50 +0100 Subject: [PATCH 3/5] Fix clippy issues --- .../runtimes/testing/penpal/src/xcm_config.rs | 11 +++++------ polkadot/xcm/xcm-executor/src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs index 8cdc120b5edf..322347143077 100644 --- a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs @@ -47,12 +47,11 @@ use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, AsPrefixedGeneralIndex, - ConvertedConcreteId, CurrencyAdapter, DenyReserveTransferToRelayChain, DenyThenTry, - EnsureXcmOrigin, FixedWeightBounds, FungiblesAdapter, IsConcrete, LocalMint, NativeAsset, - ParentAsSuperuser, ParentIsPreset, RelayChainAsNative, SiblingParachainAsNative, - SiblingParachainConvertsVia, SignedAccountId32AsNative, SignedToAccountId32, - SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, - WithComputedOrigin, WithUniqueTopic, + ConvertedConcreteId, CurrencyAdapter, EnsureXcmOrigin, FixedWeightBounds, + FungiblesAdapter, IsConcrete, LocalMint, NativeAsset, ParentAsSuperuser, + ParentIsPreset, RelayChainAsNative, SiblingParachainAsNative, SiblingParachainConvertsVia, + SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, + TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, WithComputedOrigin, WithUniqueTopic, }; use xcm_executor::{traits::JustTry, XcmExecutor}; diff --git a/polkadot/xcm/xcm-executor/src/lib.rs b/polkadot/xcm/xcm-executor/src/lib.rs index 368c6f1855e4..e1f4fc192ad1 100644 --- a/polkadot/xcm/xcm-executor/src/lib.rs +++ b/polkadot/xcm/xcm-executor/src/lib.rs @@ -639,7 +639,7 @@ impl XcmExecutor { vec![ReserveAssetDeposited(to_weigh.into()), ClearOrigin]; message_to_weigh.extend(xcm.0.clone().into_iter()); let (_, fee) = - validate_send::(dest.clone(), Xcm(message_to_weigh))?; + validate_send::(dest, Xcm(message_to_weigh))?; // set aside fee to be charged by XcmSender let parked_fee = self.holding.saturating_take(fee.into()); From 6a21fe66521fac5964b9f70ba3b06a654c4c33d4 Mon Sep 17 00:00:00 2001 From: Francisco Aguirre Date: Sat, 3 Feb 2024 10:03:45 +0100 Subject: [PATCH 4/5] Remove warnings as errors --- .gitlab/pipeline/build.yml | 6 +++--- .gitlab/pipeline/test.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 5c13045706c4..ff002ab5ab6f 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -14,7 +14,7 @@ build-linux-stable: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" # Ensure we run the UI tests. RUN_UI_TESTS: 1 script: @@ -169,7 +169,7 @@ build-linux-stable-cumulus: variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - time cargo build --release --locked -p polkadot-parachain-bin --bin polkadot-parachain @@ -189,7 +189,7 @@ build-test-parachain: variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - time cargo build --release --locked -p cumulus-test-service --bin test-parachain diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index cf29776f01aa..f7deb3d9914c 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -16,7 +16,7 @@ test-linux-stable: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" parallel: 3 script: # Build all but only execute 'runtime' tests. @@ -66,7 +66,7 @@ test-linux-stable-runtime-benchmarks: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet @@ -108,7 +108,7 @@ test-linux-stable-additional-tests: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: # tests were moved to test-linux-stable # the jobs should be removed @@ -128,7 +128,7 @@ test-linux-stable-slow: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: # tests were moved to test-linux-stable # the jobs should be removed @@ -148,7 +148,7 @@ test-linux-stable-slow: # RUST_TOOLCHAIN: stable # # Enable debug assertions since we are running optimized builds for testing # # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# RUSTFLAGS: "-Cdebug-assertions=y" # script: # - mkdir -p artifacts # - time cargo test --workspace @@ -169,7 +169,7 @@ test-doc: variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=y" script: - time cargo test --doc --workspace @@ -207,7 +207,7 @@ test-node-metrics: RUST_TOOLCHAIN: stable # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + RUSTFLAGS: "-Cdebug-assertions=ys" script: # Build the required workers. - cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked From a455c80f3d604f05ee3ce76569a2d1d721a3b186 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Wed, 31 Jan 2024 00:17:44 +0100 Subject: [PATCH 5/5] [ci] Update rust in ci image (1.75 and 2024-01-22) (#3016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cc https://github.com/paritytech/ci_cd/issues/926 --------- Signed-off-by: Oliver Tale-Yazdi Co-authored-by: command-bot <> Co-authored-by: Oliver Tale-Yazdi Co-authored-by: Bastian Köcher Co-authored-by: Liam Aharon Co-authored-by: Dónal Murray Co-authored-by: Vladimir Istyufeev --- .github/workflows/fmt-check.yml | 2 +- .gitlab-ci.yml | 3 +- .../parachain/pallets/system/src/lib.rs | 685 ++++++++++++++++++ .../development_environment_advice.rs | 113 +++ .../subsystem-bench/src/availability/mod.rs | 660 +++++++++++++++++ .../node/subsystem-bench/src/core/keyring.rs | 46 ++ .../node/subsystem-bench/src/core/mock/mod.rs | 78 ++ substrate/frame/alliance/src/mock.rs | 2 +- substrate/frame/contracts/fixtures/build.rs | 350 +++++++++ .../procedural/src/pallet/expand/call.rs | 3 + .../deprecated_where_block.stderr | 66 ++ ...lt_config_with_no_default_in_system.stderr | 2 +- .../error_does_not_derive_pallet_error.rs | 4 +- .../error_does_not_derive_pallet_error.stderr | 6 +- .../tests/pallet_ui/event_field_not_member.rs | 4 +- .../pallet_ui/event_field_not_member.stderr | 12 +- .../pallet_ui/pass/error_nested_types.rs | 18 +- ...ent_type_bound_system_config_assoc_type.rs | 1 + .../storage_result_query_missing_generics.rs | 4 +- ...orage_result_query_missing_generics.stderr | 10 +- .../ui/impl_two_traits_with_same_name.rs | 1 + .../ui/impl_two_traits_with_same_name.stderr | 8 +- 22 files changed, 2041 insertions(+), 37 deletions(-) create mode 100644 bridges/snowbridge/parachain/pallets/system/src/lib.rs create mode 100644 docs/sdk/src/reference_docs/development_environment_advice.rs create mode 100644 polkadot/node/subsystem-bench/src/availability/mod.rs create mode 100644 polkadot/node/subsystem-bench/src/core/keyring.rs create mode 100644 polkadot/node/subsystem-bench/src/core/mock/mod.rs create mode 100644 substrate/frame/contracts/fixtures/build.rs diff --git a/.github/workflows/fmt-check.yml b/.github/workflows/fmt-check.yml index e4d39acabfd7..efcf278c46e8 100644 --- a/.github/workflows/fmt-check.yml +++ b/.github/workflows/fmt-check.yml @@ -15,7 +15,7 @@ jobs: os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} container: - image: paritytech/ci-unified:bullseye-1.73.0-2023-11-01-v20231025 + image: docker.io/paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240109 steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1dc483004f24..5e9974da9941 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,8 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE] + # CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE] + CI_IMAGE: "docker.io/paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240109" # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master" diff --git a/bridges/snowbridge/parachain/pallets/system/src/lib.rs b/bridges/snowbridge/parachain/pallets/system/src/lib.rs new file mode 100644 index 000000000000..b7f38fb753d3 --- /dev/null +++ b/bridges/snowbridge/parachain/pallets/system/src/lib.rs @@ -0,0 +1,685 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023 Snowfork +//! Governance API for controlling the Ethereum side of the bridge +//! +//! # Extrinsics +//! +//! ## Agents +//! +//! Agents are smart contracts on Ethereum that act as proxies for consensus systems on Polkadot +//! networks. +//! +//! * [`Call::create_agent`]: Create agent for a sibling parachain +//! * [`Call::transfer_native_from_agent`]: Withdraw ether from an agent +//! +//! The `create_agent` extrinsic should be called via an XCM `Transact` instruction from the sibling +//! parachain. +//! +//! ## Channels +//! +//! Each sibling parachain has its own dedicated messaging channel for sending and receiving +//! messages. As a prerequisite to creating a channel, the sibling should have already created +//! an agent using the `create_agent` extrinsic. +//! +//! * [`Call::create_channel`]: Create channel for a sibling +//! * [`Call::update_channel`]: Update a channel for a sibling +//! +//! ## Governance +//! +//! Only Polkadot governance itself can call these extrinsics. Delivery fees are waived. +//! +//! * [`Call::upgrade`]`: Upgrade the gateway contract +//! * [`Call::set_operating_mode`]: Update the operating mode of the gateway contract +//! * [`Call::force_update_channel`]: Allow root to update a channel for a sibling +//! * [`Call::force_transfer_native_from_agent`]: Allow root to withdraw ether from an agent +//! +//! Typically, Polkadot governance will use the `force_transfer_native_from_agent` and +//! `force_update_channel` and extrinsics to manage agents and channels for system parachains. +#![cfg_attr(not(feature = "std"), no_std)] + +pub use pallet::*; + +#[cfg(test)] +mod mock; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; +pub mod migration; + +pub mod api; +pub mod weights; +pub use weights::*; + +use frame_support::{ + pallet_prelude::*, + traits::{ + fungible::{Inspect, Mutate}, + tokens::Preservation, + Contains, EnsureOrigin, + }, +}; +use frame_system::pallet_prelude::*; +use snowbridge_core::{ + meth, + outbound::{Command, Initializer, Message, OperatingMode, SendError, SendMessage}, + sibling_sovereign_account, AgentId, Channel, ChannelId, ParaId, + PricingParameters as PricingParametersRecord, PRIMARY_GOVERNANCE_CHANNEL, + SECONDARY_GOVERNANCE_CHANNEL, +}; +use sp_core::{RuntimeDebug, H160, H256}; +use sp_io::hashing::blake2_256; +use sp_runtime::{traits::BadOrigin, DispatchError, SaturatedConversion}; +use sp_std::prelude::*; +use xcm::prelude::*; +use xcm_executor::traits::ConvertLocation; + +#[cfg(feature = "runtime-benchmarks")] +use frame_support::traits::OriginTrait; + +pub type BalanceOf = + <::Token as Inspect<::AccountId>>::Balance; +pub type AccountIdOf = ::AccountId; +pub type PricingParametersOf = PricingParametersRecord>; + +/// Ensure origin location is a sibling +fn ensure_sibling(location: &Location) -> Result<(ParaId, H256), DispatchError> +where + T: Config, +{ + match location.unpack() { + (1, [Parachain(para_id)]) => { + let agent_id = agent_id_of::(location)?; + Ok(((*para_id).into(), agent_id)) + }, + _ => Err(BadOrigin.into()), + } +} + +/// Hash the location to produce an agent id +fn agent_id_of(location: &Location) -> Result { + T::AgentIdOf::convert_location(location).ok_or(Error::::LocationConversionFailed.into()) +} + +#[cfg(feature = "runtime-benchmarks")] +pub trait BenchmarkHelper +where + O: OriginTrait, +{ + fn make_xcm_origin(location: Location) -> O; +} + +/// Whether a fee should be withdrawn to an account for sending an outbound message +#[derive(Clone, PartialEq, RuntimeDebug)] +pub enum PaysFee +where + T: Config, +{ + /// Fully charge includes (local + remote fee) + Yes(AccountIdOf), + /// Partially charge includes local fee only + Partial(AccountIdOf), + /// No charge + No, +} + +#[frame_support::pallet] +pub mod pallet { + use snowbridge_core::StaticLookup; + use sp_core::U256; + + use super::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + + /// Send messages to Ethereum + type OutboundQueue: SendMessage>; + + /// Origin check for XCM locations that can create agents + type SiblingOrigin: EnsureOrigin; + + /// Converts Location to AgentId + type AgentIdOf: ConvertLocation; + + /// Token reserved for control operations + type Token: Mutate; + + /// TreasuryAccount to collect fees + #[pallet::constant] + type TreasuryAccount: Get; + + /// Number of decimal places of local currency + type DefaultPricingParameters: Get>; + + /// Cost of delivering a message from Ethereum + type InboundDeliveryCost: Get>; + + type WeightInfo: WeightInfo; + + #[cfg(feature = "runtime-benchmarks")] + type Helper: BenchmarkHelper; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// An Upgrade message was sent to the Gateway + Upgrade { + impl_address: H160, + impl_code_hash: H256, + initializer_params_hash: Option, + }, + /// An CreateAgent message was sent to the Gateway + CreateAgent { + location: Box, + agent_id: AgentId, + }, + /// An CreateChannel message was sent to the Gateway + CreateChannel { + channel_id: ChannelId, + agent_id: AgentId, + }, + /// An UpdateChannel message was sent to the Gateway + UpdateChannel { + channel_id: ChannelId, + mode: OperatingMode, + }, + /// An SetOperatingMode message was sent to the Gateway + SetOperatingMode { + mode: OperatingMode, + }, + /// An TransferNativeFromAgent message was sent to the Gateway + TransferNativeFromAgent { + agent_id: AgentId, + recipient: H160, + amount: u128, + }, + /// A SetTokenTransferFees message was sent to the Gateway + SetTokenTransferFees { + create_asset_xcm: u128, + transfer_asset_xcm: u128, + register_token: U256, + }, + PricingParametersChanged { + params: PricingParametersOf, + }, + } + + #[pallet::error] + pub enum Error { + LocationConversionFailed, + AgentAlreadyCreated, + NoAgent, + ChannelAlreadyCreated, + NoChannel, + UnsupportedLocationVersion, + InvalidLocation, + Send(SendError), + InvalidTokenTransferFees, + InvalidPricingParameters, + InvalidUpgradeParameters, + } + + /// The set of registered agents + #[pallet::storage] + #[pallet::getter(fn agents)] + pub type Agents = StorageMap<_, Twox64Concat, AgentId, (), OptionQuery>; + + /// The set of registered channels + #[pallet::storage] + #[pallet::getter(fn channels)] + pub type Channels = StorageMap<_, Twox64Concat, ChannelId, Channel, OptionQuery>; + + #[pallet::storage] + #[pallet::getter(fn parameters)] + pub type PricingParameters = + StorageValue<_, PricingParametersOf, ValueQuery, T::DefaultPricingParameters>; + + #[pallet::genesis_config] + #[derive(frame_support::DefaultNoBound)] + pub struct GenesisConfig { + // Own parachain id + pub para_id: ParaId, + // AssetHub's parachain id + pub asset_hub_para_id: ParaId, + #[serde(skip)] + pub _config: PhantomData, + } + + #[pallet::genesis_build] + impl BuildGenesisConfig for GenesisConfig { + fn build(&self) { + Pallet::::initialize(self.para_id, self.asset_hub_para_id).expect("infallible; qed"); + } + } + + #[pallet::call] + impl Pallet { + /// Sends command to the Gateway contract to upgrade itself with a new implementation + /// contract + /// + /// Fee required: No + /// + /// - `origin`: Must be `Root`. + /// - `impl_address`: The address of the implementation contract. + /// - `impl_code_hash`: The codehash of the implementation contract. + /// - `initializer`: Optionally call an initializer on the implementation contract. + #[pallet::call_index(0)] + #[pallet::weight((T::WeightInfo::upgrade(), DispatchClass::Operational))] + pub fn upgrade( + origin: OriginFor, + impl_address: H160, + impl_code_hash: H256, + initializer: Option, + ) -> DispatchResult { + ensure_root(origin)?; + + ensure!( + !impl_address.eq(&H160::zero()) && !impl_code_hash.eq(&H256::zero()), + Error::::InvalidUpgradeParameters + ); + + let initializer_params_hash: Option = + initializer.as_ref().map(|i| H256::from(blake2_256(i.params.as_ref()))); + let command = Command::Upgrade { impl_address, impl_code_hash, initializer }; + Self::send(PRIMARY_GOVERNANCE_CHANNEL, command, PaysFee::::No)?; + + Self::deposit_event(Event::::Upgrade { + impl_address, + impl_code_hash, + initializer_params_hash, + }); + Ok(()) + } + + /// Sends a message to the Gateway contract to change its operating mode + /// + /// Fee required: No + /// + /// - `origin`: Must be `Location` + #[pallet::call_index(1)] + #[pallet::weight((T::WeightInfo::set_operating_mode(), DispatchClass::Operational))] + pub fn set_operating_mode(origin: OriginFor, mode: OperatingMode) -> DispatchResult { + ensure_root(origin)?; + + let command = Command::SetOperatingMode { mode }; + Self::send(PRIMARY_GOVERNANCE_CHANNEL, command, PaysFee::::No)?; + + Self::deposit_event(Event::::SetOperatingMode { mode }); + Ok(()) + } + + /// Set pricing parameters on both sides of the bridge + /// + /// Fee required: No + /// + /// - `origin`: Must be root + #[pallet::call_index(2)] + #[pallet::weight((T::WeightInfo::set_pricing_parameters(), DispatchClass::Operational))] + pub fn set_pricing_parameters( + origin: OriginFor, + params: PricingParametersOf, + ) -> DispatchResult { + ensure_root(origin)?; + params.validate().map_err(|_| Error::::InvalidPricingParameters)?; + PricingParameters::::put(params.clone()); + + let command = Command::SetPricingParameters { + exchange_rate: params.exchange_rate.into(), + delivery_cost: T::InboundDeliveryCost::get().saturated_into::(), + }; + Self::send(PRIMARY_GOVERNANCE_CHANNEL, command, PaysFee::::No)?; + + Self::deposit_event(Event::PricingParametersChanged { params }); + Ok(()) + } + + /// Sends a command to the Gateway contract to instantiate a new agent contract representing + /// `origin`. + /// + /// Fee required: Yes + /// + /// - `origin`: Must be `Location` of a sibling parachain + #[pallet::call_index(3)] + #[pallet::weight(T::WeightInfo::create_agent())] + pub fn create_agent(origin: OriginFor) -> DispatchResult { + let origin_location: Location = T::SiblingOrigin::ensure_origin(origin)?; + + // Ensure that origin location is some consensus system on a sibling parachain + let (para_id, agent_id) = ensure_sibling::(&origin_location)?; + + // Record the agent id or fail if it has already been created + ensure!(!Agents::::contains_key(agent_id), Error::::AgentAlreadyCreated); + Agents::::insert(agent_id, ()); + + let command = Command::CreateAgent { agent_id }; + let pays_fee = PaysFee::::Yes(sibling_sovereign_account::(para_id)); + Self::send(SECONDARY_GOVERNANCE_CHANNEL, command, pays_fee)?; + + Self::deposit_event(Event::::CreateAgent { + location: Box::new(origin_location), + agent_id, + }); + Ok(()) + } + + /// Sends a message to the Gateway contract to create a new channel representing `origin` + /// + /// Fee required: Yes + /// + /// This extrinsic is permissionless, so a fee is charged to prevent spamming and pay + /// for execution costs on the remote side. + /// + /// The message is sent over the bridge on BridgeHub's own channel to the Gateway. + /// + /// - `origin`: Must be `Location` + /// - `mode`: Initial operating mode of the channel + #[pallet::call_index(4)] + #[pallet::weight(T::WeightInfo::create_channel())] + pub fn create_channel(origin: OriginFor, mode: OperatingMode) -> DispatchResult { + let origin_location: Location = T::SiblingOrigin::ensure_origin(origin)?; + + // Ensure that origin location is a sibling parachain + let (para_id, agent_id) = ensure_sibling::(&origin_location)?; + + let channel_id: ChannelId = para_id.into(); + + ensure!(Agents::::contains_key(agent_id), Error::::NoAgent); + ensure!(!Channels::::contains_key(channel_id), Error::::ChannelAlreadyCreated); + + let channel = Channel { agent_id, para_id }; + Channels::::insert(channel_id, channel); + + let command = Command::CreateChannel { channel_id, agent_id, mode }; + let pays_fee = PaysFee::::Yes(sibling_sovereign_account::(para_id)); + Self::send(SECONDARY_GOVERNANCE_CHANNEL, command, pays_fee)?; + + Self::deposit_event(Event::::CreateChannel { channel_id, agent_id }); + Ok(()) + } + + /// Sends a message to the Gateway contract to update a channel configuration + /// + /// The origin must already have a channel initialized, as this message is sent over it. + /// + /// A partial fee will be charged for local processing only. + /// + /// - `origin`: Must be `Location` + /// - `mode`: Initial operating mode of the channel + #[pallet::call_index(5)] + #[pallet::weight(T::WeightInfo::update_channel())] + pub fn update_channel(origin: OriginFor, mode: OperatingMode) -> DispatchResult { + let origin_location: Location = T::SiblingOrigin::ensure_origin(origin)?; + + // Ensure that origin location is a sibling parachain + let (para_id, _) = ensure_sibling::(&origin_location)?; + + let channel_id: ChannelId = para_id.into(); + + ensure!(Channels::::contains_key(channel_id), Error::::NoChannel); + + let command = Command::UpdateChannel { channel_id, mode }; + let pays_fee = PaysFee::::Partial(sibling_sovereign_account::(para_id)); + + // Parachains send the update message on their own channel + Self::send(channel_id, command, pays_fee)?; + + Self::deposit_event(Event::::UpdateChannel { channel_id, mode }); + Ok(()) + } + + /// Sends a message to the Gateway contract to update an arbitrary channel + /// + /// Fee required: No + /// + /// - `origin`: Must be root + /// - `channel_id`: ID of channel + /// - `mode`: Initial operating mode of the channel + /// - `outbound_fee`: Fee charged to users for sending outbound messages to Polkadot + #[pallet::call_index(6)] + #[pallet::weight(T::WeightInfo::force_update_channel())] + pub fn force_update_channel( + origin: OriginFor, + channel_id: ChannelId, + mode: OperatingMode, + ) -> DispatchResult { + ensure_root(origin)?; + + ensure!(Channels::::contains_key(channel_id), Error::::NoChannel); + + let command = Command::UpdateChannel { channel_id, mode }; + Self::send(PRIMARY_GOVERNANCE_CHANNEL, command, PaysFee::::No)?; + + Self::deposit_event(Event::::UpdateChannel { channel_id, mode }); + Ok(()) + } + + /// Sends a message to the Gateway contract to transfer ether from an agent to `recipient`. + /// + /// A partial fee will be charged for local processing only. + /// + /// - `origin`: Must be `Location` + #[pallet::call_index(7)] + #[pallet::weight(T::WeightInfo::transfer_native_from_agent())] + pub fn transfer_native_from_agent( + origin: OriginFor, + recipient: H160, + amount: u128, + ) -> DispatchResult { + let origin_location: Location = T::SiblingOrigin::ensure_origin(origin)?; + + // Ensure that origin location is some consensus system on a sibling parachain + let (para_id, agent_id) = ensure_sibling::(&origin_location)?; + + // Since the origin is also the owner of the channel, they only need to pay + // the local processing fee. + let pays_fee = PaysFee::::Partial(sibling_sovereign_account::(para_id)); + + Self::do_transfer_native_from_agent( + agent_id, + para_id.into(), + recipient, + amount, + pays_fee, + ) + } + + /// Sends a message to the Gateway contract to transfer ether from an agent to `recipient`. + /// + /// Privileged. Can only be called by root. + /// + /// Fee required: No + /// + /// - `origin`: Must be root + /// - `location`: Location used to resolve the agent + /// - `recipient`: Recipient of funds + /// - `amount`: Amount to transfer + #[pallet::call_index(8)] + #[pallet::weight(T::WeightInfo::force_transfer_native_from_agent())] + pub fn force_transfer_native_from_agent( + origin: OriginFor, + location: Box, + recipient: H160, + amount: u128, + ) -> DispatchResult { + ensure_root(origin)?; + + // Ensure that location is some consensus system on a sibling parachain + let location: Location = + (*location).try_into().map_err(|_| Error::::UnsupportedLocationVersion)?; + let (_, agent_id) = + ensure_sibling::(&location).map_err(|_| Error::::InvalidLocation)?; + + let pays_fee = PaysFee::::No; + + Self::do_transfer_native_from_agent( + agent_id, + PRIMARY_GOVERNANCE_CHANNEL, + recipient, + amount, + pays_fee, + ) + } + + /// Sends a message to the Gateway contract to update fee related parameters for + /// token transfers. + /// + /// Privileged. Can only be called by root. + /// + /// Fee required: No + /// + /// - `origin`: Must be root + /// - `create_asset_xcm`: The XCM execution cost for creating a new asset class on AssetHub, + /// in DOT + /// - `transfer_asset_xcm`: The XCM execution cost for performing a reserve transfer on + /// AssetHub, in DOT + /// - `register_token`: The Ether fee for registering a new token, to discourage spamming + #[pallet::call_index(9)] + #[pallet::weight((T::WeightInfo::set_token_transfer_fees(), DispatchClass::Operational))] + pub fn set_token_transfer_fees( + origin: OriginFor, + create_asset_xcm: u128, + transfer_asset_xcm: u128, + register_token: U256, + ) -> DispatchResult { + ensure_root(origin)?; + + // Basic validation of new costs. Particularly for token registration, we want to ensure + // its relatively expensive to discourage spamming. Like at least 100 USD. + ensure!( + create_asset_xcm > 0 && transfer_asset_xcm > 0 && register_token > meth(100), + Error::::InvalidTokenTransferFees + ); + + let command = Command::SetTokenTransferFees { + create_asset_xcm, + transfer_asset_xcm, + register_token, + }; + Self::send(PRIMARY_GOVERNANCE_CHANNEL, command, PaysFee::::No)?; + + Self::deposit_event(Event::::SetTokenTransferFees { + create_asset_xcm, + transfer_asset_xcm, + register_token, + }); + Ok(()) + } + } + + impl Pallet { + /// Send `command` to the Gateway on the Channel identified by `channel_id` + fn send(channel_id: ChannelId, command: Command, pays_fee: PaysFee) -> DispatchResult { + let message = Message { id: None, channel_id, command }; + let (ticket, fee) = + T::OutboundQueue::validate(&message).map_err(|err| Error::::Send(err))?; + + let payment = match pays_fee { + PaysFee::Yes(account) => Some((account, fee.total())), + PaysFee::Partial(account) => Some((account, fee.local)), + PaysFee::No => None, + }; + + if let Some((payer, fee)) = payment { + T::Token::transfer( + &payer, + &T::TreasuryAccount::get(), + fee, + Preservation::Preserve, + )?; + } + + T::OutboundQueue::deliver(ticket).map_err(|err| Error::::Send(err))?; + Ok(()) + } + + /// Issue a `Command::TransferNativeFromAgent` command. The command will be sent on the + /// channel `channel_id` + pub fn do_transfer_native_from_agent( + agent_id: H256, + channel_id: ChannelId, + recipient: H160, + amount: u128, + pays_fee: PaysFee, + ) -> DispatchResult { + ensure!(Agents::::contains_key(agent_id), Error::::NoAgent); + + let command = Command::TransferNativeFromAgent { agent_id, recipient, amount }; + Self::send(channel_id, command, pays_fee)?; + + Self::deposit_event(Event::::TransferNativeFromAgent { + agent_id, + recipient, + amount, + }); + Ok(()) + } + + /// Initializes agents and channels. + pub fn initialize(para_id: ParaId, asset_hub_para_id: ParaId) -> Result<(), DispatchError> { + // Asset Hub + let asset_hub_location: Location = + ParentThen(Parachain(asset_hub_para_id.into()).into()).into(); + let asset_hub_agent_id = agent_id_of::(&asset_hub_location)?; + let asset_hub_channel_id: ChannelId = asset_hub_para_id.into(); + Agents::::insert(asset_hub_agent_id, ()); + Channels::::insert( + asset_hub_channel_id, + Channel { agent_id: asset_hub_agent_id, para_id: asset_hub_para_id }, + ); + + // Governance channels + let bridge_hub_agent_id = agent_id_of::(&Location::here())?; + // Agent for BridgeHub + Agents::::insert(bridge_hub_agent_id, ()); + + // Primary governance channel + Channels::::insert( + PRIMARY_GOVERNANCE_CHANNEL, + Channel { agent_id: bridge_hub_agent_id, para_id }, + ); + + // Secondary governance channel + Channels::::insert( + SECONDARY_GOVERNANCE_CHANNEL, + Channel { agent_id: bridge_hub_agent_id, para_id }, + ); + + Ok(()) + } + + /// Checks if the pallet has been initialized. + pub(crate) fn is_initialized() -> bool { + let primary_exists = Channels::::contains_key(PRIMARY_GOVERNANCE_CHANNEL); + let secondary_exists = Channels::::contains_key(SECONDARY_GOVERNANCE_CHANNEL); + primary_exists && secondary_exists + } + } + + impl StaticLookup for Pallet { + type Source = ChannelId; + type Target = Channel; + fn lookup(channel_id: Self::Source) -> Option { + Channels::::get(channel_id) + } + } + + impl Contains for Pallet { + fn contains(channel_id: &ChannelId) -> bool { + Channels::::get(channel_id).is_some() + } + } + + impl Get> for Pallet { + fn get() -> PricingParametersOf { + PricingParameters::::get() + } + } +} diff --git a/docs/sdk/src/reference_docs/development_environment_advice.rs b/docs/sdk/src/reference_docs/development_environment_advice.rs new file mode 100644 index 000000000000..431769597936 --- /dev/null +++ b/docs/sdk/src/reference_docs/development_environment_advice.rs @@ -0,0 +1,113 @@ +//! # Development Environment Advice +//! +//! Large Rust projects are known for sometimes long compile times and sluggish dev tooling, and +//! polkadot-sdk is no exception. +//! +//! This page contains some advice to improve your workflow when using common tooling. +//! +//! ## Rust Analyzer Configuration +//! +//! [Rust Analyzer](https://rust-analyzer.github.io/) is the defacto [LSP](https://langserver.org/) for Rust. Its default +//! settings are fine for smaller projects, but not well optimised for polkadot-sdk. +//! +//! Below is a suggested configuration for VSCode: +//! +//! ```json +//! { +//! // Use a separate target dir for Rust Analyzer. Helpful if you want to use Rust +//! // Analyzer and cargo on the command line at the same time. +//! "rust-analyzer.rust.analyzerTargetDir": "target/vscode-rust-analyzer", +//! // Improve stability +//! "rust-analyzer.server.extraEnv": { +//! "CHALK_OVERFLOW_DEPTH": "100000000", +//! "CHALK_SOLVER_MAX_SIZE": "10000000" +//! }, +//! // Check feature-gated code +//! "rust-analyzer.cargo.features": "all", +//! "rust-analyzer.cargo.extraEnv": { +//! // Skip building WASM, there is never need for it here +//! "SKIP_WASM_BUILD": "1" +//! }, +//! // Don't expand some problematic proc_macros +//! "rust-analyzer.procMacro.ignored": { +//! "async-trait": ["async_trait"], +//! "napi-derive": ["napi"], +//! "async-recursion": ["async_recursion"], +//! "async-std": ["async_std"] +//! }, +//! // Use nightly formatting. +//! // See the polkadot-sdk CI job that checks formatting for the current version used in +//! // polkadot-sdk. +//! "rust-analyzer.rustfmt.extraArgs": ["+nightly-2024-01-22"], +//! } +//! ``` +//! +//! and the same in Lua for `neovim/nvim-lspconfig`: +//! +//! ```lua +//! ["rust-analyzer"] = { +//! rust = { +//! # Use a separate target dir for Rust Analyzer. Helpful if you want to use Rust +//! # Analyzer and cargo on the command line at the same time. +//! analyzerTargetDir = "target/nvim-rust-analyzer", +//! }, +//! server = { +//! # Improve stability +//! extraEnv = { +//! ["CHALK_OVERFLOW_DEPTH"] = "100000000", +//! ["CHALK_SOLVER_MAX_SIZE"] = "100000000", +//! }, +//! }, +//! cargo = { +//! # Check feature-gated code +//! features = "all", +//! extraEnv = { +//! # Skip building WASM, there is never need for it here +//! ["SKIP_WASM_BUILD"] = "1", +//! }, +//! }, +//! procMacro = { +//! # Don't expand some problematic proc_macros +//! ignored = { +//! ["async-trait"] = { "async_trait" }, +//! ["napi-derive"] = { "napi" }, +//! ["async-recursion"] = { "async_recursion" }, +//! ["async-std"] = { "async_std" }, +//! }, +//! }, +//! rustfmt = { +//! # Use nightly formatting. +//! # See the polkadot-sdk CI job that checks formatting for the current version used in +//! # polkadot-sdk. +//! extraArgs = { "+nightly-2024-01-22" }, +//! }, +//! }, +//! ``` +//! +//! For the full set of configuration options see . +//! +//! ## Cargo Usage +//! +//! ### Using `--package` (a.k.a. `-p`) +//! +//! polkadot-sdk is a monorepo containing many crates. When you run a cargo command without +//! `-p`, you will almost certainly compile crates outside of the scope you are working. +//! +//! Instead, you should identify the name of the crate you are working on by checking the `name` +//! field in the closest `Cargo.toml` file. Then, use `-p` with your cargo commands to only compile +//! that crate. +//! +//! ### `SKIP_WASM_BUILD=1` environment variable +//! +//! When cargo touches a runtime crate, by default it will also compile the WASM binary, +//! approximately doubling the compilation time. +//! +//! The WASM binary is usually not needed, especially when running `check` or `test`. To skip the +//! WASM build, set the `SKIP_WASM_BUILD` environment variable to `1`. For example: +//! `SKIP_WASM_BUILD=1 cargo check -p frame-support`. +//! +//! ### Cargo Remote +//! +//! If you have a powerful remote server available, you may consider using +//! [cargo-remote](https://github.com/sgeisler/cargo-remote) to execute cargo commands on it, +//! freeing up local resources for other tasks like `rust-analyzer`. diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/availability/mod.rs new file mode 100644 index 000000000000..f9892efb3c68 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/availability/mod.rs @@ -0,0 +1,660 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +use crate::{core::mock::ChainApiState, TestEnvironment}; +use av_store::NetworkAvailabilityState; +use bitvec::bitvec; +use colored::Colorize; +use itertools::Itertools; +use polkadot_availability_bitfield_distribution::BitfieldDistribution; +use polkadot_node_core_av_store::AvailabilityStoreSubsystem; +use polkadot_node_subsystem::{Overseer, OverseerConnector, SpawnGlue}; +use polkadot_node_subsystem_types::{ + messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, + Span, +}; +use polkadot_overseer::Handle as OverseerHandle; +use sc_network::{request_responses::ProtocolConfig, PeerId}; +use sp_core::H256; +use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; + +use av_store_helpers::new_av_store; +use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; +use polkadot_availability_distribution::{ + AvailabilityDistributionSubsystem, IncomingRequestReceivers, +}; +use polkadot_node_metrics::metrics::Metrics; + +use polkadot_availability_recovery::AvailabilityRecoverySubsystem; +use polkadot_node_primitives::{AvailableData, ErasureChunk}; + +use crate::GENESIS_HASH; +use parity_scale_codec::Encode; +use polkadot_node_network_protocol::{ + request_response::{v1::ChunkFetchingRequest, IncomingRequest, ReqProtocolNames}, + OurView, Versioned, VersionedValidationProtocol, +}; +use sc_network::request_responses::IncomingRequest as RawIncomingRequest; + +use polkadot_node_primitives::{BlockData, PoV}; +use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; + +use crate::core::{ + environment::TestEnvironmentDependencies, + mock::{ + av_store, + network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, + runtime_api, MockAvailabilityStore, MockChainApi, MockRuntimeApi, + }, +}; + +use super::core::{configuration::TestConfiguration, mock::dummy_builder, network::*}; + +const LOG_TARGET: &str = "subsystem-bench::availability"; + +use super::{cli::TestObjective, core::mock::AlwaysSupportsParachains}; +use polkadot_node_subsystem_test_helpers::{ + derive_erasure_chunks_with_proofs_and_root, mock::new_block_import_info, +}; +use polkadot_primitives::{ + AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, + Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, +}; +use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use sc_service::SpawnTaskHandle; + +mod av_store_helpers; +mod cli; +pub use cli::{DataAvailabilityReadOptions, NetworkEmulation}; + +fn build_overseer_for_availability_read( + spawn_task_handle: SpawnTaskHandle, + runtime_api: MockRuntimeApi, + av_store: MockAvailabilityStore, + network_bridge: (MockNetworkBridgeTx, MockNetworkBridgeRx), + availability_recovery: AvailabilityRecoverySubsystem, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let dummy = dummy_builder!(spawn_task_handle); + let builder = dummy + .replace_runtime_api(|_| runtime_api) + .replace_availability_store(|_| av_store) + .replace_network_bridge_tx(|_| network_bridge.0) + .replace_network_bridge_rx(|_| network_bridge.1) + .replace_availability_recovery(|_| availability_recovery); + + let (overseer, raw_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail"); + + (overseer, OverseerHandle::new(raw_handle)) +} + +fn build_overseer_for_availability_write( + spawn_task_handle: SpawnTaskHandle, + runtime_api: MockRuntimeApi, + network_bridge: (MockNetworkBridgeTx, MockNetworkBridgeRx), + availability_distribution: AvailabilityDistributionSubsystem, + chain_api: MockChainApi, + availability_store: AvailabilityStoreSubsystem, + bitfield_distribution: BitfieldDistribution, +) -> (Overseer, AlwaysSupportsParachains>, OverseerHandle) { + let overseer_connector = OverseerConnector::with_event_capacity(64000); + let dummy = dummy_builder!(spawn_task_handle); + let builder = dummy + .replace_runtime_api(|_| runtime_api) + .replace_availability_store(|_| availability_store) + .replace_network_bridge_tx(|_| network_bridge.0) + .replace_network_bridge_rx(|_| network_bridge.1) + .replace_chain_api(|_| chain_api) + .replace_bitfield_distribution(|_| bitfield_distribution) + // This is needed to test own chunk recovery for `n_cores`. + .replace_availability_distribution(|_| availability_distribution); + + let (overseer, raw_handle) = + builder.build_with_connector(overseer_connector).expect("Should not fail"); + + (overseer, OverseerHandle::new(raw_handle)) +} + +/// Takes a test configuration and uses it to create the `TestEnvironment`. +pub fn prepare_test( + config: TestConfiguration, + state: &mut TestState, +) -> (TestEnvironment, Vec) { + prepare_test_inner(config, state, TestEnvironmentDependencies::default()) +} + +fn prepare_test_inner( + config: TestConfiguration, + state: &mut TestState, + dependencies: TestEnvironmentDependencies, +) -> (TestEnvironment, Vec) { + // Generate test authorities. + let test_authorities = config.generate_authorities(); + + let mut candidate_hashes: HashMap> = HashMap::new(); + + // Prepare per block candidates. + // Genesis block is always finalized, so we start at 1. + for block_num in 1..=config.num_blocks { + for _ in 0..config.n_cores { + candidate_hashes + .entry(Hash::repeat_byte(block_num as u8)) + .or_default() + .push(state.next_candidate().expect("Cycle iterator")) + } + + // First candidate is our backed candidate. + state.backed_candidates.push( + candidate_hashes + .get(&Hash::repeat_byte(block_num as u8)) + .expect("just inserted above") + .first() + .expect("just inserted above") + .clone(), + ); + } + + let runtime_api = runtime_api::MockRuntimeApi::new( + config.clone(), + test_authorities.clone(), + candidate_hashes, + ); + + let availability_state = NetworkAvailabilityState { + candidate_hashes: state.candidate_hashes.clone(), + available_data: state.available_data.clone(), + chunks: state.chunks.clone(), + }; + + let mut req_cfgs = Vec::new(); + + let (collation_req_receiver, collation_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + req_cfgs.push(collation_req_cfg); + + let (pov_req_receiver, pov_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + + let (chunk_req_receiver, chunk_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + req_cfgs.push(pov_req_cfg); + + let (network, network_interface, network_receiver) = + new_network(&config, &dependencies, &test_authorities, vec![Arc::new(availability_state)]); + + let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( + network.clone(), + network_interface.subsystem_sender(), + ); + + let network_bridge_rx = + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); + + let (overseer, overseer_handle) = match &state.config().objective { + TestObjective::DataAvailabilityRead(options) => { + let use_fast_path = options.fetch_from_backers; + + let subsystem = if use_fast_path { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + }; + + // Use a mocked av-store. + let av_store = av_store::MockAvailabilityStore::new( + state.chunks.clone(), + state.candidate_hashes.clone(), + ); + + build_overseer_for_availability_read( + dependencies.task_manager.spawn_handle(), + runtime_api, + av_store, + (network_bridge_tx, network_bridge_rx), + subsystem, + ) + }, + TestObjective::DataAvailabilityWrite => { + let availability_distribution = AvailabilityDistributionSubsystem::new( + test_authorities.keyring.keystore(), + IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, + Metrics::try_register(&dependencies.registry).unwrap(), + ); + + let block_headers = (0..=config.num_blocks) + .map(|block_number| { + ( + Hash::repeat_byte(block_number as u8), + Header { + digest: Default::default(), + number: block_number as BlockNumber, + parent_hash: Default::default(), + extrinsics_root: Default::default(), + state_root: Default::default(), + }, + ) + }) + .collect::>(); + + let chain_api_state = ChainApiState { block_headers }; + let chain_api = MockChainApi::new(chain_api_state); + let bitfield_distribution = + BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); + build_overseer_for_availability_write( + dependencies.task_manager.spawn_handle(), + runtime_api, + (network_bridge_tx, network_bridge_rx), + availability_distribution, + chain_api, + new_av_store(&dependencies), + bitfield_distribution, + ) + }, + _ => { + unimplemented!("Invalid test objective") + }, + }; + + ( + TestEnvironment::new( + dependencies, + config, + network, + overseer, + overseer_handle, + test_authorities, + ), + req_cfgs, + ) +} + +#[derive(Clone)] +pub struct TestState { + // Full test configuration + config: TestConfiguration, + // A cycle iterator on all PoV sizes used in the test. + pov_sizes: Cycle>, + // Generated candidate receipts to be used in the test + candidates: Cycle>, + // Map from pov size to candidate index + pov_size_to_candidate: HashMap, + // Map from generated candidate hashes to candidate index in `available_data` + // and `chunks`. + candidate_hashes: HashMap, + // Per candidate index receipts. + candidate_receipt_templates: Vec, + // Per candidate index `AvailableData` + available_data: Vec, + // Per candiadte index chunks + chunks: Vec>, + // Per relay chain block - candidate backed by our backing group + backed_candidates: Vec, +} + +impl TestState { + fn config(&self) -> &TestConfiguration { + &self.config + } + + pub fn next_candidate(&mut self) -> Option { + let candidate = self.candidates.next(); + let candidate_hash = candidate.as_ref().unwrap().hash(); + gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); + candidate + } + + /// Generate candidates to be used in the test. + fn generate_candidates(&mut self) { + let count = self.config.n_cores * self.config.num_blocks; + gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); + + // Generate all candidates + self.candidates = (0..count) + .map(|index| { + let pov_size = self.pov_sizes.next().expect("This is a cycle; qed"); + let candidate_index = *self + .pov_size_to_candidate + .get(&pov_size) + .expect("pov_size always exists; qed"); + let mut candidate_receipt = + self.candidate_receipt_templates[candidate_index].clone(); + + // Make it unique. + candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); + // Store the new candidate in the state + self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); + + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); + + candidate_receipt + }) + .collect::>() + .into_iter() + .cycle(); + } + + pub fn new(config: &TestConfiguration) -> Self { + let config = config.clone(); + + let mut chunks = Vec::new(); + let mut available_data = Vec::new(); + let mut candidate_receipt_templates = Vec::new(); + let mut pov_size_to_candidate = HashMap::new(); + + // we use it for all candidates. + let persisted_validation_data = PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: Default::default(), + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + }; + + // For each unique pov we create a candidate receipt. + for (index, pov_size) in config.pov_sizes().iter().cloned().unique().enumerate() { + gum::info!(target: LOG_TARGET, index, pov_size, "{}", "Generating template candidate".bright_blue()); + + let mut candidate_receipt = dummy_candidate_receipt(dummy_hash()); + let pov = PoV { block_data: BlockData(vec![index as u8; pov_size]) }; + + let new_available_data = AvailableData { + validation_data: persisted_validation_data.clone(), + pov: Arc::new(pov), + }; + + let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( + config.n_validators, + &new_available_data, + |_, _| {}, + ); + + candidate_receipt.descriptor.erasure_root = erasure_root; + + chunks.push(new_chunks); + available_data.push(new_available_data); + pov_size_to_candidate.insert(pov_size, index); + candidate_receipt_templates.push(candidate_receipt); + } + + gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); + + let mut _self = Self { + available_data, + candidate_receipt_templates, + chunks, + pov_size_to_candidate, + pov_sizes: Vec::from(config.pov_sizes()).into_iter().cycle(), + candidate_hashes: HashMap::new(), + candidates: Vec::new().into_iter().cycle(), + backed_candidates: Vec::new(), + config, + }; + + _self.generate_candidates(); + _self + } + + pub fn backed_candidates(&mut self) -> &mut Vec { + &mut self.backed_candidates + } +} + +pub async fn benchmark_availability_read(env: &mut TestEnvironment, mut state: TestState) { + let config = env.config().clone(); + + env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; + + let test_start = Instant::now(); + let mut batch = FuturesUnordered::new(); + let mut availability_bytes = 0u128; + + env.metrics().set_n_validators(config.n_validators); + env.metrics().set_n_cores(config.n_cores); + + for block_num in 1..=env.config().num_blocks { + gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); + env.metrics().set_current_block(block_num); + + let block_start_ts = Instant::now(); + for candidate_num in 0..config.n_cores as u64 { + let candidate = + state.next_candidate().expect("We always send up to n_cores*num_blocks; qed"); + let (tx, rx) = oneshot::channel(); + batch.push(rx); + + let message = AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData( + candidate.clone(), + 1, + Some(GroupIndex( + candidate_num as u32 % (std::cmp::max(5, config.n_cores) / 5) as u32, + )), + tx, + ), + ); + env.send_message(message).await; + } + + gum::info!(target: LOG_TARGET, "{}", format!("{} recoveries pending", batch.len()).bright_black()); + while let Some(completed) = batch.next().await { + let available_data = completed.unwrap().unwrap(); + env.metrics().on_pov_size(available_data.encoded_size()); + availability_bytes += available_data.encoded_size() as u128; + } + + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; + env.metrics().set_block_time(block_time); + gum::info!(target: LOG_TARGET, "All work for block completed in {}", format!("{:?}ms", block_time).cyan()); + } + + let duration: u128 = test_start.elapsed().as_millis(); + let availability_bytes = availability_bytes / 1024; + gum::info!(target: LOG_TARGET, "All blocks processed in {}", format!("{:?}ms", duration).cyan()); + gum::info!(target: LOG_TARGET, + "Throughput: {}", + format!("{} KiB/block", availability_bytes / env.config().num_blocks as u128).bright_red() + ); + gum::info!(target: LOG_TARGET, + "Avg block time: {}", + format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() + ); + + env.display_network_usage(); + env.display_cpu_usage(&["availability-recovery"]); + env.stop().await; +} + +pub async fn benchmark_availability_write(env: &mut TestEnvironment, mut state: TestState) { + let config = env.config().clone(); + + env.metrics().set_n_validators(config.n_validators); + env.metrics().set_n_cores(config.n_cores); + + gum::info!(target: LOG_TARGET, "Seeding availability store with candidates ..."); + for backed_candidate in state.backed_candidates().clone() { + let candidate_index = *state.candidate_hashes.get(&backed_candidate.hash()).unwrap(); + let available_data = state.available_data[candidate_index].clone(); + let (tx, rx) = oneshot::channel(); + env.send_message(AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData { + candidate_hash: backed_candidate.hash(), + n_validators: config.n_validators as u32, + available_data, + expected_erasure_root: backed_candidate.descriptor().erasure_root, + tx, + }, + )) + .await; + + rx.await + .unwrap() + .expect("Test candidates are stored nicely in availability store"); + } + + gum::info!(target: LOG_TARGET, "Done"); + + let test_start = Instant::now(); + + for block_num in 1..=env.config().num_blocks { + gum::info!(target: LOG_TARGET, "Current block #{}", block_num); + env.metrics().set_current_block(block_num); + + let block_start_ts = Instant::now(); + let relay_block_hash = Hash::repeat_byte(block_num as u8); + env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) + .await; + + // Inform bitfield distribution about our view of current test block + let message = polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( + NetworkBridgeEvent::OurViewChange(OurView::new(vec![(relay_block_hash, Arc::new(Span::Disabled))], 0)) + ); + env.send_message(AllMessages::BitfieldDistribution(message)).await; + + let chunk_fetch_start_ts = Instant::now(); + + // Request chunks of our own backed candidate from all other validators. + let mut receivers = Vec::new(); + for index in 1..config.n_validators { + let (pending_response, pending_response_receiver) = oneshot::channel(); + + let request = RawIncomingRequest { + peer: PeerId::random(), + payload: ChunkFetchingRequest { + candidate_hash: state.backed_candidates()[block_num - 1].hash(), + index: ValidatorIndex(index as u32), + } + .encode(), + pending_response, + }; + + let peer = env + .authorities() + .validator_authority_id + .get(index) + .expect("all validators have keys"); + + if env.network().is_peer_connected(peer) && + env.network().send_request_from_peer(peer, request).is_ok() + { + receivers.push(pending_response_receiver); + } + } + + gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); + for receiver in receivers.into_iter() { + let response = receiver.await.expect("Chunk is always served succesfully"); + // TODO: check if chunk is the one the peer expects to receive. + assert!(response.result.is_ok()); + } + + let chunk_fetch_duration = Instant::now().sub(chunk_fetch_start_ts).as_millis(); + + gum::info!(target: LOG_TARGET, "All chunks received in {}ms", chunk_fetch_duration); + + let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; + let network = env.network().clone(); + let authorities = env.authorities().clone(); + let n_validators = config.n_validators; + + // Spawn a task that will generate `n_validator` - 1 signed bitfiends and + // send them from the emulated peers to the subsystem. + // TODO: Implement topology. + env.spawn_blocking("send-bitfields", async move { + for index in 1..n_validators { + let validator_public = + authorities.validator_public.get(index).expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + // TODO(soon): Use pre-signed messages. This is quite intensive on the CPU. + let signed_bitfield = Signed::::sign( + &authorities.keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + validator_public, + ) + .ok() + .flatten() + .expect("should be signed"); + + let from_peer = &authorities.validator_authority_id[index]; + + let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); + + // Send the action from peer only if it is connected to our node. + if network.is_peer_connected(from_peer) { + let _ = network.send_message_from_peer(from_peer, message); + } + } + }); + + gum::info!( + "Waiting for {} bitfields to be received and processed", + config.connected_count() + ); + + // Wait for all bitfields to be processed. + env.wait_until_metric_eq( + "polkadot_parachain_received_availabilty_bitfields_total", + config.connected_count() * block_num, + ) + .await; + + gum::info!(target: LOG_TARGET, "All bitfields processed"); + + let block_time = Instant::now().sub(block_start_ts).as_millis() as u64; + env.metrics().set_block_time(block_time); + gum::info!(target: LOG_TARGET, "All work for block completed in {}", format!("{:?}ms", block_time).cyan()); + } + + let duration: u128 = test_start.elapsed().as_millis(); + gum::info!(target: LOG_TARGET, "All blocks processed in {}", format!("{:?}ms", duration).cyan()); + gum::info!(target: LOG_TARGET, + "Avg block time: {}", + format!("{} ms", test_start.elapsed().as_millis() / env.config().num_blocks as u128).red() + ); + + env.display_network_usage(); + + env.display_cpu_usage(&[ + "availability-distribution", + "bitfield-distribution", + "availability-store", + ]); + + env.stop().await; +} + +pub fn peer_bitfield_message_v2( + relay_hash: H256, + signed_bitfield: Signed, +) -> VersionedValidationProtocol { + let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( + relay_hash, + signed_bitfield.into(), + ); + + Versioned::V2(polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( + bitfield, + )) +} diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/core/keyring.rs new file mode 100644 index 000000000000..66c7229847c3 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/keyring.rs @@ -0,0 +1,46 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_primitives::ValidatorId; +use sc_keystore::LocalKeystore; +use sp_application_crypto::AppCrypto; +use sp_core::sr25519::Public; +use sp_keystore::Keystore; +use std::sync::Arc; + +/// Set of test accounts generated and kept safe by a keystore. +#[derive(Clone)] +pub struct Keyring { + keystore: Arc, +} + +impl Default for Keyring { + fn default() -> Self { + Self { keystore: Arc::new(LocalKeystore::in_memory()) } + } +} + +impl Keyring { + pub fn sr25519_new(&self, name: String) -> Public { + self.keystore + .sr25519_generate_new(ValidatorId::ID, Some(&format!("//{}", name))) + .expect("Insert key into keystore") + } + + pub fn keystore(&self) -> Arc { + self.keystore.clone() + } +} diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/core/mock/mod.rs new file mode 100644 index 000000000000..b67c6611e8cd --- /dev/null +++ b/polkadot/node/subsystem-bench/src/core/mock/mod.rs @@ -0,0 +1,78 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_node_subsystem::HeadSupportsParachains; +use polkadot_node_subsystem_types::Hash; + +pub mod av_store; +pub mod chain_api; +pub mod dummy; +pub mod network_bridge; +pub mod runtime_api; + +pub use av_store::*; +pub use chain_api::*; +pub use runtime_api::*; + +pub struct AlwaysSupportsParachains {} +#[async_trait::async_trait] +impl HeadSupportsParachains for AlwaysSupportsParachains { + async fn head_supports_parachains(&self, _head: &Hash) -> bool { + true + } +} + +// An orchestra with dummy subsystems +macro_rules! dummy_builder { + ($spawn_task_handle: ident) => {{ + use super::core::mock::dummy::*; + + // Initialize a mock overseer. + // All subsystem except approval_voting and approval_distribution are mock subsystems. + Overseer::builder() + .approval_voting(MockApprovalVoting {}) + .approval_distribution(MockApprovalDistribution {}) + .availability_recovery(MockAvailabilityRecovery {}) + .candidate_validation(MockCandidateValidation {}) + .chain_api(MockChainApi {}) + .chain_selection(MockChainSelection {}) + .dispute_coordinator(MockDisputeCoordinator {}) + .runtime_api(MockRuntimeApi {}) + .network_bridge_tx(MockNetworkBridgeTx {}) + .availability_distribution(MockAvailabilityDistribution {}) + .availability_store(MockAvailabilityStore {}) + .pvf_checker(MockPvfChecker {}) + .candidate_backing(MockCandidateBacking {}) + .statement_distribution(MockStatementDistribution {}) + .bitfield_signing(MockBitfieldSigning {}) + .bitfield_distribution(MockBitfieldDistribution {}) + .provisioner(MockProvisioner {}) + .network_bridge_rx(MockNetworkBridgeRx {}) + .collation_generation(MockCollationGeneration {}) + .collator_protocol(MockCollatorProtocol {}) + .gossip_support(MockGossipSupport {}) + .dispute_distribution(MockDisputeDistribution {}) + .prospective_parachains(MockProspectiveParachains {}) + .activation_external_listeners(Default::default()) + .span_per_active_leaf(Default::default()) + .active_leaves(Default::default()) + .metrics(Default::default()) + .supports_parachains(AlwaysSupportsParachains {}) + .spawner(SpawnGlue($spawn_task_handle)) + }}; +} + +pub(crate) use dummy_builder; diff --git a/substrate/frame/alliance/src/mock.rs b/substrate/frame/alliance/src/mock.rs index ace5214f145f..c9e5ff84b7f5 100644 --- a/substrate/frame/alliance/src/mock.rs +++ b/substrate/frame/alliance/src/mock.rs @@ -20,7 +20,7 @@ pub use sp_core::H256; use sp_runtime::traits::Hash; pub use sp_runtime::{ - traits::{BlakeTwo256, IdentityLookup}, + traits::{BlakeTwo256, IdentifyAccount, Lazy, Verify}, BuildStorage, }; use sp_std::convert::{TryFrom, TryInto}; diff --git a/substrate/frame/contracts/fixtures/build.rs b/substrate/frame/contracts/fixtures/build.rs new file mode 100644 index 000000000000..12a7805294b6 --- /dev/null +++ b/substrate/frame/contracts/fixtures/build.rs @@ -0,0 +1,350 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Compile contracts to wasm and RISC-V binaries. +use anyhow::{bail, Context, Result}; +use parity_wasm::elements::{deserialize_file, serialize_to_file, Internal}; +use std::{ + env, fs, + hash::Hasher, + path::{Path, PathBuf}, + process::Command, +}; +use twox_hash::XxHash32; + +/// Read the file at `path` and return its hash as a hex string. +fn file_hash(path: &Path) -> String { + let data = fs::read(path).expect("file exists; qed"); + let mut hasher = XxHash32::default(); + hasher.write(&data); + hasher.write(include_bytes!("build.rs")); + let hash = hasher.finish(); + format!("{:x}", hash) +} + +/// A contract entry. +struct Entry { + /// The path to the contract source file. + path: PathBuf, + /// The hash of the contract source file. + hash: String, +} + +impl Entry { + /// Create a new contract entry from the given path. + fn new(path: PathBuf) -> Self { + let hash = file_hash(&path); + Self { path, hash } + } + + /// Return the path to the contract source file. + fn path(&self) -> &str { + self.path.to_str().expect("path is valid unicode; qed") + } + + /// Return the name of the contract. + fn name(&self) -> &str { + self.path + .file_stem() + .expect("file exits; qed") + .to_str() + .expect("name is valid unicode; qed") + } + + /// Return whether the contract has already been compiled. + fn is_cached(&self, out_dir: &Path) -> bool { + out_dir.join(self.name()).join(&self.hash).exists() + } + + /// Update the cache file for the contract. + fn update_cache(&self, out_dir: &Path) -> Result<()> { + let cache_dir = out_dir.join(self.name()); + + // clear the cache dir if it exists + if cache_dir.exists() { + fs::remove_dir_all(&cache_dir)?; + } + + // re-populate the cache dir with the new hash + fs::create_dir_all(&cache_dir)?; + fs::write(out_dir.join(&self.hash), "")?; + Ok(()) + } + + /// Return the name of the output wasm file. + fn out_wasm_filename(&self) -> String { + format!("{}.wasm", self.name()) + } + + /// Return the name of the RISC-V polkavm file. + #[cfg(feature = "riscv")] + fn out_riscv_filename(&self) -> String { + format!("{}.polkavm", self.name()) + } +} + +/// Collect all contract entries from the given source directory. +/// Contracts that have already been compiled are filtered out. +fn collect_entries(contracts_dir: &Path, out_dir: &Path) -> Vec { + fs::read_dir(contracts_dir) + .expect("src dir exists; qed") + .filter_map(|file| { + let path = file.expect("file exists; qed").path(); + if path.extension().map_or(true, |ext| ext != "rs") { + return None + } + + let entry = Entry::new(path); + if entry.is_cached(out_dir) { + None + } else { + Some(entry) + } + }) + .collect::>() +} + +/// Create a `Cargo.toml` to compile the given contract entries. +fn create_cargo_toml<'a>( + fixtures_dir: &Path, + entries: impl Iterator, + output_dir: &Path, +) -> Result<()> { + let mut cargo_toml: toml::Value = toml::from_str(include_str!("./build/Cargo.toml"))?; + let mut set_dep = |name, path| -> Result<()> { + cargo_toml["dependencies"][name]["path"] = toml::Value::String( + fixtures_dir.join(path).canonicalize()?.to_str().unwrap().to_string(), + ); + Ok(()) + }; + set_dep("uapi", "../uapi")?; + set_dep("common", "./contracts/common")?; + + cargo_toml["bin"] = toml::Value::Array( + entries + .map(|entry| { + let name = entry.name(); + let path = entry.path(); + toml::Value::Table(toml::toml! { + name = name + path = path + }) + }) + .collect::>(), + ); + + let cargo_toml = toml::to_string_pretty(&cargo_toml)?; + fs::write(output_dir.join("Cargo.toml"), cargo_toml).map_err(Into::into) +} + +/// Invoke `cargo fmt` to check that fixtures files are formatted. +fn invoke_cargo_fmt<'a>( + config_path: &Path, + files: impl Iterator, + contract_dir: &Path, +) -> Result<()> { + // If rustfmt is not installed, skip the check. + if !Command::new("rustup") + .args(["nightly-2024-01-22", "run", "rustfmt", "--version"]) + .output() + .map_or(false, |o| o.status.success()) + { + return Ok(()) + } + + let fmt_res = Command::new("rustup") + .args(["nightly-2024-01-22", "run", "rustfmt", "--check", "--config-path"]) + .arg(config_path) + .args(files) + .output() + .expect("failed to execute process"); + + if fmt_res.status.success() { + return Ok(()) + } + + let stdout = String::from_utf8_lossy(&fmt_res.stdout); + let stderr = String::from_utf8_lossy(&fmt_res.stderr); + eprintln!("{}\n{}", stdout, stderr); + eprintln!( + "Fixtures files are not formatted.\n + Please run `rustup nightly-2024-01-22 run rustfmt --config-path {} {}/*.rs`", + config_path.display(), + contract_dir.display() + ); + + anyhow::bail!("Fixtures files are not formatted") +} + +/// Build contracts for wasm. +fn invoke_wasm_build(current_dir: &Path) -> Result<()> { + let encoded_rustflags = [ + "-Clink-arg=-zstack-size=65536", + "-Clink-arg=--import-memory", + "-Clinker-plugin-lto", + "-Ctarget-cpu=mvp", + "-Dwarnings", + ] + .join("\x1f"); + + let build_res = Command::new(env::var("CARGO")?) + .current_dir(current_dir) + .env("CARGO_TARGET_DIR", current_dir.join("target").display().to_string()) + .env("CARGO_ENCODED_RUSTFLAGS", encoded_rustflags) + .args(["build", "--release", "--target=wasm32-unknown-unknown"]) + .output() + .expect("failed to execute process"); + + if build_res.status.success() { + return Ok(()) + } + + let stderr = String::from_utf8_lossy(&build_res.stderr); + eprintln!("{}", stderr); + bail!("Failed to build wasm contracts"); +} + +/// Post-process the compiled wasm contracts. +fn post_process_wasm(input_path: &Path, output_path: &Path) -> Result<()> { + let mut module = + deserialize_file(input_path).with_context(|| format!("Failed to read {:?}", input_path))?; + if let Some(section) = module.export_section_mut() { + section.entries_mut().retain(|entry| { + matches!(entry.internal(), Internal::Function(_)) && + (entry.field() == "call" || entry.field() == "deploy") + }); + } + + serialize_to_file(output_path, module).map_err(Into::into) +} + +/// Build contracts for RISC-V. +#[cfg(feature = "riscv")] +fn invoke_riscv_build(current_dir: &Path) -> Result<()> { + let encoded_rustflags = [ + "-Crelocation-model=pie", + "-Clink-arg=--emit-relocs", + "-Clink-arg=--export-dynamic-symbol=__polkavm_symbol_export_hack__*", + ] + .join("\x1f"); + + let build_res = Command::new(env::var("CARGO")?) + .current_dir(current_dir) + .env_clear() + .env("PATH", env::var("PATH").unwrap_or_default()) + .env("CARGO_ENCODED_RUSTFLAGS", encoded_rustflags) + .env("RUSTUP_TOOLCHAIN", "rve-nightly") + .env("RUSTUP_HOME", env::var("RUSTUP_HOME").unwrap_or_default()) + .args(["build", "--release", "--target=riscv32ema-unknown-none-elf"]) + .output() + .expect("failed to execute process"); + + if build_res.status.success() { + return Ok(()) + } + + let stderr = String::from_utf8_lossy(&build_res.stderr); + + if stderr.contains("'rve-nightly' is not installed") { + eprintln!("RISC-V toolchain is not installed.\nDownload and install toolchain from https://github.com/paritytech/rustc-rv32e-toolchain."); + eprintln!("{}", stderr); + } else { + eprintln!("{}", stderr); + } + + bail!("Failed to build contracts"); +} +/// Post-process the compiled wasm contracts. +#[cfg(feature = "riscv")] +fn post_process_riscv(input_path: &Path, output_path: &Path) -> Result<()> { + let mut config = polkavm_linker::Config::default(); + config.set_strip(true); + let orig = fs::read(input_path).with_context(|| format!("Failed to read {:?}", input_path))?; + let linked = polkavm_linker::program_from_elf(config, orig.as_ref()) + .map_err(|err| anyhow::format_err!("Failed to link polkavm program: {}", err))?; + fs::write(output_path, linked.as_bytes()).map_err(Into::into) +} + +/// Write the compiled contracts to the given output directory. +fn write_output(build_dir: &Path, out_dir: &Path, entries: Vec) -> Result<()> { + for entry in entries { + let wasm_output = entry.out_wasm_filename(); + post_process_wasm( + &build_dir.join("target/wasm32-unknown-unknown/release").join(&wasm_output), + &out_dir.join(&wasm_output), + )?; + + #[cfg(feature = "riscv")] + post_process_riscv( + &build_dir.join("target/riscv32ema-unknown-none-elf/release").join(entry.name()), + &out_dir.join(entry.out_riscv_filename()), + )?; + + entry.update_cache(out_dir)?; + } + + Ok(()) +} + +/// Returns the root path of the wasm workspace. +fn find_workspace_root(current_dir: &Path) -> Option { + let mut current_dir = current_dir.to_path_buf(); + + while current_dir.parent().is_some() { + if current_dir.join("Cargo.toml").exists() { + let cargo_toml_contents = + std::fs::read_to_string(current_dir.join("Cargo.toml")).ok()?; + if cargo_toml_contents.contains("[workspace]") { + return Some(current_dir); + } + } + + current_dir.pop(); + } + + None +} + +fn main() -> Result<()> { + let fixtures_dir: PathBuf = env::var("CARGO_MANIFEST_DIR")?.into(); + let contracts_dir = fixtures_dir.join("contracts"); + let out_dir: PathBuf = env::var("OUT_DIR")?.into(); + let workspace_root = find_workspace_root(&fixtures_dir).expect("workspace root exists; qed"); + + let entries = collect_entries(&contracts_dir, &out_dir); + if entries.is_empty() { + return Ok(()) + } + + let tmp_dir = tempfile::tempdir()?; + let tmp_dir_path = tmp_dir.path(); + + create_cargo_toml(&fixtures_dir, entries.iter(), tmp_dir.path())?; + invoke_cargo_fmt( + &workspace_root.join(".rustfmt.toml"), + entries.iter().map(|entry| &entry.path as _), + &contracts_dir, + )?; + + invoke_wasm_build(tmp_dir_path)?; + + #[cfg(feature = "riscv")] + invoke_riscv_build(tmp_dir_path)?; + + write_output(tmp_dir_path, &out_dir, entries)?; + Ok(()) +} diff --git a/substrate/frame/support/procedural/src/pallet/expand/call.rs b/substrate/frame/support/procedural/src/pallet/expand/call.rs index cf302faafc78..74332ecf499e 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/call.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/call.rs @@ -252,6 +252,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { }); quote::quote_spanned!(span => + #[doc(hidden)] mod warnings { #( #call_index_warnings @@ -261,6 +262,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { )* } + #[allow(unused_imports)] #[doc(hidden)] pub mod __substrate_call_check { #[macro_export] @@ -441,6 +443,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { } impl<#type_impl_gen> #pallet_ident<#type_use_gen> #where_clause { + #[allow(dead_code)] #[doc(hidden)] pub fn call_functions() -> #frame_support::__private::metadata_ir::PalletCallMetadataIR { #frame_support::__private::scale_info::meta_type::<#call_ident<#type_use_gen>>().into() diff --git a/substrate/frame/support/test/tests/construct_runtime_ui/deprecated_where_block.stderr b/substrate/frame/support/test/tests/construct_runtime_ui/deprecated_where_block.stderr index 08954bb6ab5c..09d62c8ee9e6 100644 --- a/substrate/frame/support/test/tests/construct_runtime_ui/deprecated_where_block.stderr +++ b/substrate/frame/support/test/tests/construct_runtime_ui/deprecated_where_block.stderr @@ -448,3 +448,69 @@ note: required by a bound in `Result` | pub enum Result { | ^ required by this bound in `Result` = note: this error originates in the derive macro `self::sp_api_hidden_includes_construct_runtime::hidden_include::__private::codec::Decode` which comes from the expansion of the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `Runtime: Config` is not satisfied + --> tests/construct_runtime_ui/deprecated_where_block.rs:20:1 + | +20 | // construct_runtime! { +21 | || pub struct Runtime where +22 | || Block = Block, +23 | || NodeBlock = Block, +... || +27 | || } +28 | || } + | ||_- in this macro invocation +... | + | + = note: required for `Pallet` to implement `Callable` +note: required because it appears within the type `RuntimeCall` + --> tests/construct_runtime_ui/deprecated_where_block.rs:20:1 + | +20 | // construct_runtime! { +21 | || pub struct Runtime where +22 | || Block = Block, +23 | || NodeBlock = Block, +... || +27 | || } +28 | || } + | ||_- in this macro invocation +... | +note: required by a bound in `frame_support::sp_runtime::traits::Dispatchable::Config` + --> $WORKSPACE/substrate/primitives/runtime/src/traits.rs + | + | type Config; + | ^^^^^^^^^^^^ required by this bound in `Dispatchable::Config` + = note: this error originates in the macro `frame_support::construct_runtime` which comes from the expansion of the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `Runtime: Config` is not satisfied + --> tests/construct_runtime_ui/deprecated_where_block.rs:20:1 + | +20 | // construct_runtime! { +21 | || pub struct Runtime where +22 | || Block = Block, +23 | || NodeBlock = Block, +... || +27 | || } +28 | || } + | ||_- in this macro invocation +... | + | + = note: required for `Pallet` to implement `Callable` +note: required because it appears within the type `RuntimeCall` + --> tests/construct_runtime_ui/deprecated_where_block.rs:20:1 + | +20 | // construct_runtime! { +21 | || pub struct Runtime where +22 | || Block = Block, +23 | || NodeBlock = Block, +... || +27 | || } +28 | || } + | ||_- in this macro invocation +... | +note: required by a bound in `frame_support::pallet_prelude::ValidateUnsigned::Call` + --> $WORKSPACE/substrate/primitives/runtime/src/traits.rs + | + | type Call; + | ^^^^^^^^^^ required by this bound in `ValidateUnsigned::Call` + = note: this error originates in the macro `frame_support::construct_runtime` which comes from the expansion of the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/substrate/frame/support/test/tests/pallet_ui/default_config_with_no_default_in_system.stderr b/substrate/frame/support/test/tests/pallet_ui/default_config_with_no_default_in_system.stderr index a86fd96ea28d..5a5b48097369 100644 --- a/substrate/frame/support/test/tests/pallet_ui/default_config_with_no_default_in_system.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/default_config_with_no_default_in_system.stderr @@ -2,4 +2,4 @@ error[E0220]: associated type `Block` not found for `Self` --> tests/pallet_ui/default_config_with_no_default_in_system.rs:25:31 | 25 | type MyGetParam2: Get; - | ^^^^^ there is a similarly named associated type `Block` in the trait `frame_system::Config` + | ^^^^^ there is an associated type `Block` in the trait `frame_system::Config` diff --git a/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs b/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs index 2e92f5358eab..3c6730471cb5 100644 --- a/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs +++ b/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs @@ -16,6 +16,7 @@ // limitations under the License. #[frame_support::pallet] +#[allow(unused_imports)] mod pallet { #[pallet::config] pub trait Config: frame_system::Config {} @@ -32,5 +33,4 @@ mod pallet { #[derive(scale_info::TypeInfo, codec::Encode, codec::Decode)] enum MyError {} -fn main() { -} +fn main() {} diff --git a/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr b/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr index ea1d0ed99cd3..f8277d8d8404 100644 --- a/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr @@ -1,8 +1,8 @@ error[E0277]: the trait bound `MyError: PalletError` is not satisfied - --> tests/pallet_ui/error_does_not_derive_pallet_error.rs:18:1 + --> tests/pallet_ui/error_does_not_derive_pallet_error.rs:29:15 | -18 | #[frame_support::pallet] - | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PalletError` is not implemented for `MyError` +29 | CustomError(crate::MyError), + | ^^^^^^^^^^^^^^ the trait `PalletError` is not implemented for `MyError` | = help: the following other types implement trait `PalletError`: bool diff --git a/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.rs b/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.rs index a7e6d213318f..33603d6f559a 100644 --- a/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.rs +++ b/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.rs @@ -16,6 +16,7 @@ // limitations under the License. #[frame_support::pallet] +#[allow(unused_imports)] mod pallet { use frame_support::pallet_prelude::{Hooks, IsType}; use frame_system::pallet_prelude::BlockNumberFor; @@ -41,5 +42,4 @@ mod pallet { } } -fn main() { -} +fn main() {} diff --git a/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.stderr b/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.stderr index fc4a33b72150..44660d269060 100644 --- a/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/event_field_not_member.stderr @@ -1,19 +1,19 @@ error[E0277]: the trait bound `::Bar: Clone` is not satisfied - --> tests/pallet_ui/event_field_not_member.rs:40:7 + --> tests/pallet_ui/event_field_not_member.rs:41:7 | -40 | B { b: T::Bar }, +41 | B { b: T::Bar }, | ^ the trait `Clone` is not implemented for `::Bar` error[E0369]: binary operation `==` cannot be applied to type `&::Bar` - --> tests/pallet_ui/event_field_not_member.rs:40:7 + --> tests/pallet_ui/event_field_not_member.rs:41:7 | -40 | B { b: T::Bar }, +41 | B { b: T::Bar }, | ^ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` - --> tests/pallet_ui/event_field_not_member.rs:40:7 + --> tests/pallet_ui/event_field_not_member.rs:41:7 | -40 | B { b: T::Bar }, +41 | B { b: T::Bar }, | ^ `::Bar` cannot be formatted using `{:?}` because it doesn't implement `std::fmt::Debug` | = help: the trait `std::fmt::Debug` is not implemented for `::Bar` diff --git a/substrate/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs b/substrate/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs index 2e0df1ac398a..0e0dc5890519 100644 --- a/substrate/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs +++ b/substrate/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs @@ -19,6 +19,7 @@ use codec::{Decode, Encode}; use frame_support::PalletError; #[frame_support::pallet] +#[allow(unused_imports)] mod pallet { #[pallet::config] pub trait Config: frame_system::Config {} @@ -34,25 +35,24 @@ mod pallet { #[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] pub enum MyError { - Foo, - Bar, - Baz(NestedError), - Struct(MyStruct), - Wrapper(Wrapper), + Foo, + Bar, + Baz(NestedError), + Struct(MyStruct), + Wrapper(Wrapper), } #[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] pub enum NestedError { - Quux + Quux, } #[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] pub struct MyStruct { - field: u8, + field: u8, } #[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] pub struct Wrapper(bool); -fn main() { -} +fn main() {} diff --git a/substrate/frame/support/test/tests/pallet_ui/pass/event_type_bound_system_config_assoc_type.rs b/substrate/frame/support/test/tests/pallet_ui/pass/event_type_bound_system_config_assoc_type.rs index 0a70c4a5e68a..d84d320b0b24 100644 --- a/substrate/frame/support/test/tests/pallet_ui/pass/event_type_bound_system_config_assoc_type.rs +++ b/substrate/frame/support/test/tests/pallet_ui/pass/event_type_bound_system_config_assoc_type.rs @@ -16,6 +16,7 @@ // limitations under the License. #[frame_support::pallet] +#[allow(unused_imports)] mod pallet { use frame_support::pallet_prelude::{Hooks, IsType}; use frame_system::pallet_prelude::BlockNumberFor; diff --git a/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.rs b/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.rs index 4171596ddde7..60f8b9a0cad8 100644 --- a/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.rs +++ b/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.rs @@ -16,6 +16,7 @@ // limitations under the License. #[frame_support::pallet] +#[allow(unused_imports)] mod pallet { use frame_support::pallet_prelude::*; @@ -34,5 +35,4 @@ mod pallet { type Foo = StorageValue<_, u8, ResultQuery>; } -fn main() { -} +fn main() {} diff --git a/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.stderr b/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.stderr index 7ca7ec8bbb7d..af8ff50bf421 100644 --- a/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/storage_result_query_missing_generics.stderr @@ -1,15 +1,15 @@ error[E0107]: missing generics for enum `pallet::Error` - --> tests/pallet_ui/storage_result_query_missing_generics.rs:34:56 + --> tests/pallet_ui/storage_result_query_missing_generics.rs:35:56 | -34 | type Foo = StorageValue<_, u8, ResultQuery>; +35 | type Foo = StorageValue<_, u8, ResultQuery>; | ^^^^^ expected 1 generic argument | note: enum defined here, with 1 generic parameter: `T` - --> tests/pallet_ui/storage_result_query_missing_generics.rs:29:11 + --> tests/pallet_ui/storage_result_query_missing_generics.rs:30:11 | -29 | pub enum Error { +30 | pub enum Error { | ^^^^^ - help: add missing generic argument | -34 | type Foo = StorageValue<_, u8, ResultQuery::NonExistentValue>>; +35 | type Foo = StorageValue<_, u8, ResultQuery::NonExistentValue>>; | +++ diff --git a/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs b/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs index 4647a78809ab..e6533b4ccda6 100644 --- a/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs +++ b/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs @@ -25,6 +25,7 @@ sp_api::decl_runtime_apis! { } } +#[allow(unused_imports)] mod second { sp_api::decl_runtime_apis! { pub trait Api { diff --git a/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr b/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr index 2197bbc99cfb..c401c7076e5c 100644 --- a/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr +++ b/substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr @@ -1,11 +1,11 @@ error: Two traits with the same name detected! The trait name is used to generate its ID. Please rename one trait at the declaration! - --> tests/ui/impl_two_traits_with_same_name.rs:41:15 + --> tests/ui/impl_two_traits_with_same_name.rs:42:15 | -41 | impl second::Api for Runtime { +42 | impl second::Api for Runtime { | ^^^ error: First trait implementation. - --> tests/ui/impl_two_traits_with_same_name.rs:37:13 + --> tests/ui/impl_two_traits_with_same_name.rs:38:13 | -37 | impl self::Api for Runtime { +38 | impl self::Api for Runtime { | ^^^