Bulletin as parachain missing features

cumulus/pallets/parachain-system/Cargo.toml

@@ -125,3 +125,13 @@ try-runtime = [
 	"polkadot-runtime-parachains/try-runtime",
 	"sp-runtime/try-runtime",
 ]
+
+# Enables `TransactionIndex` host functions for parachain validation
+# without requiring maintenance of a transaction index.
+# However, the host environment (polkadot-prepare/execute-worker) still expects these functions to exist,
+# so we provide a replacement no-op implementation gated by this feature.
+#
+# Used by: `validate_block::implementation::validate_block`.
+#
+# Note: The parachain collator software is expected to use/include `transaction_index::HostFunctions`.
+transaction-index = []

cumulus/pallets/parachain-system/src/validate_block/implementation.rs

@@ -122,6 +122,10 @@ where
 		sp_io::offchain_index::host_clear.replace_implementation(host_offchain_index_clear),
 		cumulus_primitives_proof_size_hostfunction::storage_proof_size::host_storage_proof_size
 			.replace_implementation(host_storage_proof_size),
+		#[cfg(feature = "transaction-index")]
+		sp_io::transaction_index::host_index.replace_implementation(host_transaction_index_index),
+		#[cfg(feature = "transaction-index")]
+		sp_io::transaction_index::host_renew.replace_implementation(host_transaction_index_renew),
 	);
 
 	let block_data = codec::decode_from_bytes::<ParachainBlockData<B::LazyBlock>>(block_data)
@@ -230,7 +234,6 @@ where
 		if overlay.storage(well_known_keys::CODE).is_some() && num_blocks > 1 {
 			panic!("When applying a runtime upgrade, only one block per PoV is allowed. Received {num_blocks}.")
 		}
-
 		run_with_externalities_and_recorder::<B, _, _>(
 			&backend,
 			&mut Default::default(),
@@ -548,3 +551,21 @@ fn host_default_child_storage_next_key(storage_key: &[u8], key: &[u8]) -> Option
 fn host_offchain_index_set(_key: &[u8], _value: &[u8]) {}
 
 fn host_offchain_index_clear(_key: &[u8]) {}
+
+/// Parachain validation does not require maintaining a transaction index,
+/// and indexing transactions does **not** contribute to the parachain state.
+/// However, the host environment still expects this function to exist,
+/// so we provide a no-op implementation.
+#[cfg(feature = "transaction-index")]
+fn host_transaction_index_index(_extrinsic: u32, _size: u32, _context_hash: [u8; 32]) {
+	// No-op host function used during parachain validation.
+}
+
+/// Parachain validation does not require maintaining a transaction index,
+/// and indexing transactions does **not** contribute to the parachain state.
+/// However, the host environment still expects this function to exist,
+/// so we provide a no-op implementation.
+#[cfg(feature = "transaction-index")]
+fn host_transaction_index_renew(_extrinsic: u32, _context_hash: [u8; 32]) {
+	// No-op host function used during parachain validation.
+}

cumulus/polkadot-omni-node/lib/Cargo.toml

@@ -80,6 +80,7 @@ sp-statement-store = { workspace = true, default-features = true }
 sp-storage = { workspace = true, default-features = true }
 sp-timestamp = { workspace = true, default-features = true }
 sp-transaction-pool = { workspace = true, default-features = true }
+sp-transaction-storage-proof = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }
 sp-weights = { workspace = true, default-features = true }
 substrate-frame-rpc-system = { workspace = true, default-features = true }

cumulus/polkadot-omni-node/lib/src/cli.rs

@@ -223,6 +223,10 @@ pub struct Cli<Config: CliConfig> {
 	#[arg(long)]
 	pub enable_statement_store: bool,
 
+	/// Enable the storage inherent data provider.
+	#[arg(long)]
+	pub enable_tx_storage_idp: bool,
+
 	#[arg(skip)]
 	pub(crate) _phantom: PhantomData<Config>,
 }
@@ -269,6 +273,7 @@ impl<Config: CliConfig> Cli<Config> {
 			max_pov_percentage: self.run.experimental_max_pov_percentage,
 			enable_statement_store: self.enable_statement_store,
 			storage_monitor: self.storage_monitor.clone(),
+			enable_tx_storage_idp: self.enable_tx_storage_idp,
 		}
 	}
 

cumulus/polkadot-omni-node/lib/src/common/mod.rs

@@ -129,4 +129,7 @@ pub struct NodeExtraArgs {
 
 	/// Parameters for storage monitoring.
 	pub storage_monitor: sc_storage_monitor::StorageMonitorParams,
+
+	/// Enable transaction storage inherent data provider.
+	pub enable_tx_storage_idp: bool,
 }

cumulus/polkadot-omni-node/lib/src/common/spec.rs

@@ -39,7 +39,7 @@ use cumulus_client_service::{
 use cumulus_primitives_core::{BlockT, GetParachainInfo, ParaId};
 use cumulus_relay_chain_interface::{OverseerHandle, RelayChainInterface};
 use futures::FutureExt;
-use log::info;
+use log::{debug, info};
 use parachains_common_types::Hash;
 use polkadot_primitives::CollatorPair;
 use prometheus_endpoint::Registry;
@@ -61,6 +61,13 @@ use sp_keystore::KeystorePtr;
 use sp_runtime::traits::AccountIdConversion;
 use std::{future::Future, pin::Pin, sync::Arc, time::Duration};
 
+// Override default idle connection timeout of 10 seconds to give IPFS clients more
+// time to query data over Bitswap. This is needed when manually adding our node
+// to a swarm of an IPFS node, because the IPFS node doesn't keep any active
+// substreams with us and our node closes a connection after
+// `idle_connection_timeout`.
+const IPFS_WORKAROUND_TIMEOUT: Duration = Duration::from_secs(3600);
+
 pub(crate) trait BuildImportQueue<
 	Block: BlockT,
 	RuntimeApi,
@@ -326,7 +333,16 @@ pub(crate) trait NodeSpec: BaseNodeSpec {
 		Net: NetworkBackend<Self::Block, Hash>,
 	{
 		let fut = async move {
-			let parachain_config = prepare_node_config(parachain_config);
+			let mut parachain_config = prepare_node_config(parachain_config);
+
+			// Some additional customization in relation to starting the node as an ipfs server.
+			if parachain_config.network.idle_connection_timeout < IPFS_WORKAROUND_TIMEOUT &&
+				parachain_config.network.ipfs_server
+			{
+				debug!("Overriding `config.network.idle_connection_timeout` to allow long-lived connections with IPFS nodes. The old value: {:?} is replaced by: {:?}.", parachain_config.network.idle_connection_timeout, IPFS_WORKAROUND_TIMEOUT);
+				parachain_config.network.idle_connection_timeout = IPFS_WORKAROUND_TIMEOUT;
+			}
+
 			let parachain_public_addresses = parachain_config.network.public_addresses.clone();
 			let parachain_fork_id = parachain_config.chain_spec.fork_id().map(ToString::to_string);
 			let advertise_non_global_ips = parachain_config.network.allow_non_globals_in_dht;

cumulus/polkadot-omni-node/lib/src/nodes/aura.rs

@@ -591,8 +591,25 @@ where
 		);
 
 		let client_for_aura = client.clone();
+		let enable_tx_storage_idp = node_extra_args.enable_tx_storage_idp;
+		let client_clone = client.clone();
 		let params = SlotBasedParams {
-			create_inherent_data_providers: move |_, ()| async move { Ok(()) },
+			create_inherent_data_providers: move |parent, ()| {
+				let client_clone = client_clone.clone();
+				async move {
+					if enable_tx_storage_idp {
+						let storage_proof =
+							sp_transaction_storage_proof::registration::new_data_provider(
+								&*client_clone,
+								&parent,
+								sp_transaction_storage_proof::runtime_api::client::retrieve_storage_period(&client_clone, parent)?,
+							)?;
+						Ok(vec![storage_proof])
+					} else {
+						Ok(vec![])
+					}
+				}
+			},
 			block_import,
 			para_client: client.clone(),
 			para_backend: backend.clone(),
@@ -618,7 +635,6 @@ where
 
 		// We have a separate function only to be able to use `docify::export` on this piece of
 		// code.
-
 		Self::launch_slot_based_collator(params);
 
 		Ok(())
@@ -716,10 +732,27 @@ where
 			client.clone(),
 		);
 
+		let enable_tx_storage_idp = node_extra_args.enable_tx_storage_idp;
+		let client_clone = client.clone();
 		let params = aura::ParamsWithExport {
 			export_pov: node_extra_args.export_pov,
 			params: AuraParams {
-				create_inherent_data_providers: move |_, ()| async move { Ok(()) },
+				create_inherent_data_providers: move |parent, ()| {
+					let client_clone = client_clone.clone();
+					async move {
+						if enable_tx_storage_idp {
+							let storage_proof =
+								sp_transaction_storage_proof::registration::new_data_provider(
+									&*client_clone,
+									&parent,
+									sp_transaction_storage_proof::runtime_api::client::retrieve_storage_period(&client_clone, parent)?,
+								)?;
+							Ok(vec![storage_proof])
+						} else {
+							Ok(vec![])
+						}
+					}
+				},
 				block_import,
 				para_client: client.clone(),
 				para_backend: backend,

prdoc/pr_10662.prdoc

@@ -0,0 +1,29 @@
+title: "Bulletin as parachain missing features"
+doc:
+- audience: Runtime Dev
+  description: |-
+    This PR adds the required support and features for running Bulletin as a parachain. It is a top-level PR that merges three partial features:
+
+    1. Add transaction_index::HostFunctions with NO-OP impl to the cumulus ParachainSystem validate_block for polkadot-prepare/execute-worker
+    2. Add custom inherent provider for pallet-transaction-storage to omni node
+    3. Configurable StoragePeriod feeded to the inherent provider over runtime API
+
+    This PR also refactors `pallet-transaction-storage` and `sp-transaction-storage-proof` (the `new_data_provider` inherent provider), both of which rely on a hard-coded `DEFAULT_STORAGE_PERIOD`. This PR:
+    - adds a new configurable argument `storage_period` to the `new_data_provider`
+    - introduces the `TransactionStorageApi::storage_period` runtime API, which the runtime can specify arbitrary
+    - provides an example of using `new_data_provider`, with the node client calling the runtime API when constructing inherent provider data
+- audience: [ Node Dev, Node Operator ]
+  description: |
+    * Node developers/operators could enable the transaction storage inherent data provider setup by using --enable-tx-storage-idp flag. This is especially useful in the context of bulletin chain.
+    * Node developers will set up the network `idle_connection_timeout` to 1h when using `--ipfs-server` flag, again, useful in the context of bulletin chain.
+crates:
+- name: cumulus-pallet-parachain-system
+  bump: patch
+- name: polkadot-omni-node-lib
+  bump: major
+- name: sp-inherents
+  bump: minor
+- name: sp-transaction-storage-proof
+  bump: major
+- name: pallet-transaction-storage
+  bump: minor

substrate/bin/node/cli/src/service.rs

@@ -627,11 +627,11 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
 							slot_duration,
 						);
 
-					let storage_proof =
-						sp_transaction_storage_proof::registration::new_data_provider(
-							&*client_clone,
-							&parent,
-						)?;
+					let storage_proof = sp_transaction_storage_proof::registration::new_data_provider(
+						&*client_clone,
+						&parent,
+						sp_transaction_storage_proof::runtime_api::client::retrieve_storage_period(&client_clone, parent)?,
+					)?;
 
 					Ok((slot, timestamp, storage_proof))
 				}

substrate/bin/node/runtime/src/lib.rs

@@ -3776,6 +3776,12 @@ pallet_revive::impl_runtime_apis_plus_revive_traits!(
 		}
 	}
 
+	impl sp_transaction_storage_proof::runtime_api::TransactionStorageApi<Block> for Runtime {
+		fn storage_period() -> NumberFor<Block> {
+			TransactionStorage::storage_period()
+		}
+	}
+
 	#[cfg(feature = "try-runtime")]
 	impl frame_try_runtime::TryRuntime<Block> for Runtime {
 		fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) {

substrate/frame/transaction-storage/Cargo.toml

@@ -46,6 +46,7 @@ std = [
 	"sp-inherents/std",
 	"sp-io/std",
 	"sp-runtime/std",
+	"sp-transaction-storage-proof/std",
 	"tracing/std",
 ]
 runtime-benchmarks = [

substrate/frame/transaction-storage/src/benchmarking.rs

@@ -169,7 +169,7 @@ mod benchmarks {
 				vec![0u8; T::MaxTransactionSize::get() as usize],
 			)?;
 		}
-		run_to_block::<T>(StoragePeriod::<T>::get() + BlockNumberFor::<T>::one());
+		run_to_block::<T>(crate::Pallet::<T>::storage_period() + BlockNumberFor::<T>::one());
 		let encoded_proof = proof();
 		let proof = TransactionStorageProof::decode(&mut &*encoded_proof).unwrap();
 

substrate/frame/transaction-storage/src/lib.rs

@@ -57,6 +57,9 @@ pub type CreditOf<T> = Credit<<T as frame_system::Config>::AccountId, <T as Conf
 pub use pallet::*;
 pub use weights::WeightInfo;
 
+/// Default storage period for data (in blocks).
+pub const DEFAULT_STORAGE_PERIOD: u32 = 100800;
+
 // TODO: https://github.com/paritytech/polkadot-bulletin-chain/issues/139 - Clarify purpose of allocator limits and decide whether to remove or use these constants.
 /// Maximum bytes that can be stored in one transaction.
 // Setting higher limit also requires raising the allocator limit.
@@ -218,7 +221,7 @@ pub mod pallet {
 			// Drop obsolete roots. The proof for `obsolete` will be checked later
 			// in this block, so we drop `obsolete` - 1.
 			weight.saturating_accrue(db_weight.reads(1));
-			let period = StoragePeriod::<T>::get();
+			let period = Self::storage_period();
 			let obsolete = n.saturating_sub(period.saturating_add(One::one()));
 			if obsolete > Zero::zero() {
 				weight.saturating_accrue(db_weight.writes(1));
@@ -235,7 +238,7 @@ pub mod pallet {
 				ProofChecked::<T>::take() || {
 					// Proof is not required for early or empty blocks.
 					let number = frame_system::Pallet::<T>::block_number();
-					let period = StoragePeriod::<T>::get();
+					let period = Self::storage_period();
 					let target_number = number.saturating_sub(period);
 
 					target_number.is_zero() || {
@@ -264,7 +267,7 @@ pub mod pallet {
 				!T::MaxTransactionSize::get().is_zero(),
 				"MaxTransactionSize must be greater than zero"
 			);
-			let default_period = sp_transaction_storage_proof::DEFAULT_STORAGE_PERIOD.into();
+			let default_period = DEFAULT_STORAGE_PERIOD.into();
 			let storage_period = GenesisConfig::<T>::default().storage_period;
 			assert_eq!(
 				storage_period, default_period,
@@ -394,7 +397,7 @@ pub mod pallet {
 
 			// Get the target block metadata.
 			let number = frame_system::Pallet::<T>::block_number();
-			let period = StoragePeriod::<T>::get();
+			let period = Self::storage_period();
 			let target_number = number.saturating_sub(period);
 			ensure!(!target_number.is_zero(), Error::<T>::UnexpectedProof);
 			let transactions =
@@ -456,8 +459,7 @@ pub mod pallet {
 	/// Storage fee per transaction.
 	pub type EntryFee<T: Config> = StorageValue<_, BalanceOf<T>>;
 
-	/// Storage period for data in blocks. Should match `sp_storage_proof::DEFAULT_STORAGE_PERIOD`
-	/// for block authoring.
+	/// Storage period for data in blocks.
 	#[pallet::storage]
 	pub type StoragePeriod<T: Config> = StorageValue<_, BlockNumberFor<T>, ValueQuery>;
 
@@ -482,7 +484,7 @@ pub mod pallet {
 			Self {
 				byte_fee: 10u32.into(),
 				entry_fee: 1000u32.into(),
-				storage_period: sp_transaction_storage_proof::DEFAULT_STORAGE_PERIOD.into(),
+				storage_period: DEFAULT_STORAGE_PERIOD.into(),
 			}
 		}
 	}
@@ -525,14 +527,18 @@ pub mod pallet {
 		) -> Option<BoundedVec<TransactionInfo, T::MaxBlockTransactions>> {
 			Transactions::<T>::get(block)
 		}
-		/// Get ByteFee storage information from outside of this pallet.
+		/// Get ByteFee storage information from the outside of this pallet.
 		pub fn byte_fee() -> Option<BalanceOf<T>> {
 			ByteFee::<T>::get()
 		}
-		/// Get EntryFee storage information from outside of this pallet.
+		/// Get EntryFee storage information from the outside of this pallet.
 		pub fn entry_fee() -> Option<BalanceOf<T>> {
 			EntryFee::<T>::get()
 		}
+		/// Get StoragePeriod storage information from the outside of this pallet.
+		pub fn storage_period() -> BlockNumberFor<T> {
+			StoragePeriod::<T>::get()
+		}
 
 		fn apply_fee(sender: T::AccountId, size: u32) -> DispatchResult {
 			let byte_fee = ByteFee::<T>::get().ok_or(Error::<T>::NotConfigured)?;