Bulletin as parachain missing features

cumulus/pallets/parachain-system/Cargo.toml

@@ -125,3 +125,13 @@ try-runtime = [
 	"polkadot-runtime-parachains/try-runtime",
 	"sp-runtime/try-runtime",
 ]
+
+# Enables `TransactionIndex` host functions for parachain validation
+# without requiring maintenance of a transaction index.
+# However, the host environment (polkadot-prepare/execute-worker) still expects these functions to exist,
+# so we provide a replacement no-op implementation gated by this feature.
+#
+# Used by: `validate_block::implementation::validate_block`.
+#
+# Note: The parachain collator software is expected to use/include `transaction_index::HostFunctions`.
+transaction-index = []

cumulus/pallets/parachain-system/src/validate_block/implementation.rs

@@ -122,6 +122,10 @@ where
 		sp_io::offchain_index::host_clear.replace_implementation(host_offchain_index_clear),
 		cumulus_primitives_proof_size_hostfunction::storage_proof_size::host_storage_proof_size
 			.replace_implementation(host_storage_proof_size),
+		#[cfg(feature = "transaction-index")]
+		sp_io::transaction_index::host_index.replace_implementation(host_transaction_index_index),
+		#[cfg(feature = "transaction-index")]
+		sp_io::transaction_index::host_renew.replace_implementation(host_transaction_index_renew),
 	);
 
 	let block_data = codec::decode_from_bytes::<ParachainBlockData<B::LazyBlock>>(block_data)
@@ -230,7 +234,6 @@ where
 		if overlay.storage(well_known_keys::CODE).is_some() && num_blocks > 1 {
 			panic!("When applying a runtime upgrade, only one block per PoV is allowed. Received {num_blocks}.")
 		}
-
 		run_with_externalities_and_recorder::<B, _, _>(
 			&backend,
 			&mut Default::default(),
@@ -548,3 +551,21 @@ fn host_default_child_storage_next_key(storage_key: &[u8], key: &[u8]) -> Option
 fn host_offchain_index_set(_key: &[u8], _value: &[u8]) {}
 
 fn host_offchain_index_clear(_key: &[u8]) {}
+
+/// Parachain validation does not require maintaining a transaction index,
+/// and indexing transactions does **not** contribute to the parachain state.
+/// However, the host environment still expects this function to exist,
+/// so we provide a no-op implementation.
+#[cfg(feature = "transaction-index")]
+fn host_transaction_index_index(_extrinsic: u32, _size: u32, _context_hash: [u8; 32]) {
+	// No-op host function used during parachain validation.
+}
+
+/// Parachain validation does not require maintaining a transaction index,
+/// and indexing transactions does **not** contribute to the parachain state.
+/// However, the host environment still expects this function to exist,
+/// so we provide a no-op implementation.
+#[cfg(feature = "transaction-index")]
+fn host_transaction_index_renew(_extrinsic: u32, _context_hash: [u8; 32]) {
+	// No-op host function used during parachain validation.
+}

cumulus/polkadot-omni-node/lib/Cargo.toml

@@ -80,6 +80,7 @@ sp-statement-store = { workspace = true, default-features = true }
 sp-storage = { workspace = true, default-features = true }
 sp-timestamp = { workspace = true, default-features = true }
 sp-transaction-pool = { workspace = true, default-features = true }
+sp-transaction-storage-proof = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }
 sp-weights = { workspace = true, default-features = true }
 substrate-frame-rpc-system = { workspace = true, default-features = true }

cumulus/polkadot-omni-node/lib/src/common/mod.rs

@@ -42,6 +42,7 @@ use sp_runtime::{
 use sp_session::SessionKeys;
 use sp_statement_store::runtime_api::ValidateStatement;
 use sp_transaction_pool::runtime_api::TaggedTransactionQueue;
+use sp_transaction_storage_proof::runtime_api::TransactionStorageApi;
 use std::{fmt::Debug, path::PathBuf, str::FromStr};
 
 pub trait NodeBlock:
@@ -74,6 +75,7 @@ pub trait NodeRuntimeApi<Block: BlockT>:
 	+ CollectCollationInfo<Block>
 	+ ValidateStatement<Block>
 	+ GetParachainInfo<Block>
+	+ TransactionStorageApi<Block>
 	+ RelayParentOffsetApi<Block>
 	+ Sized
 {
@@ -90,6 +92,7 @@ impl<T, Block: BlockT> NodeRuntimeApi<Block> for T where
 		+ CollectCollationInfo<Block>
 		+ ValidateStatement<Block>
 		+ GetParachainInfo<Block>
+		+ TransactionStorageApi<Block>
 {
 }
 

cumulus/polkadot-omni-node/lib/src/common/spec.rs

@@ -39,7 +39,7 @@ use cumulus_client_service::{
 use cumulus_primitives_core::{BlockT, GetParachainInfo, ParaId};
 use cumulus_relay_chain_interface::{OverseerHandle, RelayChainInterface};
 use futures::FutureExt;
-use log::info;
+use log::{debug, info};
 use parachains_common_types::Hash;
 use polkadot_primitives::CollatorPair;
 use prometheus_endpoint::Registry;
@@ -61,6 +61,13 @@ use sp_keystore::KeystorePtr;
 use sp_runtime::traits::AccountIdConversion;
 use std::{future::Future, pin::Pin, sync::Arc, time::Duration};
 
+// Override default idle connection timeout of 10 seconds to give IPFS clients more
+// time to query data over Bitswap. This is needed when manually adding our node
+// to a swarm of an IPFS node, because the IPFS node doesn't keep any active
+// substreams with us and our node closes a connection after
+// `idle_connection_timeout`.
+const IPFS_WORKAROUND_TIMEOUT: Duration = Duration::from_secs(3600);
+
 pub(crate) trait BuildImportQueue<
 	Block: BlockT,
 	RuntimeApi,
@@ -326,7 +333,16 @@ pub(crate) trait NodeSpec: BaseNodeSpec {
 		Net: NetworkBackend<Self::Block, Hash>,
 	{
 		let fut = async move {
-			let parachain_config = prepare_node_config(parachain_config);
+			let mut parachain_config = prepare_node_config(parachain_config);
+
+			// Some additional customization in relation to starting the node as an ipfs server.
+			if parachain_config.network.idle_connection_timeout < IPFS_WORKAROUND_TIMEOUT &&
+				parachain_config.network.ipfs_server
+			{
+				debug!("Overriding `config.network.idle_connection_timeout` to allow long-lived connections with IPFS nodes. The old value: {:?} is replaced by: {:?}.", parachain_config.network.idle_connection_timeout, IPFS_WORKAROUND_TIMEOUT);
+				parachain_config.network.idle_connection_timeout = IPFS_WORKAROUND_TIMEOUT;
+			}
+
 			let parachain_public_addresses = parachain_config.network.public_addresses.clone();
 			let parachain_fork_id = parachain_config.chain_spec.fork_id().map(ToString::to_string);
 			let advertise_non_global_ips = parachain_config.network.allow_non_globals_in_dht;

cumulus/polkadot-omni-node/lib/src/fake_runtime_api/utils.rs

@@ -248,6 +248,12 @@ macro_rules! impl_node_runtime_apis {
 					unimplemented!()
 				}
 			}
+
+			impl sp_transaction_storage_proof::runtime_api::TransactionStorageApi<$block> for $runtime {
+				fn retention_period() -> sp_runtime::traits::NumberFor<$block> {
+					unimplemented!()
+				}
+			}
 		}
 	};
 }

cumulus/polkadot-omni-node/lib/src/nodes/aura.rs

@@ -68,7 +68,7 @@ use sc_service::{Configuration, Error, PartialComponents, TaskManager};
 use sc_telemetry::TelemetryHandle;
 use sc_transaction_pool::TransactionPoolHandle;
 use sc_transaction_pool_api::OffchainTransactionPoolFactory;
-use sp_api::ProvideRuntimeApi;
+use sp_api::{ApiExt, ProvideRuntimeApi};
 use sp_consensus::Environment;
 use sp_core::traits::SpawnEssentialNamed;
 use sp_inherents::CreateInherentDataProviders;
@@ -77,6 +77,7 @@ use sp_runtime::{
 	app_crypto::AppCrypto,
 	traits::{Block as BlockT, Header as HeaderT, UniqueSaturatedInto},
 };
+use sp_transaction_storage_proof::runtime_api::TransactionStorageApi;
 use std::{marker::PhantomData, sync::Arc, time::Duration};
 
 struct Verifier<Block, Client, AuraId> {
@@ -596,8 +597,28 @@ where
 		);
 
 		let client_for_aura = client.clone();
+		let client_clone = client.clone();
 		let params = SlotBasedParams {
-			create_inherent_data_providers: move |_, ()| async move { Ok(()) },
+			create_inherent_data_providers: move |parent, ()| {
+				let client_clone = client_clone.clone();
+				async move {
+					let has_tx_storage_api = client_clone
+						.runtime_api()
+						.has_api::<dyn TransactionStorageApi<Block>>(parent)
+						.unwrap_or(false);
+					if has_tx_storage_api {
+						let storage_proof =
+							sp_transaction_storage_proof::registration::new_data_provider(
+								&*client_clone,
+								&parent,
+								client_clone.runtime_api().retention_period(parent)?,
+							)?;
+						Ok(vec![storage_proof])
+					} else {
+						Ok(vec![])
+					}
+				}
+			},
 			block_import,
 			para_client: client.clone(),
 			para_backend: backend.clone(),
@@ -623,7 +644,6 @@ where
 
 		// We have a separate function only to be able to use `docify::export` on this piece of
 		// code.
-
 		Self::launch_slot_based_collator(params);
 
 		Ok(())
@@ -721,10 +741,30 @@ where
 			client.clone(),
 		);
 
+		let client_clone = client.clone();
 		let params = aura::ParamsWithExport {
 			export_pov: node_extra_args.export_pov,
 			params: AuraParams {
-				create_inherent_data_providers: move |_, ()| async move { Ok(()) },
+				create_inherent_data_providers: move |parent, ()| {
+					let client_clone = client_clone.clone();
+					async move {
+						let has_tx_storage_api = client_clone
+							.runtime_api()
+							.has_api::<dyn TransactionStorageApi<Block>>(parent)
+							.unwrap_or(false);
+						if has_tx_storage_api {
+							let storage_proof =
+								sp_transaction_storage_proof::registration::new_data_provider(
+									&*client_clone,
+									&parent,
+									client_clone.runtime_api().retention_period(parent)?,
+								)?;
+							Ok(vec![storage_proof])
+						} else {
+							Ok(vec![])
+						}
+					}
+				},
 				block_import,
 				para_client: client.clone(),
 				para_backend: backend,

prdoc/pr_10662.prdoc

@@ -0,0 +1,29 @@
+title: "Bulletin as parachain missing features"
+doc:
+- audience: Runtime Dev
+  description: |-
+    This PR adds the required support and features for running Bulletin as a parachain. It is a top-level PR that merges three partial features:
+
+    1. Add transaction_index::HostFunctions with NO-OP impl to the cumulus ParachainSystem validate_block for polkadot-prepare/execute-worker
+    2. Add custom inherent provider for pallet-transaction-storage to omni node
+    3. Configurable RetentionPeriod feeded to the inherent provider over runtime API
+
+    This PR also refactors `pallet-transaction-storage` and `sp-transaction-storage-proof` (the `new_data_provider` inherent provider), both of which rely on a hard-coded `DEFAULT_RETENTION_PERIOD`. This PR:
+    - adds a new configurable argument `retention_period` to the `new_data_provider`
+    - introduces the `TransactionStorageApi::retention_period` runtime API, which the runtime can specify arbitrary
+    - provides an example of using `new_data_provider`, with the node client calling the runtime API when constructing inherent provider data
+- audience: [ Node Dev, Node Operator ]
+  description: |
+    * Node developers/operators could enable the transaction storage inherent data provider setup by using --enable-tx-storage-idp flag. This is especially useful in the context of bulletin chain.
+    * Node developers will set up the network `idle_connection_timeout` to 1h when using `--ipfs-server` flag, again, useful in the context of bulletin chain.
+crates:
+- name: cumulus-pallet-parachain-system
+  bump: patch
+- name: polkadot-omni-node-lib
+  bump: major
+- name: sp-inherents
+  bump: minor
+- name: sp-transaction-storage-proof
+  bump: major
+- name: pallet-transaction-storage
+  bump: major

substrate/bin/node/cli/src/service.rs

@@ -47,6 +47,7 @@ use sc_transaction_pool_api::OffchainTransactionPoolFactory;
 use sp_api::ProvideRuntimeApi;
 use sp_core::crypto::Pair;
 use sp_runtime::{generic, traits::Block as BlockT, SaturatedConversion};
+use sp_transaction_storage_proof::runtime_api::TransactionStorageApi;
 use std::{path::Path, sync::Arc};
 
 /// Host functions required for kitchensink runtime and Substrate node.
@@ -631,6 +632,8 @@ pub fn new_full_base<N: NetworkBackend<Block, <Block as BlockT>::Hash>>(
 						sp_transaction_storage_proof::registration::new_data_provider(
 							&*client_clone,
 							&parent,
+							// Use `unwrap_or` in case the runtime api is not available.
+							client_clone.runtime_api().retention_period(parent).unwrap_or(100800),
 						)?;
 
 					Ok((slot, timestamp, storage_proof))

substrate/bin/node/cli/tests/res/default_genesis_config.json

@@ -100,7 +100,7 @@
   "transactionStorage": {
     "byteFee": 10,
     "entryFee": 1000,
-    "storagePeriod": 100800
+    "retentionPeriod": 100800
   },
   "allianceMotion": {
     "members": []

substrate/bin/node/runtime/src/lib.rs

@@ -3776,6 +3776,12 @@ pallet_revive::impl_runtime_apis_plus_revive_traits!(
 		}
 	}
 
+	impl sp_transaction_storage_proof::runtime_api::TransactionStorageApi<Block> for Runtime {
+		fn retention_period() -> NumberFor<Block> {
+			TransactionStorage::retention_period()
+		}
+	}
+
 	#[cfg(feature = "try-runtime")]
 	impl frame_try_runtime::TryRuntime<Block> for Runtime {
 		fn on_runtime_upgrade(checks: frame_try_runtime::UpgradeCheckSelect) -> (Weight, Weight) {

substrate/frame/transaction-storage/Cargo.toml

@@ -46,6 +46,7 @@ std = [
 	"sp-inherents/std",
 	"sp-io/std",
 	"sp-runtime/std",
+	"sp-transaction-storage-proof/std",
 	"tracing/std",
 ]
 runtime-benchmarks = [

substrate/frame/transaction-storage/README.md

@@ -2,8 +2,8 @@
 
 Indexes transactions and manages storage proofs.
 
-Allows storing arbitrary data on the chain. Data is automatically removed after `StoragePeriod` blocks, unless the
-storage is renewed. Validators must submit proof of storing a random chunk of data for block `N - StoragePeriod` when
+Allows storing arbitrary data on the chain. Data is automatically removed after `RetentionPeriod` blocks, unless the
+storage is renewed. Validators must submit proof of storing a random chunk of data for block `N - RetentionPeriod` when
 producing block `N`.
 
 # Running a chain
@@ -17,7 +17,7 @@ cargo run --release -- build-spec --chain=local > sc_init.json
 ```
 
 Edit the json chain spec file to customise the chain. The storage chain genesis params are configured in the
-`transactionStorage` section. Note that `storagePeriod` is specified in blocks and changing it also requires code
+`transactionStorage` section. Note that `retentionPeriod` is specified in blocks and changing it also requires code
 changes at the moment.
 
 Build a raw spec from the init spec.
@@ -34,7 +34,7 @@ cargo run --release -- --chain=sc.json -d /tmp/bob --storage-chain --keep-blocks
 ```
 
 `--storage-chain` enables transaction indexing. `--keep-blocks=100800` enables block pruning. The value here should be
-greater or equal than the storage period. `--ipfs-server` enables serving stored content over IPFS.
+greater or equal than the retention period. `--ipfs-server` enables serving stored content over IPFS.
 
 Once the network is started, any other joining nodes need to sync with `--sync=fast`. Regular sync will fail because
 block pruning removes old blocks. The chain does not keep full block history.
@@ -78,7 +78,7 @@ ipfs swarm connect <substrate peer address>
 ipfs block get /ipfs/<CID> > kitten.jpeg
 ```
 
-To renew data and prevent it from being disposed after the storage period, use `transactionStorage.renew(block, index)`
+To renew data and prevent it from being disposed after the retention period, use `transactionStorage.renew(block, index)`
 where `block` is the block number of the previous store or renew transaction, and index is the index of that transaction
 in the block.
 

substrate/frame/transaction-storage/src/benchmarking.rs

@@ -169,7 +169,7 @@ mod benchmarks {
 				vec![0u8; T::MaxTransactionSize::get() as usize],
 			)?;
 		}
-		run_to_block::<T>(StoragePeriod::<T>::get() + BlockNumberFor::<T>::one());
+		run_to_block::<T>(crate::Pallet::<T>::retention_period() + BlockNumberFor::<T>::one());
 		let encoded_proof = proof();
 		let proof = TransactionStorageProof::decode(&mut &*encoded_proof).unwrap();