Adds --exclude-extrinsics support for frame-omni-bencher

prdoc/pr_8257.prdoc

@@ -0,0 +1,9 @@
+title: Adds `--exclude-extrinsics` support for frame-omni-bencher
+doc:
+- audience: Runtime Dev
+  description: null
+crates:
+- name: frame-benchmarking-cli
+  bump: patch
+- name: frame-support-procedural
+  bump: patch

substrate/frame/support/procedural/src/benchmark.rs

@@ -774,19 +774,15 @@ pub fn benchmarks(
 						#krate::benchmarking::reset_read_write_count();
 					};
 
+					let root = #krate::__private::sp_io::storage::root(#krate::__private::sp_runtime::StateVersion::V1);
+					#krate::__private::log::trace!(
+						target: "benchmark",
+						"Root: {:?}",
+						root
+					);
+
 					// Always do at least one internal repeat...
 					for _ in 0 .. internal_repeats.max(1) {
-						// Always reset the state after the benchmark.
-						#krate::__private::defer!(#krate::benchmarking::wipe_db());
-
-						// Time the extrinsic logic.
-						#krate::__private::log::trace!(
-							target: "benchmark",
-							"Start Benchmark: {} ({:?})",
-							extrinsic,
-							c
-						);
-
 						let mut recording = #krate::BenchmarkRecording::new(&on_before_start);
 						<SelectedBenchmark as #krate::BenchmarkingSetup<#type_use_generics>>::instance(&selected_benchmark, &mut recording, c, verify)?;
 

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -556,19 +556,11 @@ impl PalletCmd {
 	}
 
 	fn select_benchmarks_to_run(&self, list: Vec<BenchmarkList>) -> Result<Vec<SelectedBenchmark>> {
-		let extrinsic = self.extrinsic.clone().unwrap_or_default();
-		let extrinsic_split: Vec<&str> = extrinsic.split(',').collect();
-		let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
-
 		// Use the benchmark list and the user input to determine the set of benchmarks to run.
 		let mut benchmarks_to_run = Vec::new();
 		list.iter().filter(|item| self.pallet_selected(&item.pallet)).for_each(|item| {
 			for benchmark in &item.benchmarks {
-				let benchmark_name = &benchmark.name;
-				if extrinsic.is_empty() ||
-					extrinsic.as_bytes() == &b"*"[..] ||
-					extrinsics.contains(&&benchmark_name[..])
-				{
+				if self.extrinsic_selected(&item.pallet, &benchmark.name) {
 					benchmarks_to_run.push((
 						item.pallet.clone(),
 						benchmark.name.clone(),
@@ -609,14 +601,39 @@ impl PalletCmd {
 
 	/// Whether this pallet should be run.
 	fn pallet_selected(&self, pallet: &Vec<u8>) -> bool {
-		let include = self.pallet.clone().unwrap_or_default();
+		let include = self.pallets.clone();
 
-		let included = include.is_empty() || include == "*" || include.as_bytes() == pallet;
+		let included = include.is_empty()
+			|| include.iter().any(|p| p.as_bytes() == pallet)
+			|| include.iter().any(|p| p == "*");
 		let excluded = self.exclude_pallets.iter().any(|p| p.as_bytes() == pallet);
 
 		included && !excluded
 	}
 
+	fn extrinsic_selected(&self, pallet: &Vec<u8>, extrinsic: &Vec<u8>) -> bool {
+		if !self.pallet_selected(pallet) {
+			return false;
+		}
+
+		let extrinsic_filter = self.extrinsic.clone().unwrap_or_default();
+		let extrinsic_split: Vec<&str> = extrinsic_filter.split(',').collect();
+		let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
+
+		let included = extrinsic_filter.is_empty() || extrinsic_filter == "*" || 
+			extrinsics.contains(&&extrinsic[..]);
+		let excluded = self.exclude_extrinsics.iter().any(|e| {
+			let splits = e.split("::").collect::<Vec<_>>();
+			if splits.len() != 2 {
+				panic!("Invalid argument for '--exclude-extrinsic'. Expected format: 'Pallet::Extrinsic' but got '{}'", e);
+			}
+			let (p, e) = (splits[0], splits[1]);
+			p.as_bytes() == pallet && e.as_bytes() == extrinsic
+		});
+
+		included && !excluded
+	}
+
 	/// Execute a state machine and decode its return value as `R`.
 	fn exec_state_machine<R: Decode, H: Hash, Exec: CodeExecutor>(
 		mut machine: StateMachine<BenchmarkingState<H>, H, Exec>,

substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs

@@ -46,9 +46,9 @@ pub enum ListOutput {
 /// Benchmark the extrinsic weight of FRAME Pallets.
 #[derive(Debug, clap::Parser)]
 pub struct PalletCmd {
-	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
-	#[arg(short, long, value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
-	pub pallet: Option<String>,
+	/// Select a FRAME Pallets to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
+	#[arg(short, long, alias = "pallet", num_args = 1.., value_delimiter = ',', value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
+	pub pallets: Vec<String>,
 
 	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
 	#[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
@@ -58,6 +58,12 @@ pub struct PalletCmd {
 	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
 	pub exclude_pallets: Vec<String>,
 
+	/// Comma separated list of `pallet::extrinsic` combinations that should not be run.
+	///
+	/// Example: `frame_system::remark,pallet_balances::transfer_keep_alive`
+	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
+	pub exclude_extrinsics: Vec<String>,
+
 	/// Run benchmarks for all pallets and extrinsics.
 	///
 	/// This is equivalent to running `--pallet * --extrinsic *`.