Adds --exclude-extrinsics support for frame-omni-bencher

.github/scripts/release/build-linux-release.sh

@@ -3,7 +3,7 @@
 # This is used to build our binaries:
 # - polkadot
 # - polkadot-parachain
-# - polkadot-omni-node 
+# - polkadot-omni-node
 #
 # set -e
 
@@ -12,7 +12,6 @@ PACKAGE=${2:-$BIN}
 
 PROFILE=${PROFILE:-production}
 ARTIFACTS=/artifacts/$BIN
-VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
 
 echo "Artifacts will be copied into $ARTIFACTS"
 mkdir -p "$ARTIFACTS"
@@ -25,10 +24,10 @@ echo "Artifact target: $ARTIFACTS"
 cp ./target/$PROFILE/$BIN "$ARTIFACTS"
 pushd "$ARTIFACTS" > /dev/null
 sha256sum "$BIN" | tee "$BIN.sha256"
-
-EXTRATAG="$($ARTIFACTS/$BIN --version |
+chmod a+x "$BIN"
+VERSION="$($ARTIFACTS/$BIN --version)"
+EXTRATAG="$(echo "${VERSION}" |
     sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
-
 EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
 
 echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"

.github/workflows/release-build-binary.yml

@@ -0,0 +1,76 @@
+name: Binary Build
+# This workflow can be used to build a binary like polkadot + workers, omninode or polkadot-parachain
+# from any branch with release or profuction profile to be later used for testing.
+# ⚠️ IT should not be used for release purposes!
+
+on:
+  workflow_dispatch:
+    inputs:
+      binary:
+        required: true
+        default: "polkadot"
+        description: "The binary to build"
+      package:
+        description: Package to be built, can be polkadot, polkadot-parachain-bin, polkadot-omni-node etc.
+        required: true
+        type: string
+      profile:
+        required: true
+        default: "release"
+        description: "The profile to use for the binary build"
+
+jobs:
+
+  setup:
+    # GitHub Actions allows using 'env' in a container context.
+    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
+    # This workaround sets the container image for each job using 'set-image' job output.
+    runs-on: ubuntu-latest
+    outputs:
+      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+      RUNNER: ${{ steps.set_runner.outputs.RUNNER }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Set image
+        id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+
+      - name: Set runner
+        id: set_runner
+        shell: bash
+        run: |
+          if [[ "${{ inputs.binary }}" == "polkadot-parachain" ]]; then
+            echo "RUNNER=parity-large" >> $GITHUB_OUTPUT
+          else
+            echo "RUNNER=ubuntu-latest" >> $GITHUB_OUTPUT
+          fi
+
+  build:
+    needs: [setup]
+    runs-on: ${{ needs.setup.outputs.RUNNER }}
+    container:
+      image: ${{ needs.setup.outputs.IMAGE }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Build binary
+        run: |
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}" #avoid "detected dubious ownership" error
+          PROFILE=${{ inputs.profile }}
+          if [ "${{ inputs.binary }}" = "polkadot" ]; then
+              for binary in polkadot polkadot-prepare-worker polkadot-execute-worker; do
+                  echo "Building $binary..."
+                  ./.github/scripts/release/build-linux-release.sh $binary ${{ inputs.package }} "${PROFILE}"
+              done
+          else
+            ./.github/scripts/release/build-linux-release.sh ${{ inputs.binary }} ${{ inputs.package }} "${PROFILE}"
+          fi
+
+      - name: Upload ${{ inputs.binary }} artifacts
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: ${{ inputs.binary }}
+          path: /artifacts/**

prdoc/pr_8257.prdoc

@@ -0,0 +1,9 @@
+title: Adds `--exclude-extrinsics` support for frame-omni-bencher
+doc:
+- audience: Runtime Dev
+  description: null
+crates:
+- name: frame-benchmarking-cli
+  bump: patch
+- name: frame-support-procedural
+  bump: patch

substrate/frame/support/procedural/src/benchmark.rs

@@ -774,19 +774,15 @@ pub fn benchmarks(
 						#krate::benchmarking::reset_read_write_count();
 					};
 
+					let root = #krate::__private::sp_io::storage::root(#krate::__private::sp_runtime::StateVersion::V1);
+					#krate::__private::log::trace!(
+						target: "benchmark",
+						"Root: {:?}",
+						root
+					);
+
 					// Always do at least one internal repeat...
 					for _ in 0 .. internal_repeats.max(1) {
-						// Always reset the state after the benchmark.
-						#krate::__private::defer!(#krate::benchmarking::wipe_db());
-
-						// Time the extrinsic logic.
-						#krate::__private::log::trace!(
-							target: "benchmark",
-							"Start Benchmark: {} ({:?})",
-							extrinsic,
-							c
-						);
-
 						let mut recording = #krate::BenchmarkRecording::new(&on_before_start);
 						<SelectedBenchmark as #krate::BenchmarkingSetup<#type_use_generics>>::instance(&selected_benchmark, &mut recording, c, verify)?;
 

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -556,19 +556,11 @@ impl PalletCmd {
 	}
 
 	fn select_benchmarks_to_run(&self, list: Vec<BenchmarkList>) -> Result<Vec<SelectedBenchmark>> {
-		let extrinsic = self.extrinsic.clone().unwrap_or_default();
-		let extrinsic_split: Vec<&str> = extrinsic.split(',').collect();
-		let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
-
 		// Use the benchmark list and the user input to determine the set of benchmarks to run.
 		let mut benchmarks_to_run = Vec::new();
 		list.iter().filter(|item| self.pallet_selected(&item.pallet)).for_each(|item| {
 			for benchmark in &item.benchmarks {
-				let benchmark_name = &benchmark.name;
-				if extrinsic.is_empty() ||
-					extrinsic.as_bytes() == &b"*"[..] ||
-					extrinsics.contains(&&benchmark_name[..])
-				{
+				if self.extrinsic_selected(&item.pallet, &benchmark.name) {
 					benchmarks_to_run.push((
 						item.pallet.clone(),
 						benchmark.name.clone(),
@@ -609,14 +601,39 @@ impl PalletCmd {
 
 	/// Whether this pallet should be run.
 	fn pallet_selected(&self, pallet: &Vec<u8>) -> bool {
-		let include = self.pallet.clone().unwrap_or_default();
+		let include = self.pallets.clone();
 
-		let included = include.is_empty() || include == "*" || include.as_bytes() == pallet;
+		let included = include.is_empty()
+			|| include.iter().any(|p| p.as_bytes() == pallet)
+			|| include.iter().any(|p| p == "*");
 		let excluded = self.exclude_pallets.iter().any(|p| p.as_bytes() == pallet);
 
 		included && !excluded
 	}
 
+	fn extrinsic_selected(&self, pallet: &Vec<u8>, extrinsic: &Vec<u8>) -> bool {
+		if !self.pallet_selected(pallet) {
+			return false;
+		}
+
+		let extrinsic_filter = self.extrinsic.clone().unwrap_or_default();
+		let extrinsic_split: Vec<&str> = extrinsic_filter.split(',').collect();
+		let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
+
+		let included = extrinsic_filter.is_empty() || extrinsic_filter == "*" || 
+			extrinsics.contains(&&extrinsic[..]);
+		let excluded = self.exclude_extrinsics.iter().any(|e| {
+			let splits = e.split("::").collect::<Vec<_>>();
+			if splits.len() != 2 {
+				panic!("Invalid argument for '--exclude-extrinsic'. Expected format: 'Pallet::Extrinsic' but got '{}'", e);
+			}
+			let (p, e) = (splits[0], splits[1]);
+			p.as_bytes() == pallet && e.as_bytes() == extrinsic
+		});
+
+		included && !excluded
+	}
+
 	/// Execute a state machine and decode its return value as `R`.
 	fn exec_state_machine<R: Decode, H: Hash, Exec: CodeExecutor>(
 		mut machine: StateMachine<BenchmarkingState<H>, H, Exec>,

substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs

@@ -46,9 +46,9 @@ pub enum ListOutput {
 /// Benchmark the extrinsic weight of FRAME Pallets.
 #[derive(Debug, clap::Parser)]
 pub struct PalletCmd {
-	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
-	#[arg(short, long, value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
-	pub pallet: Option<String>,
+	/// Select a FRAME Pallets to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
+	#[arg(short, long, alias = "pallet", num_args = 1.., value_delimiter = ',', value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
+	pub pallets: Vec<String>,
 
 	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
 	#[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
@@ -58,6 +58,12 @@ pub struct PalletCmd {
 	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
 	pub exclude_pallets: Vec<String>,
 
+	/// Comma separated list of `pallet::extrinsic` combinations that should not be run.
+	///
+	/// Example: `frame_system::remark,pallet_balances::transfer_keep_alive`
+	#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
+	pub exclude_extrinsics: Vec<String>,
+
 	/// Run benchmarks for all pallets and extrinsics.
 	///
 	/// This is equivalent to running `--pallet * --extrinsic *`.