fio: Adding fio engine

README.md

@@ -26,6 +26,7 @@ The current version of hwbench supports 3 different engines.
 - [stress-ng](https://github.com/ColinIanKing/stress-ng): no need to present this very popular low-level benchmarking tool
 - spike: a custom engine used to make fans spike. Very useful to study the cooling strategy of a server.
 - sleep: a stupid sleep call used to observe how the system is behaving in idle mode
+- [fio](https://github.com/axboe/fio): a flexible storage benchmarking tool, see [documentation](./documentation/fio.md)
 
 Benchmark performance metrics are extracted and saved for later analysis.
 

configs/fio.conf

@@ -0,0 +1,15 @@
+# This configuration will :
+# - test /dev/nvme0n1 in 4k randread for 40 seconds
+# -- first with 4 stressors
+# -- then with 6 stressors
+[global]
+runtime=40
+monitor=all
+
+[randread_cmdline]
+engine=fio
+engine_module=cmdline
+engine_module_parameter_base=--filename=/dev/nvme0n1 --direct=1 --rw=randread --bs=4k --ioengine=libaio --iodepth=256 --group_reporting --readonly
+hosting_cpu_cores=all
+hosting_cpu_cores_scaling=none
+stressor_range=4,6

documentation/fio.md

@@ -0,0 +1,52 @@
+# FIO
+
+hwbench can use [fio](https://github.com/axboe/fio) to perform storage benchmarking.
+The current implementation requires fio >= 3.19.
+
+# Concept
+Fio is operated in three(3) different modes by selecting the `engine_module` directive.
+
+## Command line
+
+When `engine_module=cmdline` is used, the content of `engine_module_parameter_base` will be passed directly to fio with some limitations.
+
+The following fio keywords are automatically defined, or replaced if present, by hwbench :
+
+- `--runtime`: set to match the exact duration of the current hwbench benchmark.
+- `--time_based`: it's mandatory to have a benchmark lasting `runtime` seconds.
+- `--output-format`: hwbench need the output to be set in `json+` for an easy integration.
+- `--name`: hwbench will use the current job name to ensure its unique over the runs.
+- `--numjobs`: defined by `stressor_range`, can be set as a unique value or a list of values. Each value will generate a new benchmark.
+- `--write_{bw|lat|hist|iops}_logs`: hwbench will automatically collect the performance logs to let hwgraph doing time-based graphs.
+- `--invalidate`: hwbench ensure that every benchmark will be done out of cache.
+
+### Sample configuration file
+
+The following job defines two benchmarks on the same device (nvme0n1).
+
+The `randread_cmdline` job will create :
+- `randread_cmdline_0` benchmark with ``numjobs=4`` extracted from `stressor_range` list
+- `randread_cmdline_1` benchmark with ``numjobs=6`` extracted from `stressor_range` list
+
+```
+[randread_cmdline]
+runtime=600
+engine=fio
+engine_module=cmdline
+engine_module_parameter_base=--filename=/dev/nvme0n1 --direct=1 --rw=randread --bs=4k --ioengine=libaio --iodepth=256 --group_reporting --readonly
+hosting_cpu_cores=all
+hosting_cpu_cores_scaling=none
+stressor_range=4,6
+```
+
+Please note the `hosting_cpu_cores` only selects a set of cores to pin fio. A possible usage would be using a list of cores with a `hosting_cpu_cores_scaling` to study the performance of the same storage device from different NUMA domains.
+
+## External file execution
+Hwbench execute an already existing fio job file.
+
+Not yet implemented.
+
+## Automatic job definition
+Hwbench automatically creates jobs based on some hardware detection and profiles.
+
+Not yet implemented.

hwbench/bench/benchmark.py

@@ -20,6 +20,7 @@ def __init__(
         self.job_number = job_number
         self.enginemodule = enginemodule
         self.parameters = parameters
+        self.parameters.set_benchmark(self)
 
     def get_enginemodule(self) -> EngineModuleBase:
         return self.enginemodule
@@ -114,17 +115,17 @@ def pre_run(self):
         cpu_location = ""
         if p.get_pinned_cpu():
             if isinstance(p.get_pinned_cpu(), (int, str)):
-                cpu_location = f" on CPU {p.get_pinned_cpu():3d}"
+                cpu_location = f" pinned on CPU {p.get_pinned_cpu():3d}"
             elif isinstance(p.get_pinned_cpu(), list):
-                cpu_location = f" on CPU [{h.cpu_list_to_range(p.get_pinned_cpu())}]"
+                cpu_location = f" pinned on CPU [{h.cpu_list_to_range(p.get_pinned_cpu())}]"
             else:
                 h.fatal(f"Unsupported get_pinned_cpu() format :{type(p.get_pinned_cpu())}")
 
         monitoring = ""
         if self.parameters.get_monitoring():
             monitoring = "(M)"
         print(
-            f"[{p.get_name()}] {self.engine_module.get_engine().get_name()}/"
+            f"[{p.get_name_with_position()}] {self.engine_module.get_engine().get_name()}/"
             f"{self.engine_module.get_name()}/{p.get_engine_module_parameter()}{monitoring}: "
             f"{p.get_engine_instances_count():3d} stressor{cpu_location} for {p.get_runtime()}s{status}"
         )

hwbench/bench/benchmarks.py

@@ -244,7 +244,7 @@ def run(self):
                 print(f"hwbench: [{bench_name}]: started at {datetime.datetime.utcnow()}")
 
             # Save each benchmark result
-            results[f"{benchmark.get_parameters().get_name()}_{benchmark.get_job_number()}"] = benchmark.run()
+            results[benchmark.get_parameters().get_name_with_position()] = benchmark.run()
         return results
 
     def dump(self):
@@ -253,7 +253,7 @@ def dump(self):
                 engine = bench.get_enginemodule().get_engine()
                 em = bench.get_enginemodule()
                 param = bench.get_parameters()
-                print(f"[{param.get_name()}_{bench.get_job_number()}]", file=f)
+                print(f"[{param.get_name_with_position()}]", file=f)
                 print(f"runtime={param.get_runtime()}", file=f)
                 print(f"monitoring={param.get_monitoring_config()}", file=f)
                 print(f"engine={engine.get_name()}", file=f)

hwbench/bench/parameters.py

@@ -37,6 +37,12 @@ def __init__(
         self.skip_method = skip_method
         self.sync_start = sync_start
 
+    def get_benchmark(self):
+        return self.benchmark
+
+    def set_benchmark(self, benchmark):
+        self.benchmark = benchmark
+
     def get_pinned_cpu(self):
         if self.pinned_cpu == "none":
             return ""
@@ -45,6 +51,11 @@ def get_pinned_cpu(self):
     def get_name(self) -> str:
         return self.job_name
 
+    def get_name_with_position(self) -> str:
+        if not self.benchmark:
+            return self.get_name()
+        return f"{self.get_name()}_{self.benchmark.get_job_number()}"
+
     def get_engine_instances_count(self) -> int:
         return self.engine_instances
 

hwbench/bench/test_fio.py

@@ -0,0 +1,38 @@
+from . import test_benchmarks_common as tbc
+
+
+class TestFio(tbc.TestCommon):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.load_mocked_hardware(
+            cpucores="./hwbench/tests/parsing/cpu_cores/v2321",
+            cpuinfo="./hwbench/tests/parsing/cpu_info/v2321",
+            numa="./hwbench/tests/parsing/numa/8domainsllc",
+        )
+        self.load_benches("./hwbench/config/fio.conf")
+        self.parse_jobs_config()
+        self.QUADRANT0 = list(range(0, 16)) + list(range(64, 80))
+        self.QUADRANT1 = list(range(16, 32)) + list(range(80, 96))
+        self.ALL = list(range(0, 128))
+
+    def test_fio(self):
+        """Check fio syntax."""
+        assert self.benches.count_benchmarks() == 2
+        assert self.benches.count_jobs() == 1
+        assert self.benches.runtime() == 80
+
+        for bench in self.benches.benchs:
+            assert bench.validate_parameters() is None
+            assert bench.get_parameters().get_name() == "randread_cmdline"
+
+        bench_0 = self.get_bench_parameters(0)
+        assert (
+            bench_0.get_engine_module_parameter_base()
+            == "--filename=/dev/nvme0n1 --direct=1 --rw=randread --bs=4k --ioengine=libaio --iodepth=256 --group_reporting --readonly --runtime=40 --time_based --output-format=json+ --numjobs=4 --name=randread_cmdline_0 --invalidate=1 --log_avg_msec=20000 --write_bw_log=fio/randread_cmdline_0_bw.log --write_lat_log=fio/randread_cmdline_0_lat.log --write_hist_log=fio/randread_cmdline_0_hist.log --write_iops_log=fio/randread_cmdline_0_iops.log"
+        )
+
+        bench_1 = self.get_bench_parameters(1)
+        assert (
+            bench_1.get_engine_module_parameter_base()
+            == "--filename=/dev/nvme0n1 --direct=1 --rw=randread --bs=4k --ioengine=libaio --iodepth=256 --group_reporting --readonly --runtime=40 --time_based --output-format=json+ --numjobs=6 --name=randread_cmdline_1 --invalidate=1 --log_avg_msec=20000 --write_bw_log=fio/randread_cmdline_1_bw.log --write_lat_log=fio/randread_cmdline_1_lat.log --write_hist_log=fio/randread_cmdline_1_hist.log --write_iops_log=fio/randread_cmdline_1_iops.log"
+        )

hwbench/config/fio.conf

@@ -0,0 +1,18 @@
+# This configuration will :
+# - test /dev/nvme0n1 in 4k randread for 40 seconds
+# -- first with 4 stressors
+# -- then with 6 stressors
+#
+# As runtime is set to 30s by the user, it should be replaced by runtime=40 defined by hardware bench
+[global]
+runtime=40
+monitor=all
+
+[randread_cmdline]
+engine=fio
+engine_module=cmdline
+engine_module_parameter_base=--filename=/dev/nvme0n1 --direct=1 --rw=randread --bs=4k --name TOBEREMOVED --ioengine=libaio --iodepth=256 --group_reporting --readonly --runtime=30 --time_based --time_based --numjobs=10 --name=plop --name _REMOVE_ME
+hosting_cpu_cores=all
+hosting_cpu_cores_scaling=none
+stressor_range=4,6
+

hwbench/config/test_parse_fio.py

@@ -0,0 +1,29 @@
+from unittest.mock import patch
+
+import pytest
+
+from hwbench.bench import test_benchmarks_common as tbc
+from hwbench.environment.mock import MockHardware
+
+
+class TestParseConfig(tbc.TestCommon):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hw = MockHardware()
+        self.load_benches("./hwbench/config/fio.conf")
+
+    def test_sections_name(self):
+        """Check if sections names are properly detected."""
+        sections = self.get_jobs_config().get_sections()
+        assert sections == [
+            "randread_cmdline",
+        ]
+
+    def test_keywords(self):
+        """Check if all keywords are valid."""
+        try:
+            with patch("hwbench.utils.helpers.is_binary_available") as iba:
+                iba.return_value = True
+                self.get_jobs_config().validate_sections()
+        except Exception as exc:
+            pytest.fail(f"'validate_sections' detected a syntax error {exc}")