Test benchmarking

.github/workflows/benchmark.yml

@@ -0,0 +1,95 @@
+name: Benchmark
+on:
+  push:
+    branches:
+      - test_benchmarking
+
+jobs:
+  master-benchmark:
+    name: Benchmark Master
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - name: compile
+        run: |
+          git submodule update --init
+          mkdir build
+          ./compile.sh --docker-build
+          docker run -i --rm -v $(pwd):/host_pwd -w /host_pwd artss_docker sh -c "cd build && cmake .. && make artss_bench && ./benchmarks/bin/artss_bench --benchmark_format=json | tee ../benchmark_result.json"
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'googlecpp'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark_result.json
+          # Where the previous data file is stored
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - name: compile
+        run: |
+          git submodule update --init
+          mkdir build
+          ./compile.sh --docker-build
+          docker run -i --rm -v $(pwd):/host_pwd -w /host_pwd artss_docker sh -c "cd build && cmake .. && make artss_bench && ./benchmarks/bin/artss_bench --benchmark_format=json | tee ../benchmark_result.json"
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'googlecpp'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark_result.json
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-always: true
+
+  benchmark-gpu:
+    name: Benchmark-GPU
+    runs-on: gpu
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - name: compile
+        run: |
+          git submodule update --init
+          mkdir build
+          ./compile.sh --docker-build
+          docker run --gpus all -i --rm -v $(pwd):/host_pwd -w /host_pwd --user 1000:1000 artss_docker sh -c "./compile.sh --jobs 32 --gpu_benchmark && cd build && make artss_bench && ./benchmarks/bin/artss_bench --benchmark_format=json | tee ../benchmark_result_gb.json"
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'googlecpp'
+          # Where the output from the benchmark tool is stored
+          output-file-path: benchmark_result_gb.json
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-always: true

.github/workflows/ccpp.yml

@@ -16,6 +16,7 @@ jobs:
     - name: compile
       run: |
         git submodule update --init
+        sudo apt-get install -y mpich libhdf5-dev
         ./compile.sh --jobs 4 --gcc -s --checkout
 
   build-gcc-serial-benchmarking:
@@ -27,6 +28,7 @@ jobs:
     - name: compile
       run: |
         git submodule update --init
+        sudo apt-get install -y mpich libhdf5-dev
         ./compile.sh --jobs 4 --gcc --sb --checkout
 
   build-pgi-multicore:
@@ -97,6 +99,16 @@ jobs:
           ./compile.sh --docker-build
           docker run -i --rm -v $(pwd):/host_pwd -w /host_pwd artss_docker sh -c "./compile.sh --jobs 4 -s -m -d && cd build && ctest --output-on-failure -E 'gpu|gtest' --checkout"
 
+  cmake-tests-gpu:
+
+    runs-on: gpu
+    steps:
+      - uses: actions/checkout@v2
+      - name: compile
+        run: |
+          git submodule update --init
+          ./compile.sh --docker-build
+          docker run -i --rm -v $(pwd):/host_pwd -w /host_pwd --user 1000:1000 artss_docker sh -c "./compile.sh --jobs 32 -g -m -d && cd build && ctest --output-on-failure --checkout"
   unit-tests_serial:
 
     runs-on: ubuntu-latest

.gitmodules

@@ -7,3 +7,9 @@
 [submodule "external/googletest"]
 	path = external/googletest
 	url = https://github.com/google/googletest/
+[submodule "external/HighFive"]
+	path = external/HighFive
+	url = https://github.com/BlueBrain/HighFive.git
+[submodule "external/benchmark"]
+	path = external/benchmark
+	url = https://github.com/google/benchmark

CMakeLists.txt

@@ -146,6 +146,15 @@ if(NOT TARGET gtest::gtest)
     add_subdirectory(external/googletest)
 endif()
 
+if(NOT TARGET benchmark::benchmark)
+    # Stand-alone build
+    find_package(benchmark::benchmark QUIET)
+endif()
+# use local fallback
+if(NOT TARGET benchmark::benchmark)
+    add_subdirectory(external/benchmark)
+endif()
+
 if(NOT TARGET MPI)
     find_package(MPI)
 endif()
@@ -197,10 +206,19 @@ add_subdirectory(gtests)
 
 add_subdirectory(src/dataAssimilation)
 add_subdirectory(src/TCP)
+
+set(HIGHFIVE_USE_BOOST OFF)
+add_subdirectory(external/HighFive)
+
 get_target_property(ARTSS_SOURCE_FILES artss SOURCES)
 add_executable(artss_data_assimilation_serial ${ARTSS_SOURCE_FILES} src/main.cpp)
 target_compile_options(artss_data_assimilation_serial PUBLIC ${ARTSS_COMPILE_OPTIONS})
 target_compile_options(artss_data_assimilation_serial PUBLIC -DASSIMILATION)
 target_link_libraries(artss_data_assimilation_serial PRIVATE spdlog::spdlog)
 target_link_libraries(artss_data_assimilation_serial PRIVATE fmt::fmt)
 target_link_libraries(artss_data_assimilation_serial PRIVATE MPI::MPI_CXX)
+target_link_libraries(artss_data_assimilation_serial PRIVATE HighFive)
+
+
+set(BENCHMARK_ENABLE_TESTING OFF)
+add_subdirectory(benchmarks)

benchmarks/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_executable(artss_bench ${ARTSS_SOURCE_FILES})
+
+add_subdirectory(field)
+
+target_include_directories(artss_bench PRIVATE ${CMAKE_SOURCE_DIR})
+target_link_libraries(artss_bench benchmark::benchmark)
+target_link_libraries(artss_bench gtest)
+target_link_libraries(artss_bench spdlog::spdlog)
+target_link_libraries(artss_bench fmt::fmt)
+target_link_libraries(artss_bench MPI::MPI_CXX)
+target_link_libraries(artss_bench HighFive)
+
+add_test(NAME artss_bench COMMAND artss_bench)

benchmarks/Environment.h

@@ -0,0 +1,25 @@
+/// \file       Environment.cpp
+/// \brief      
+/// \date       Dec 10, 2021
+/// \author     My Linh Wuerzburger
+/// \copyright  <2015-2021> Forschungszentrum Juelich All rights reserved.
+
+
+#ifndef BENCH_ENVIRONMENT_H_
+#define BENCH_ENVIRONMENT_H_
+
+#include <benchmark/benchmark.h>
+#include "src/utility/Utility.h"
+
+static int setup_done = 0;
+
+// Override this to define how to set up the environment.
+static void DoSetup(const benchmark::State& state) {
+    if (setup_done)
+        return;
+
+    Utility::create_logger("info", "gtest.log");
+    setup_done = 1;
+}
+
+#endif /* BENCH_FIELD_FIELD_H_ */

benchmarks/field/CMakeLists.txt

@@ -0,0 +1,4 @@
+target_sources(artss_bench
+    PRIVATE
+    ${CMAKE_CURRENT_LIST_DIR}/Field.cpp
+)

benchmarks/field/Field.cpp

@@ -0,0 +1,56 @@
+#include <benchmark/benchmark.h>
+
+#include "../Environment.h"
+#include "src/field/Field.h"
+
+#include <math.h>
+
+
+static void BM_AddScalar(benchmark::State &state) {
+    // Utility::create_logger("info", "bench.log");
+    size_t size = state.range(0);
+
+    for (auto _ : state) {
+        Field a(UNKNOWN_FIELD, 0.0, 0, size);
+        a += M_E;
+    }
+}
+
+static void BM_AddFields(benchmark::State &state) {
+    // Utility::create_logger("info", "bench.log");
+    size_t size = state.range(0);
+
+    for (auto _ : state) {
+        Field a(UNKNOWN_FIELD, 0.0, 0, size);
+        Field b(UNKNOWN_FIELD, 0.0, 0, size);
+        a += b;
+    }
+}
+
+static void BM_MulScalar(benchmark::State &state) {
+    // Utility::create_logger("info", "bench.log");
+    size_t size = state.range(0);
+
+    for (auto _ : state) {
+        Field a(UNKNOWN_FIELD, 0.0, 0, size);
+        a *= M_E;
+    }
+}
+
+static void BM_MulFields(benchmark::State &state) {
+    // Utility::create_logger("info", "bench.log");
+    size_t size = state.range(0);
+
+    for (auto _ : state) {
+        Field a(UNKNOWN_FIELD, 0.0, 0, size);
+        Field b(UNKNOWN_FIELD, 0.0, 0, size);
+        a *= b;
+    }
+}
+
+BENCHMARK(BM_AddScalar)->ThreadRange(1, 8)->Range(8, 8<<20)->Setup(DoSetup);
+BENCHMARK(BM_AddFields)->ThreadRange(1, 8)->Range(8, 8<<20)->Setup(DoSetup);
+BENCHMARK(BM_MulScalar)->ThreadRange(1, 8)->Range(8, 8<<20)->Setup(DoSetup);
+BENCHMARK(BM_MulFields)->ThreadRange(1, 8)->Range(8, 8<<20)->Setup(DoSetup);
+
+BENCHMARK_MAIN();