This post describes a minimal example showing how to use CMake and the google micro-benchmarking library.
We use the cmake download_project function that can be found here github.com/Crascit/DownloadProject/.
The final project struture will be:
.
├── bench
│ ├── CMakeLists.txt
│ └── one_example.cpp
├── cmake
│ ├── ConfigGBench.cmake
│ ├── ConfigSafeGuards.cmake
│ ├── DownloadProject.cmake
│ └── DownloadProject.CMakeLists.cmake.in
└── CMakeLists.txtFor your convenience, all the code can be found on GitHub. You can easily clone it for tests.
The master CMakeLists.txt code is:
cmake_minimum_required(VERSION 3.0)
project(Benchmarking_Demo LANGUAGES CXX)
# Location of additional cmake modules
#
set(CMAKE_MODULE_PATH
${CMAKE_MODULE_PATH}
${PROJECT_SOURCE_DIR}/cmake
)
# Guard against in-source builds and bad build-type strings
#
include(ConfigSafeGuards)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# This include is required if you want
# to use CTest framework for you benchmarks with add_test()
#
include(CTest)
# If you want to self-test benchmark lib too, turn me ON
#
set(BENCHMARK_ENABLE_TESTING OFF)
# Configure google micro benchmark
#
# c++11 is required
#
if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang))
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g3")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
endif()
include(ConfigGBench)
add_subdirectory(${PROJECT_SOURCE_DIR}/bench/)it includes include(ConfigGBench) in charge of downloading and configuring the googlebenchmark subproject.
# Adapted from https://github.com/Crascit/DownloadProject/blob/master/CMakeLists.txt
#
# CAVEAT: use DownloadProject.cmake
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
if (CMAKE_VERSION VERSION_LESS 3.2)
set(UPDATE_DISCONNECTED_IF_AVAILABLE "")
else()
set(UPDATE_DISCONNECTED_IF_AVAILABLE "UPDATE_DISCONNECTED 1")
endif()
include(DownloadProject)
download_project(PROJ googlebenchmark
GIT_REPOSITORY https://github.com/google/benchmark.git
GIT_TAG master
${UPDATE_DISCONNECTED_IF_AVAILABLE}
)
add_subdirectory(${googlebenchmark_SOURCE_DIR} ${googlebenchmark_BINARY_DIR})
include_directories("${googlebenchmark_SOURCE_DIR}/include")Now we have to define the CMakeLists.txt file of the ./bench directory
find_package(Threads REQUIRED)
#~~~~~~~~~~~~~~~~
file(GLOB_RECURSE ALL_BENCH_CPP *.cpp)
foreach(ONE_BENCH_CPP ${ALL_BENCH_CPP})
get_filename_component(ONE_BENCH_EXEC ${ONE_BENCH_CPP} NAME_WE)
# Avoid name collision
set(TARGET_NAME Bench_${ONE_BENCH_EXEC})
add_executable(${TARGET_NAME} ${ONE_BENCH_CPP})
set_target_properties(${TARGET_NAME} PROPERTIES OUTPUT_NAME ${ONE_BENCH_EXEC})
target_link_libraries(${TARGET_NAME}
# If required, you can add your project library here
# ...
benchmark
${CMAKE_THREAD_LIBS_INIT})
# If you want to run benchmarks with the "make test" command, uncomment me
add_test(${TARGET_NAME} ${ONE_BENCH_EXEC})
endforeach()Finally we also need a small stuff to bench
#include "benchmark/benchmark_api.h"
#include <set>
#include <vector>
static void BM_VectorInsert(benchmark::State &state) {
while (state.KeepRunning()) {
std::vector<int> insertion_test;
for (int i = 0, i_end = state.range_x(); i < i_end; i++) {
insertion_test.push_back(i);
}
}
}
// Register the function as a benchmark
BENCHMARK(BM_VectorInsert)->Range(8, 8 << 10);
//~~~~~~~~~~~~~~~~
// Define another benchmark
static void BM_SetInsert(benchmark::State &state) {
while (state.KeepRunning()) {
std::set<int> insertion_test;
for (int i = 0, i_end = state.range_x(); i < i_end; i++) {
insertion_test.insert(i);
}
}
}
BENCHMARK(BM_SetInsert)->Range(8, 8 << 10);
BENCHMARK_MAIN();You can go to the google micro-benchmarking library to see other examples.
Do the usual:
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make
make testit should work… you get something like:
Running tests...
Test project /home/picaud/GitLab/Reports/Blog/Bench/code/build
Start 1: Bench_one_example
1/1 Test #1: Bench_one_example ................ Passed 1.99 sec
100% tests passed, 0 tests failed out of 1
Total Test time (real) = 1.99 secThe result is in the Testing/Temporary/LastTest.log file and looks like:
Start testing: May 22 19:37 CEST
----------------------------------------------------------
1/1 Testing: Bench_one_example
1/1 Test: Bench_one_example
Command: "/home/picaud/GitLab/Reports/Blog/Bench/code/build/bench/one_example"
Directory: /home/picaud/GitLab/Reports/Blog/Bench/code/build/bench
"Bench_one_example" start time: May 22 19:37 CEST
Output:
----------------------------------------------------------
Run on (4 X 2533 MHz CPU s)
2016-05-22 19:37:35
***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
***WARNING*** Library was built as DEBUG. Timings may be affected.
Benchmark Time CPU Iterations
----------------------------------------------------------
BM_VectorInsert/8 1541 ns 1542 ns 448718
BM_VectorInsert/64 3626 ns 3621 ns 182292
BM_VectorInsert/512 13501 ns 13458 ns 54687
BM_VectorInsert/4k 75578 ns 75562 ns 9211
BM_VectorInsert/8k 146739 ns 146472 ns 4861
BM_SetInsert/8 3024 ns 3017 ns 233333
BM_SetInsert/64 31640 ns 31629 ns 23649
BM_SetInsert/512 303617 ns 303472 ns 2333
BM_SetInsert/4k 2954728 ns 2949153 ns 236
BM_SetInsert/8k 6345872 ns 6311927 ns 109
<end of output>
Test time = 9.45 sec
----------------------------------------------------------
Test Passed.
"Bench_one_example" end time: May 22 19:37 CEST
"Bench_one_example" time elapsed: 00:00:09
----------------------------------------------------------
End testing: May 22 19:37 CESTOther wise you can execute each benchmark individually
/one_example --benchmark_format=jsonyou get, on your terminal:
{
"context": {
"date": "2016-05-22 19:43:47",
"num_cpus": 4,
"mhz_per_cpu": 2533,
"cpu_scaling_enabled": false,
"library_build_type": "release"
},
"benchmarks": [
{
"name": "BM_VectorInsert/8",
"iterations": 2966102,
"real_time": 226,
"cpu_time": 227,
"time_unit": "ns"
},
{
"name": "BM_VectorInsert/64",
"iterations": 972222,
"real_time": 605,
"cpu_time": 605,
"time_unit": "ns"
},
{
"name": "BM_VectorInsert/512",
"iterations": 380435,
"real_time": 1795,
"cpu_time": 1798,
"time_unit": "ns"
},
{
"name": "BM_VectorInsert/4k",
"iterations": 97222,
"real_time": 7235,
"cpu_time": 7200,
"time_unit": "ns"
},
{
"name": "BM_VectorInsert/8k",
"iterations": 51471,
"real_time": 13302,
"cpu_time": 13289,
"time_unit": "ns"
},
{
"name": "BM_SetInsert/8",
"iterations": 1093750,
"real_time": 581,
"cpu_time": 578,
"time_unit": "ns"
},
{
"name": "BM_SetInsert/64",
"iterations": 109375,
"real_time": 5904,
"cpu_time": 5925,
"time_unit": "ns"
},
{
"name": "BM_SetInsert/512",
"iterations": 11667,
"real_time": 52307,
"cpu_time": 52113,
"time_unit": "ns"
},
{
"name": "BM_SetInsert/4k",
"iterations": 1346,
"real_time": 499965,
"cpu_time": 499257,
"time_unit": "ns"
},
{
"name": "BM_SetInsert/8k",
"iterations": 700,
"real_time": 999270,
"cpu_time": 994286,
"time_unit": "ns"
}
]
}During benchmark you must use performance mode for your CPU. To setup your cpu governance mode you can do as follow.
Tested on Debian, source of information here.
apt-get install linux-cpupowerTurns on performance mode (in su mode)
cpupower frequency-set --governor performanceAfter benchmarking your software you can go back to the more conservative option:
cpupower frequency-set --governor ondemandA nice thing is the google benchmark warns you
***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.if you are not using the right cpu mode.
It also warns you if you are in debug mode:
***WARNING*** Library was built as DEBUG. Timings may be affected.