From be588b148359463031429e0201783b9e1f095131 Mon Sep 17 00:00:00 2001 From: Aaron Rossetto Date: Mon, 7 Mar 2022 08:38:58 -0600 Subject: tests: Fix converter benchmark disable on Boost <1.68 Boost versions prior to 1.68 appear to have a bug where a decorator to denote a test as disabled is not honored when affixed to a data-driven test case, which is how the benchmarks in convert_test are skipped when the unit test is run. (The tests take some time to complete and we don't want them running with every CI pass.) This commit adds an alternative benchmark skipping mechanism when Boost <1.68 is used. The benchmark test cases perform a runtime check for the user-provided `--benchmark` command-line option. If not found, the test case returns prematurely. If found, the test case will execute. Note that because `--benchmark` is a command-line option specific to this test, and not to Boost, the options must follow `--` in the command line in order to take effect: `convert_test -- --benchmark`. --- host/tests/convert_test.cpp | 80 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/host/tests/convert_test.cpp b/host/tests/convert_test.cpp index 67d035e7a..40ec51bd2 100644 --- a/host/tests/convert_test.cpp +++ b/host/tests/convert_test.cpp @@ -53,6 +53,32 @@ const std::array CONV_PRIO_TYPES{-1, 0, 1, 2, 3} return; \ } +// There appears to be a bug with Boost <1.68 where decorators on Boost data +// test cases are not honored. (While I could not find the bug itself, it is +// referred to in a comment at https://github.com/boostorg/test/issues/139 on +// Boost's GitHub bug tracker.) Unfortunately, that means that the benchmarks +// are run by default on those versions of Boost, causing this unit test to +// take much longer than usual and thus slowing down our entire CI pipeline. +// The SKIP_BENCHMARK_CHECK macro implements a run-time check on older versions +// of Boost to determine whether the benchmarks should be skipped or not. To +// 'unskip' the benchmarks on those versions, pass `--benchmark` to the +// invocation of `convert_test` (note that the `--benchmark` flag must be +// specified after `--` to ensure it is passed directly to the test and not +// interpreted by Boost.) +#if (BOOST_VERSION < 106800) +#define SKIP_BENCHMARK_CHECK \ + if (!(boost::unit_test::framework::master_test_suite().argc >= 2 \ + && std::string(boost::unit_test::framework::master_test_suite().argv[1]) \ + == "--benchmark")) { \ + return; \ + } +#else +// For versions of Boost where this issue has been fixed, the benchmarks can +// be enabled by invoking convert_test with `--run-test=+benchmark*` to +// explicitly enable all the disabled benchmark tests. +#define SKIP_BENCHMARK_CHECK +#endif + // Shorthand for defining a test case that tests all prios. Creates a variable // 'conv_prio_type' #define MULTI_CONVERTER_TEST_CASE(test_name) \ @@ -272,6 +298,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_be_sc16) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_be_sc16) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -302,6 +330,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_le_sc16) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_le_sc16) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -332,6 +362,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_chdr_sc16) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_chdr_sc16) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -468,6 +500,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_be_fc32_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_be_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -512,6 +546,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_le_fc32_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_le_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -556,6 +592,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_chdr_fc32_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_chdr_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -600,6 +638,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_be_fc64_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_be_fc64) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc64"; id.num_inputs = 1; @@ -644,6 +684,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_le_fc64_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_le_fc64) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc64"; id.num_inputs = 1; @@ -688,6 +730,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_chdr_fc64_with_saturation) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_chdr_fc64) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc64"; id.num_inputs = 1; @@ -722,6 +766,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_le_sc12_with_fc32) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_le_sc12_with_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -752,6 +798,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_be_sc12_with_fc32) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_be_sc12_with_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -782,6 +830,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_le_sc16_and_sc12) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_le_sc16_and_sc12) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -812,6 +862,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_be_sc16_and_sc12) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_be_sc16_and_sc12) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -846,6 +898,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_le_fc32_with_fc32) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_le_fc32_with_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -876,6 +930,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_be_fc32_with_fc32) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_be_fc32_with_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -906,6 +962,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_fc32_with_fc32_chdr) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_fc32_with_fc32_chdr) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -945,6 +1003,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_fc64_and_sc8) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_fc64_and_sc8) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc64"; id.num_inputs = 1; @@ -988,6 +1048,8 @@ BOOST_AUTO_TEST_CASE(test_convert_types_fc32_and_sc8) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) BOOST_AUTO_TEST_CASE(benchmark_convert_types_fc32_and_sc8) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -1032,6 +1094,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_sc16_and_sc8) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_sc16_and_sc8) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "sc16"; id.num_inputs = 1; @@ -1100,6 +1164,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_u8_and_u8) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_u8_and_u8) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "u8"; id.num_inputs = 1; @@ -1138,6 +1204,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_u8_and_u8_chdr) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_u8_and_u8_chdr) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "u8"; id.output_format = "u8_chdr"; @@ -1197,6 +1265,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_s8_and_s8) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_s8_and_s8) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "s8"; id.num_inputs = 1; @@ -1236,6 +1306,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_s8_and_s8_chdr) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_s8_and_s8_chdr) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "s8"; id.output_format = "s8_chdr"; @@ -1295,6 +1367,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_s16_and_s16) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_s16_and_s16) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "s16"; id.num_inputs = 1; @@ -1333,6 +1407,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_s16_and_s16_chdr) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_s16_and_s16_chdr) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "s16"; id.output_format = "s16_chdr"; @@ -1394,6 +1470,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_fc32_and_fc32) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_fc32_and_fc32) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.num_inputs = 1; @@ -1431,6 +1509,8 @@ MULTI_CONVERTER_TEST_CASE(test_convert_types_fc32_and_fc32_chdr) BOOST_TEST_DECORATOR(*boost::unit_test::disabled()) MULTI_CONVERTER_TEST_CASE(benchmark_convert_types_fc32_and_fc32_chdr) { + SKIP_BENCHMARK_CHECK; + convert::id_type id; id.input_format = "fc32"; id.output_format = "fc32_chdr"; -- cgit v1.2.3