From 2bf1f0acaa3595a121d4140d78d50c52b817ce09 Mon Sep 17 00:00:00 2001 From: Ciro Nishiguchi Date: Tue, 26 Nov 2019 08:30:05 -0600 Subject: tests: Add script to execute batch of benchmark_rate runs Script runs benchmark_rate repeatedly, parses results, and calculates average, min, and max of each value reported. --- host/tests/CMakeLists.txt | 1 + host/tests/streaming_performance/CMakeLists.txt | 14 ++ .../batch_run_benchmark_rate.py | 174 +++++++++++++++++++++ .../streaming_performance/parse_benchmark_rate.py | 149 ++++++++++++++++++ .../streaming_performance/run_benchmark_rate.py | 86 ++++++++++ 5 files changed, 424 insertions(+) create mode 100644 host/tests/streaming_performance/CMakeLists.txt create mode 100755 host/tests/streaming_performance/batch_run_benchmark_rate.py create mode 100644 host/tests/streaming_performance/parse_benchmark_rate.py create mode 100644 host/tests/streaming_performance/run_benchmark_rate.py (limited to 'host') diff --git a/host/tests/CMakeLists.txt b/host/tests/CMakeLists.txt index d3c0b8e6a..8e58dd591 100644 --- a/host/tests/CMakeLists.txt +++ b/host/tests/CMakeLists.txt @@ -248,3 +248,4 @@ if(MSVC OR APPLE OR LINUX) endif() add_subdirectory(devtest) +add_subdirectory(streaming_performance) diff --git a/host/tests/streaming_performance/CMakeLists.txt b/host/tests/streaming_performance/CMakeLists.txt new file mode 100644 index 000000000..0ea828737 --- /dev/null +++ b/host/tests/streaming_performance/CMakeLists.txt @@ -0,0 +1,14 @@ +# +# Copyright 2019 Ettus Research, a National Instruments Brand +# +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set(streaming_performance_files + parse_benchmark_rate.py + run_benchmark_rate.py + batch_run_benchmark_rate.py + run_X300_performance_tests.py +) + +UHD_INSTALL(PROGRAMS ${streaming_performance_files} DESTINATION ${PKG_LIB_DIR}/tests/streaming_performance COMPONENT tests) diff --git a/host/tests/streaming_performance/batch_run_benchmark_rate.py b/host/tests/streaming_performance/batch_run_benchmark_rate.py new file mode 100755 index 000000000..ad47cfd5c --- /dev/null +++ b/host/tests/streaming_performance/batch_run_benchmark_rate.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Copyright 2019 Ettus Research, A National Instrument Brand + +SPDX-License-Identifier: GPL-3.0-or-later + +Runs the benchmark rate C++ example for a specified number of iterations and +aggregates results. + +Example usage: +batch_run_benchmark_rate.py --path /benchmark_rate --iterations 1 --args "addr=192.168.30.2" --rx_rate 1e6 +""" +import argparse +import collections +import parse_benchmark_rate +import run_benchmark_rate + +Results = collections.namedtuple( + 'Results', + """ + avg_vals + min_vals + max_vals + non_zero_vals + """ +) + +def calculate_stats(results): + """ + Calculates performance metrics from list of parsed benchmark rate results. + """ + result_avg = parse_benchmark_rate.average(results) + result_min = parse_benchmark_rate.min_vals(results) + result_max = parse_benchmark_rate.max_vals(results) + result_nz = parse_benchmark_rate.non_zero_vals(results) + return Results( + avg_vals = result_avg, + min_vals = result_min, + max_vals = result_max, + non_zero_vals = result_nz) + +def run(path, iterations, benchmark_rate_params, stop_on_error=True): + """ + Runs benchmark rate multiple times and returns a list of parsed results. + """ + print("Running benchmark rate {} times with the following arguments: ".format(iterations)) + for key, val in benchmark_rate_params.items(): + print("{:14} {}".format(key, val)) + + parsed_results = [] + iteration = 0 + while iteration < iterations: + proc = run_benchmark_rate.run(path, benchmark_rate_params) + result = parse_benchmark_rate.parse(proc.stdout.decode('ASCII')) + if result != None: + parsed_results.append(result) + iteration += 1 + else: + if stop_on_error: + msg = "Could not parse results of benchmark_rate\n" + msg += "Benchmark rate arguments:\n" + msg += str(proc.args) + "\n" + msg += "Stderr capture:\n" + msg += proc.stderr.decode('ASCII') + msg += "Stdout capture:\n" + msg += proc.stdout.decode('ASCII') + raise RuntimeError(msg) + else: + print("Failed to parse benchmark rate results") + print(proc.stderr.decode('ASCII')) + + return parsed_results + +def get_summary_string(summary, params=None): + """ + Returns summary info in a string resembling benchmark_rate output. + """ + statistics_msg = """ +Benchmark rate summary: + Num received samples: avg {}, min {}, max {}, non-zero {} + Num dropped samples: avg {}, min {}, max {}, non-zero {} + Num overruns detected: avg {}, min {}, max {}, non-zero {} + Num transmitted samples: avg {}, min {}, max {}, non-zero {} + Num sequence errors (Tx): avg {}, min {}, max {}, non-zero {} + Num sequence errors (Rx): avg {}, min {}, max {}, non-zero {} + Num underruns detected: avg {}, min {}, max {}, non-zero {} + Num late commands: avg {}, min {}, max {}, non-zero {} + Num timeouts (Tx): avg {}, min {}, max {}, non-zero {} + Num timeouts (Rx): avg {}, min {}, max {}, non-zero {} +""".format( + summary.avg_vals.received_samps, + summary.min_vals.received_samps, + summary.max_vals.received_samps, + summary.non_zero_vals.received_samps, + summary.avg_vals.dropped_samps, + summary.min_vals.dropped_samps, + summary.max_vals.dropped_samps, + summary.non_zero_vals.dropped_samps, + summary.avg_vals.overruns, + summary.min_vals.overruns, + summary.max_vals.overruns, + summary.non_zero_vals.overruns, + summary.avg_vals.transmitted_samps, + summary.min_vals.transmitted_samps, + summary.max_vals.transmitted_samps, + summary.non_zero_vals.transmitted_samps, + summary.avg_vals.tx_seq_errs, + summary.min_vals.tx_seq_errs, + summary.max_vals.tx_seq_errs, + summary.non_zero_vals.tx_seq_errs, + summary.avg_vals.rx_seq_errs, + summary.min_vals.rx_seq_errs, + summary.max_vals.rx_seq_errs, + summary.non_zero_vals.rx_seq_errs, + summary.avg_vals.underruns, + summary.min_vals.underruns, + summary.max_vals.underruns, + summary.non_zero_vals.underruns, + summary.avg_vals.late_cmds, + summary.min_vals.late_cmds, + summary.max_vals.late_cmds, + summary.non_zero_vals.late_cmds, + summary.avg_vals.tx_timeouts, + summary.min_vals.tx_timeouts, + summary.max_vals.tx_timeouts, + summary.non_zero_vals.tx_timeouts, + summary.avg_vals.rx_timeouts, + summary.min_vals.rx_timeouts, + summary.max_vals.rx_timeouts, + summary.non_zero_vals.rx_timeouts) + + if params is not None: + duration = 10 + if "duration" in params: + duration = int(params["duration"]) + + num_rx_channels = summary.min_vals.num_rx_channels + rx_rate = summary.min_vals.rx_rate + + if num_rx_channels != 0: + avg_samps = summary.avg_vals.received_samps + expected_samps = num_rx_channels * duration * rx_rate + percent = (avg_samps) / expected_samps * 100 + statistics_msg += " Expected samps (Rx): {}\n".format(expected_samps) + statistics_msg += " Actual samps % (Rx): {}\n".format(round(percent, 1)) + + num_tx_channels = summary.min_vals.num_tx_channels + tx_rate = summary.min_vals.tx_rate + + if num_tx_channels != 0: + avg_samps = summary.avg_vals.transmitted_samps + expected_samps = num_tx_channels * duration * tx_rate + percent = (avg_samps) / expected_samps * 100 + statistics_msg += " Expected samps (Tx): {}\n".format(expected_samps) + statistics_msg += " Actual samps % (Tx): {}\n".format(round(percent, 1)) + + return statistics_msg + +def parse_args(): + """ + Parse the command line arguments for batch run benchmark rate. + """ + benchmark_rate_params, rest = run_benchmark_rate.parse_known_args() + parser = argparse.ArgumentParser() + parser.add_argument("--path", type=str, required=True, help="path to benchmark rate example") + parser.add_argument("--iterations", type=int, default=100, help="number of iterations to run") + params = parser.parse_args(rest) + return params.path, params.iterations, benchmark_rate_params + +if __name__ == "__main__": + path, iterations, params = parse_args(); + results = run(path, iterations, params) + summary = calculate_stats(results) + print(get_summary_string(summary, params)) diff --git a/host/tests/streaming_performance/parse_benchmark_rate.py b/host/tests/streaming_performance/parse_benchmark_rate.py new file mode 100644 index 000000000..e30c3e6aa --- /dev/null +++ b/host/tests/streaming_performance/parse_benchmark_rate.py @@ -0,0 +1,149 @@ +""" +Copyright 2019 Ettus Research, A National Instrument Brand + +SPDX-License-Identifier: GPL-3.0-or-later + +Helper script that parses the results of benchmark_rate and extracts numeric +values printed at the end of execution. +""" + +import collections +import re +import csv + +Results = collections.namedtuple( + 'Results', + """ + num_rx_channels + num_tx_channels + rx_rate + tx_rate + received_samps + dropped_samps + overruns + transmitted_samps + tx_seq_errs + rx_seq_errs + underruns + late_cmds + tx_timeouts + rx_timeouts + """ +) + +def average(results): + """ + Returns the average of a list of results. + """ + results_as_lists = [list(r) for r in results] + avg_vals = [sum(x)/len(results) for x in zip(*results_as_lists)] + return Results(*avg_vals) + +def min_vals(results): + """ + Returns the minimum values of a list of results. + """ + results_as_lists = [list(r) for r in results] + min_vals = [min(x) for x in zip(*results_as_lists)] + return Results(*min_vals) + +def max_vals(results): + """ + Returns the maximum values of a list of results. + """ + results_as_lists = [list(r) for r in results] + max_vals = [max(x) for x in zip(*results_as_lists)] + return Results(*max_vals) + +def non_zero_vals(results): + """ + Returns the number of non-zero values from a list of results. + """ + results_as_lists = [list(r) for r in results] + results_as_lists = [[1 if x > 0 else 0 for x in y] for y in results_as_lists] + non_zero_vals = [sum(x) for x in zip(*results_as_lists)] + return Results(*non_zero_vals) + +def parse(result_str): + """ + Parses benchmark results and returns numerical values. + """ + # Parse rx rate + rx_rate = 0.0 + num_rx_channels = 0 + expr = "Testing receive rate ([0-9]+\.[0-9]+) Msps on (\d+) channels" + match = re.search(expr, result_str) + if match is not None: + rx_rate = float(match.group(1)) * 1.0e6 + num_rx_channels = int(match.group(2)) + + tx_rate = 0.0 + num_tx_channels = 0 + expr = "Testing transmit rate ([0-9]+\.[0-9]+) Msps on (\d+) channels" + match = re.search(expr, result_str) + if match is not None: + tx_rate = float(match.group(1)) * 1.0e6 + num_tx_channels = int(match.group(2)) + + # Parse results + expr = "Benchmark rate summary:" + expr += r"\s*Num received samples:\s*(\d+)" + expr += r"\s*Num dropped samples:\s*(\d+)" + expr += r"\s*Num overruns detected:\s*(\d+)" + expr += r"\s*Num transmitted samples:\s*(\d+)" + expr += r"\s*Num sequence errors \(Tx\):\s*(\d+)" + expr += r"\s*Num sequence errors \(Rx\):\s*(\d+)" + expr += r"\s*Num underruns detected:\s*(\d+)" + expr += r"\s*Num late commands:\s*(\d+)" + expr += r"\s*Num timeouts \(Tx\):\s*(\d+)" + expr += r"\s*Num timeouts \(Rx\):\s*(\d+)" + match = re.search(expr, result_str) + if match: + return Results( + num_rx_channels = num_rx_channels, + num_tx_channels = num_tx_channels, + rx_rate = rx_rate, + tx_rate = tx_rate, + received_samps = int(match.group(1)), + dropped_samps = int(match.group(2)), + overruns = int(match.group(3)), + transmitted_samps = int(match.group(4)), + tx_seq_errs = int(match.group(5)), + rx_seq_errs = int(match.group(6)), + underruns = int(match.group(7)), + late_cmds = int(match.group(8)), + tx_timeouts = int(match.group(9)), + rx_timeouts = int(match.group(10)) + ) + else: + return None + +def write_benchmark_rate_csv(results, file_name): + with open(file_name, 'w', newline='') as f: + w = csv.writer(f) + w.writerow(results[0]._fields) + w.writerows(results) + +if __name__ == "__main__": + result_str = """ + [00:00:00.000376] Creating the usrp device with: addr=192.168.30.2, second_addr=192.168.40.2... + [00:00:05.63100253] Testing receive rate 200.000000 Msps on 2 channels + [00:00:05.73100253] Testing transmit rate 100.000000 Msps on 1 channels + [00:00:15.113339078] Benchmark complete. + + Benchmark rate summary: + Num received samples: 10000 + Num dropped samples: 200 + Num overruns detected: 10 + Num transmitted samples: 20000 + Num sequence errors (Tx): 5 + Num sequence errors (Rx): 6 + Num underruns detected: 20 + Num late commands: 2 + Num timeouts (Tx): 0 + Num timeouts (Rx): 100 + + Done! + """ + print("Parsing hardcoded string for testing only") + print(parse(result_str)) diff --git a/host/tests/streaming_performance/run_benchmark_rate.py b/host/tests/streaming_performance/run_benchmark_rate.py new file mode 100644 index 000000000..1989b6d92 --- /dev/null +++ b/host/tests/streaming_performance/run_benchmark_rate.py @@ -0,0 +1,86 @@ +""" +Copyright 2019 Ettus Research, A National Instrument Brand + +SPDX-License-Identifier: GPL-3.0-or-later + +Helper script that provides a Python interface to run the benchmark rate C++ +example. +""" +import argparse +import subprocess + +def run(path, params): + """ + Run benchmark rate and return a CompletedProcess object. + """ + proc_params = [path] + + for key, val in params.items(): + proc_params.append("--" + str(key)) + proc_params.append(str(val)) + + return subprocess.run(proc_params, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--args", type=str, help="single uhd device address args") + parser.add_argument("--duration", type=str, help="duration for the test in seconds") + parser.add_argument("--rx_subdev", type=str, help="specify the device subdev for RX") + parser.add_argument("--tx_subdev", type=str, help="specify the device subdev for TX") + parser.add_argument("--rx_stream_args", type=str, help="stream args for RX streamer") + parser.add_argument("--tx_stream_args", type=str, help="stream args for TX streamer") + parser.add_argument("--rx_rate", type=str, help="specify to perform a RX rate test (sps)") + parser.add_argument("--tx_rate", type=str, help="specify to perform a TX rate test (sps)") + parser.add_argument("--rx_otw", type=str, help="specify the over-the-wire sample mode for RX") + parser.add_argument("--tx_otw", type=str, help="specify the over-the-wire sample mode for TX") + parser.add_argument("--rx_cpu", type=str, help="specify the host/cpu sample mode for RX") + parser.add_argument("--tx_cpu", type=str, help="specify the host/cpu sample mode for TX") + parser.add_argument("--ref", type=str, help="clock reference (internal, external, mimo, gpsdo)") + parser.add_argument("--pps", type=str, help="PPS source (internal, external, mimo, gpsdo)") + parser.add_argument("--random", type=str, help="Run with random values of samples in send() and recv()") + parser.add_argument("--rx_channels", type=str, help="which RX channel(s) to use") + parser.add_argument("--tx_channels", type=str, help="which TX channel(s) to use") + return parser + +def parse_args(): + """ + Parse the command line arguments for benchmark rate, and returns arguments + in a dict. + """ + parser = create_parser() + params = vars(parser.parse_args()) + return { key : params[key] for key in params if params[key] != None } + +def parse_known_args(): + """ + Parse the command line arguments for benchmark rate. Returns a dict + containing the benchmark rate args, and a list containing args that + are not recognized by benchmark rate. + """ + parser = create_parser() + params, rest = parser.parse_known_args() + params = vars(params) + return { key : params[key] for key in params if params[key] != None }, rest + +if __name__ == "__main__": + """ + Main function used for testing only. Requires path to the example to be + passed as a command line parameter. + """ + benchmark_rate_params, rest = parse_known_args() + parser = argparse.ArgumentParser() + parser.add_argument("--path", type=str, required=True, help="path to benchmark rate example") + params = parser.parse_args(rest); + proc = run(params.path, benchmark_rate_params) + + print("ARGS") + print(proc.args) + + print("RETURNCODE") + print(proc.returncode) + + print("STDERR") + print(proc.stderr.decode('ASCII')) + + print("STDOUT") + print(proc.stdout.decode('ASCII')) -- cgit v1.2.3