#!/usr/bin/env python3 """ Copyright 2019 Ettus Research, A National Instrument Brand SPDX-License-Identifier: GPL-3.0-or-later Runs the benchmark rate C++ example for a specified number of iterations and aggregates results. Example usage: batch_run_benchmark_rate.py --path /benchmark_rate --iterations 1 --args "addr=192.168.30.2" --rx_rate 1e6 """ import argparse import collections import re import parse_benchmark_rate import run_benchmark_rate Results = collections.namedtuple( 'Results', """ avg_vals min_vals max_vals non_zero_vals """ ) def calculate_stats(results): """ Calculates performance metrics from list of parsed benchmark rate results. """ result_avg = parse_benchmark_rate.average(results) result_min = parse_benchmark_rate.min_vals(results) result_max = parse_benchmark_rate.max_vals(results) result_nz = parse_benchmark_rate.non_zero_vals(results) return Results( avg_vals = result_avg, min_vals = result_min, max_vals = result_max, non_zero_vals = result_nz) def run(path, iterations, benchmark_rate_params, stop_on_error=True): """ Runs benchmark rate multiple times and returns a list of parsed results. """ print("Running benchmark rate {} times with the following arguments: ".format(iterations)) for key, val in benchmark_rate_params.items(): print("{:14} {}".format(key, val)) parsed_results = [] iteration = 0 while iteration < iterations: proc = run_benchmark_rate.run(path, benchmark_rate_params) result = parse_benchmark_rate.parse(proc.stdout.decode('ASCII')) if result != None: parsed_results.append(result) iteration += 1 else: if stop_on_error: msg = "Could not parse results of benchmark_rate\n" msg += "Benchmark rate arguments:\n" msg += str(proc.args) + "\n" msg += "Stderr capture:\n" msg += proc.stderr.decode('ASCII') msg += "Stdout capture:\n" msg += proc.stdout.decode('ASCII') raise RuntimeError(msg) else: print("Failed to parse benchmark rate results") print(proc.stderr.decode('ASCII')) return parsed_results def get_summary_string(stats, iterations, params): """ Returns summary info in a table format. """ header = "| stat | rx samps | tx samps | rx dropped | overrun | rx seq | tx seq | underrun | rx tmo | tx tmo | late |" ruler = "|------|----------|----------|------------|---------|--------|--------|----------|--------|--------|------|" def get_params_row(results, iterations, duration, mcr): """ Returns a row containing the test setup, e.g.: 1 rx, 1 tx, rate 6.452e+06 sps, 1 iterations, 10s duration """ rate = max(results.rx_rate, results.tx_rate) s = "" s += "{} rx".format(int(results.num_rx_channels)) s += ", " s += "{} tx".format(int(results.num_tx_channels)) s += ", " s += "rate {:.3e} sps".format(round(rate, 2)) s += ", " s += "{} iterations".format(iterations) s += ", " s += "{}s duration".format(duration) if mcr is not None: s += ", " s += "mcr {}".format(mcr) return "| " + s + " "*(len(ruler)-len(s)-3) + "|" def get_table_row(results, iterations, duration, stat_label): """ Returns a row of numeric results. """ expected_samps = results.num_rx_channels * duration * results.rx_rate rx_samps = 0 rx_dropped = 0 if expected_samps > 0: rx_samps = results.received_samps / expected_samps * 100 rx_dropped = results.dropped_samps / expected_samps * 100 tx_samps = 0 expected_samps = results.num_tx_channels * duration * results.tx_rate if expected_samps > 0: tx_samps = results.transmitted_samps / expected_samps * 100 s = ( "| {} ".format(stat_label) + "| {:>8} ".format(round(rx_samps, 1)) + "| {:>8} ".format(round(tx_samps, 1)) + "| {:>10} ".format(round(rx_dropped, 1)) + "| {:>7} ".format(round(results.overruns, 1)) + "| {:>6} ".format(round(results.rx_seq_errs, 1)) + "| {:>6} ".format(round(results.tx_seq_errs, 1)) + "| {:>8.1e} ".format(round(results.underruns, 1)) + "| {:>6} ".format(round(results.rx_timeouts, 1)) + "| {:>6} ".format(round(results.tx_timeouts, 1)) + "| {:>4} ".format(round(results.late_cmds, 1)) ) return s + "|" def get_non_zero_row(results): """ Returns a row with the number of non-zero values for each value. """ s = ( "| nz " + "| {:>8} ".format(int(results.received_samps)) + "| {:>8} ".format(int(results.transmitted_samps)) + "| {:>10} ".format(int(results.dropped_samps)) + "| {:>7} ".format(int(results.overruns)) + "| {:>6} ".format(int(results.rx_seq_errs)) + "| {:>6} ".format(int(results.tx_seq_errs)) + "| {:>8} ".format(int(results.underruns)) + "| {:>6} ".format(int(results.rx_timeouts)) + "| {:>6} ".format(int(results.tx_timeouts)) + "| {:>4} ".format(int(results.late_cmds)) ) return s + "|" duration = 10 if "duration" in params: duration = int(params["duration"]) mcr = None if "args" in params: args = params["args"] expr = "" expr += r"master_clock_rate\s*=\s*(\d[\deE+-.]*)" match = re.search(expr, args) if match: mcr = match.group(1) s = "" s += header + "\n" s += ruler + "\n" s += get_params_row(stats.avg_vals, iterations, duration, mcr) + "\n" s += get_table_row(stats.avg_vals, iterations, duration, "avg") + "\n" s += get_table_row(stats.min_vals, iterations, duration, "min") + "\n" s += get_table_row(stats.max_vals, iterations, duration, "max") + "\n" s += get_non_zero_row(stats.non_zero_vals) + "\n" return s def parse_args(): """ Parse the command line arguments for batch run benchmark rate. """ benchmark_rate_params, rest = run_benchmark_rate.parse_known_args() parser = argparse.ArgumentParser() parser.add_argument("--path", type=str, required=True, help="path to benchmark rate example") parser.add_argument("--iterations", type=int, default=100, help="number of iterations to run") params = parser.parse_args(rest) return params.path, params.iterations, benchmark_rate_params if __name__ == "__main__": path, iterations, params = parse_args(); results = run(path, iterations, params) stats = calculate_stats(results) print(get_summary_string(stats, iterations, params))