diff options
Diffstat (limited to 'fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb')
10 files changed, 1261 insertions, 0 deletions
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/Makefile b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/Makefile new file mode 100644 index 000000000..7fa7ae03b --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/Makefile @@ -0,0 +1,52 @@ +# +# Copyright 2015 Ettus Research LLC +# + +#------------------------------------------------- +# Top-of-Makefile +#------------------------------------------------- +# Define BASE_DIR to point to the "top" dir +BASE_DIR = $(abspath ../../../../top) +# Include viv_sim_preamble after defining BASE_DIR +include $(BASE_DIR)/../tools/make/viv_sim_preamble.mak + +#------------------------------------------------- +# Design Specific +#------------------------------------------------- +# Define part using PART_ID (<device>/<package>/<speedgrade>) +ARCH = kintex7 +PART_ID = xc7k410t/ffg900/-2 + +# Include makefiles and sources for the DUT and its dependencies +include $(BASE_DIR)/../lib/control/Makefile.srcs +include $(BASE_DIR)/../lib/fifo/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/crossbar/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/core/Makefile.srcs + +DESIGN_SRCS = $(abspath \ +$(FIFO_SRCS) \ +$(CONTROL_LIB_SRCS) \ +$(RFNOC_XBAR_SRCS) \ +$(RFNOC_CORE_SRCS) \ +) + +#------------------------------------------------- +# Testbench Specific +#------------------------------------------------- +# Define only one toplevel module +TB_TOP_MODULE ?= crossbar_tb +SIM_TOP = $(TB_TOP_MODULE) + +SIM_SRCS = \ +$(abspath chdr_traffic_source_sim.sv) \ +$(abspath chdr_traffic_sink_sim.sv) \ +$(abspath crossbar_tb.sv) \ +$(abspath $(TB_TOP_MODULE).sv) + +#------------------------------------------------- +# Bottom-of-Makefile +#------------------------------------------------- +# Include all simulator specific makefiles here +# Each should define a unique target to simulate +# e.g. xsim, vsim, etc and a common "clean" target +include $(BASE_DIR)/../tools/make/viv_simulator.mak diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/Makefile b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/Makefile new file mode 100644 index 000000000..0f1a10a6e --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/Makefile @@ -0,0 +1,51 @@ +# +# Copyright 2015 Ettus Research LLC +# + +#------------------------------------------------- +# Top-of-Makefile +#------------------------------------------------- +# Define BASE_DIR to point to the "top" dir +BASE_DIR = $(abspath ../../../../../top) +# Include viv_sim_preamble after defining BASE_DIR +include $(BASE_DIR)/../tools/make/viv_sim_preamble.mak + +#------------------------------------------------- +# Design Specific +#------------------------------------------------- +# Define part using PART_ID (<device>/<package>/<speedgrade>) +ARCH = kintex7 +PART_ID = xc7k410t/ffg900/-2 + +# Include makefiles and sources for the DUT and its dependencies +include $(BASE_DIR)/../lib/control/Makefile.srcs +include $(BASE_DIR)/../lib/fifo/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/crossbar/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/core/Makefile.srcs + +DESIGN_SRCS = $(abspath \ +$(FIFO_SRCS) \ +$(CONTROL_LIB_SRCS) \ +$(RFNOC_XBAR_SRCS) \ +$(RFNOC_CORE_SRCS) \ +) + +#------------------------------------------------- +# Testbench Specific +#------------------------------------------------- +# Define only one toplevel module +SIM_TOP = axis_ctrl_crossbar_nxn_tb + +SIM_SRCS = \ +$(abspath axis_ctrl_crossbar_nxn_tb.sv) \ +$(abspath ../crossbar_tb.sv) \ +$(abspath ../chdr_traffic_source_sim.sv) \ +$(abspath ../chdr_traffic_sink_sim.sv) + +#------------------------------------------------- +# Bottom-of-Makefile +#------------------------------------------------- +# Include all simulator specific makefiles here +# Each should define a unique target to simulate +# e.g. xsim, vsim, etc and a common "clean" target +include $(BASE_DIR)/../tools/make/viv_simulator.mak diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/axis_ctrl_crossbar_nxn_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/axis_ctrl_crossbar_nxn_tb.sv new file mode 100644 index 000000000..fa112f5cb --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/axis_ctrl_crossbar_nxn_tb/axis_ctrl_crossbar_nxn_tb.sv @@ -0,0 +1,26 @@ +// +// Copyright 2018 Ettus Research, A National Instruments Company +// +// SPDX-License-Identifier: LGPL-3.0-or-later + + +`timescale 1ns/1ps + +module axis_ctrl_crossbar_nxn_tb(); + crossbar_tb #( + .TEST_NAME ("axis_ctrl_crossbar_nxn_tb"), + .ROUTER_IMPL ("axis_ctrl_2d_torus" ), // Router implementation + .ROUTER_PORTS (20 ), // Number of ports + .ROUTER_DWIDTH (64 ), // Router datapath width + .MTU_LOG2 (5 ), // log2 of max packet size for router + .NUM_MASTERS (4 ), // Number of data generators in test + .TEST_MAX_PACKETS (100 ), // How many packets to stream per test case? + .TEST_LPP (20 ), // Lines per packet + .TEST_MIN_INJ_RATE (10 ), // Minimum injection rate to test + .TEST_MAX_INJ_RATE (40 ), // Maximum injection rate to test + .TEST_INJ_RATE_INCR (10 ), // Injection rate increment + .TEST_GEN_LL_FILES (0 ) // Generate files to produce load-latency graphs? + ) impl ( + /* no IO */ + ); +endmodule diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile new file mode 100644 index 000000000..399515640 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile @@ -0,0 +1,51 @@ +# +# Copyright 2015 Ettus Research LLC +# + +#------------------------------------------------- +# Top-of-Makefile +#------------------------------------------------- +# Define BASE_DIR to point to the "top" dir +BASE_DIR = $(abspath ../../../../../top) +# Include viv_sim_preamble after defining BASE_DIR +include $(BASE_DIR)/../tools/make/viv_sim_preamble.mak + +#------------------------------------------------- +# Design Specific +#------------------------------------------------- +# Define part using PART_ID (<device>/<package>/<speedgrade>) +ARCH = kintex7 +PART_ID = xc7k410t/ffg900/-2 + +# Include makefiles and sources for the DUT and its dependencies +include $(BASE_DIR)/../lib/control/Makefile.srcs +include $(BASE_DIR)/../lib/fifo/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/crossbar/Makefile.srcs +include $(BASE_DIR)/../lib/rfnoc/core/Makefile.srcs + +DESIGN_SRCS = $(abspath \ +$(FIFO_SRCS) \ +$(CONTROL_LIB_SRCS) \ +$(RFNOC_XBAR_SRCS) \ +$(RFNOC_CORE_SRCS) \ +) + +#------------------------------------------------- +# Testbench Specific +#------------------------------------------------- +# Define only one toplevel module +SIM_TOP = chdr_crossbar_nxn_tb + +SIM_SRCS = \ +$(abspath chdr_crossbar_nxn_tb.sv) \ +$(abspath ../crossbar_tb.sv) \ +$(abspath ../chdr_traffic_source_sim.sv) \ +$(abspath ../chdr_traffic_sink_sim.sv) + +#------------------------------------------------- +# Bottom-of-Makefile +#------------------------------------------------- +# Include all simulator specific makefiles here +# Each should define a unique target to simulate +# e.g. xsim, vsim, etc and a common "clean" target +include $(BASE_DIR)/../tools/make/viv_simulator.mak diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv new file mode 100644 index 000000000..1c5cace63 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv @@ -0,0 +1,26 @@ +// +// Copyright 2018 Ettus Research, A National Instruments Company +// +// SPDX-License-Identifier: LGPL-3.0-or-later + + +`timescale 1ns/1ps + +module chdr_crossbar_nxn_tb(); + crossbar_tb #( + .TEST_NAME ("chdr_crossbar_nxn_tb"), + .ROUTER_IMPL ("chdr_crossbar_nxn" ), // Router implementation + .ROUTER_PORTS (10 ), // Number of ports + .ROUTER_DWIDTH (64 ), // Router datapath width + .MTU_LOG2 (7 ), // log2 of max packet size for router + .NUM_MASTERS (10 ), // Number of data generators in test + .TEST_MAX_PACKETS (100 ), // How many packets to stream per test case? + .TEST_LPP (100 ), // Lines per packet + .TEST_MIN_INJ_RATE (60 ), // Minimum injection rate to test + .TEST_MAX_INJ_RATE (100 ), // Maximum injection rate to test + .TEST_INJ_RATE_INCR (10 ), // Injection rate increment + .TEST_GEN_LL_FILES (0 ) // Generate files to produce load-latency graphs? + ) impl ( + /* no IO */ + ); +endmodule diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv new file mode 100644 index 000000000..a9fe3ba27 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv @@ -0,0 +1,150 @@ +// +// Copyright 2018 Ettus Research, A National Instruments Company +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// Module: chdr_traffic_sink_sim +// Description: +// A sink for CHDR traffic. Simulation only. +// Accepts packets and computes the following metrics: +// - Data integrity errors +// - Packet latency +// - Throughput counts +// All metrics can optionally be written to a file to +// generate load-latency graphs. + +`timescale 1ns/1ps + +`include "sim_cvita_lib.svh" + +module chdr_traffic_sink_sim #( + parameter WIDTH = 64, + parameter MTU = 5, + parameter [15:0] NODE_ID = 'd0, + parameter [15:0] NUM_NODES = 'd16, + parameter FILE_PATH = ".", + parameter FLUSH_N = 4 +) ( + // Clocks and resets + input clk, + input rst, + // Settings + input [63:0] current_time, + input start_stb, + input [7:0] injection_rate, + input [15:0] lines_per_pkt, + input [7:0] traffic_patt, + // CHDR master interface + input [WIDTH-1:0] s_axis_tdata, + input s_axis_tlast, + input s_axis_tvalid, + output s_axis_tready, + // Metrics + output session_active, + output [31:0] xfer_count, + output [31:0] pkt_count, + output [31:0] data_err_count, + output [31:0] route_err_count +); + + // Constants + localparam integer ERR_BIT_PKT_SIZE_MISMATCH = 1; + localparam integer ERR_BIT_PKT_DATA_MISMATCH = 2; + localparam integer ERR_BIT_PKT_DEST_MISMATCH = 4; + localparam integer ERR_BIT_PKT_SEQUENCE_ERR = 8; + + cvita_slave #(.DWIDTH(WIDTH)) s_chdr (.clk(clk)); + cvita_pkt_t pkt; + + assign s_chdr.axis.tdata = s_axis_tdata; + assign s_chdr.axis.tlast = s_axis_tlast; + assign s_chdr.axis.tvalid = s_axis_tvalid; + assign s_axis_tready = s_chdr.axis.tready; + + logic running = 0; + integer num_data_errs = 0; + integer num_route_errs = 0; + logic [31:0] num_pkts_xferd = 0; + logic [31:0] num_samps_xferd = 0; + + assign data_err_count = num_data_errs; + assign route_err_count = num_route_errs; + assign xfer_count = num_samps_xferd; + assign pkt_count = num_pkts_xferd; + assign session_active = running; + + integer session = 0; + string filename; + integer handle = 0; + integer err = 0; + integer bus_idle_cnt = 0; + logic [WIDTH-1:0] i; + + // Egress buff in source is MTU + 4 + localparam integer IDLE_TIMEOUT = (1 << (MTU + 4 + FLUSH_N)); + + initial begin: consume_blk + // Consume infinitely + s_chdr.reset(); + while (1) begin + // A session begins on the posedge of start_stb + while (~start_stb) @(posedge clk); + session = session + 1; + $sformat(filename, "%s/pkts_node%05d_inj%03d_lpp%05d_traffic%c_sess%04d.csv", + FILE_PATH, NODE_ID, injection_rate, lines_per_pkt, traffic_patt, session); + if (FILE_PATH != "") begin + handle = $fopen(filename, "w"); + if (handle == 0) begin + $error("Could not open file: %s", filename); + $finish(); + end + end + if (handle != 0) $fdisplay(handle, "Src,Dst,Seqno,Error,Latency"); + s_chdr.reset(); + num_data_errs = 0; + num_route_errs = 0; + num_pkts_xferd = 0; + num_samps_xferd = 0; + bus_idle_cnt = 0; + running = 1; + while (1) begin + // Pull packet from bus + err = 0; + if (~s_chdr.axis.tvalid[0]) begin + @(posedge clk); + bus_idle_cnt = bus_idle_cnt + 1; + if (bus_idle_cnt <= IDLE_TIMEOUT) + continue; + else + break; + end + s_chdr.pull_pkt(pkt, 0); + bus_idle_cnt = 0; + num_pkts_xferd = num_pkts_xferd + 1; + num_samps_xferd = num_samps_xferd + lines_per_pkt; + // Validate packet + if (pkt.hdr.dst_sid != NODE_ID) begin + err = err + ERR_BIT_PKT_DEST_MISMATCH; + num_route_errs = num_route_errs + 1; + end + if (pkt.payload.size() != lines_per_pkt-2) begin + err = err + ERR_BIT_PKT_SIZE_MISMATCH; + num_data_errs = num_data_errs + 1; + end else begin + for (i = 'd0; i < (lines_per_pkt-2); i=i+1) begin + if (pkt.payload[i] != i) begin + err = err + ERR_BIT_PKT_DATA_MISMATCH; + num_data_errs = num_data_errs + 1; + break; + end + end + end + if (handle != 0) $fdisplay(handle, "%00d,%00d,%00d,%00d,%00d", + pkt.hdr.src_sid, pkt.hdr.dst_sid, pkt.hdr.seqnum, err, (current_time - pkt.hdr.timestamp)); + end + running = 0; + if (handle != 0) $fclose(handle); + end + end + +endmodule
\ No newline at end of file diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv new file mode 100644 index 000000000..8c3d974c9 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv @@ -0,0 +1,202 @@ +// +// Copyright 2018 Ettus Research, A National Instruments Company +// +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// Module: chdr_traffic_source_sim +// Description: +// A traffic generator for CHDR traffic. Simulation only. +// Supports multiple traffic pattern and injection rates. +// + +`timescale 1ns/1ps + +`include "sim_cvita_lib.svh" + +module chdr_traffic_source_sim #( + parameter WIDTH = 64, // Width of the AXI-Stream data bus + parameter MTU = 5, // log2 of the max number of lines in a packet + parameter [15:0] NODE_ID = 'd0, // Node ID for this generator + parameter [15:0] NUM_NODES = 'd16 // Total number of generators in the application +) ( + // Clocks and resets + input clk, // AXI-Stream clock + input rst, // AXI-Stream reset + // Settings + input [63:0] current_time, // The current value of the global timebase (synch to clk) + input start_stb, // A strobe that indicates the start of a generation session + input [7:0] injection_rate, // The inject rate (in percent) to simulate + input [15:0] lines_per_pkt, // Number of lines per packet to generate + input [7:0] traffic_patt, // The traffic pattern (see localparams below for values) + input [31:0] num_pkts_to_send, // Number of packets to send + // CHDR master interface + output [WIDTH-1:0] m_axis_tdata, // AXI-Stream master tdata + output m_axis_tlast, // AXI-Stream master tlast + output m_axis_tvalid, // AXI-Stream master tvalid + input m_axis_tready, // AXI-Stream master tready + // Metrics + output session_active, // Signal indicating if generation session is active + output [63:0] session_duration, // Session duration (only valid after session ends) + output [31:0] xfer_count, // Number of lines transferred (only valid after session ends) + output [31:0] pkt_count // Number of packets transferred (only valid after session ends) +); + // **** Supported Traffic Patters **** + localparam [7:0] TRAFFIC_PATT_LOOPBACK = 8'd76; //L + localparam [7:0] TRAFFIC_PATT_NEIGHBOR = 8'd78; //N + localparam [7:0] TRAFFIC_PATT_BIT_COMPLEMENT = 8'd67; //C + localparam [7:0] TRAFFIC_PATT_SEQUENTIAL = 8'd83; //S + localparam [7:0] TRAFFIC_PATT_UNIFORM = 8'd85; //U + localparam [7:0] TRAFFIC_PATT_UNIFORM_OTHERS = 8'd79; //O + localparam [7:0] TRAFFIC_PATT_RANDOM_PERM = 8'd82; //R + + cvita_master #(.DWIDTH(WIDTH)) m_chdr (.clk(clk)); + axis_t #(.DWIDTH(WIDTH)) post_fifo (.clk(clk)); + axis_t #(.DWIDTH(WIDTH)) pre_gate (.clk(clk)); + cvita_hdr_t header; + reg throttle = 1'b1; + + logic running = 0; + logic [31:0] curr_pkt_num = 'd0; + logic [31:0] num_samps_xferd = 'd0; + logic [63:0] start_time = 0; + logic [63:0] stop_time = 0; + logic [15:0] last_gen_sid = (NODE_ID - 16'd1); + + assign xfer_count = num_samps_xferd; + assign pkt_count = curr_pkt_num; + assign session_duration = (stop_time - start_time); + assign session_active = running; + + // Utility function to assign SIDs based on traffic pattern + function [15:0] gen_dst_sid; + input [7:0] traffic_patt; + input [15:0] last_sid; + + if (traffic_patt == TRAFFIC_PATT_UNIFORM) begin + gen_dst_sid = $urandom_range('d0, NUM_NODES-'d1); + end else if (traffic_patt == TRAFFIC_PATT_UNIFORM_OTHERS) begin + logic [31:0] rnum = $urandom_range('d0, NUM_NODES-'d2); + if (rnum < NODE_ID) + gen_dst_sid = rnum[15:0]; + else + gen_dst_sid = rnum[15:0] + 16'd1; + end else if (traffic_patt == TRAFFIC_PATT_SEQUENTIAL) begin + gen_dst_sid = (last_sid + 16'd1) % NUM_NODES; + end else if (traffic_patt == TRAFFIC_PATT_NEIGHBOR) begin + gen_dst_sid = (NODE_ID + 16'd1) % NUM_NODES; + end else if (traffic_patt == TRAFFIC_PATT_LOOPBACK) begin + gen_dst_sid = NODE_ID; + end else if (traffic_patt == TRAFFIC_PATT_BIT_COMPLEMENT) begin + gen_dst_sid = (NUM_NODES - NODE_ID - 1) % NUM_NODES; + end else if (traffic_patt == TRAFFIC_PATT_RANDOM_PERM) begin + //TODO: Implement me + gen_dst_sid = 0; + end else begin + gen_dst_sid = 'd0; + end + endfunction + + // Generation loop. Push to m_chdr infinitely fast + initial begin: gen_blk + // Generate infinitely + $srandom(NODE_ID + NUM_NODES); + m_chdr.reset(); + while (1) begin + // A generation session begins on the posedge of start_stb + while (~start_stb) @(posedge clk); + curr_pkt_num = 'd0; + m_chdr.reset(); + num_samps_xferd = 'd0; + start_time = current_time; + running = 1; + while (curr_pkt_num < num_pkts_to_send) begin + header = '{ + pkt_type:DATA, has_time:1, eob:0, + seqnum:curr_pkt_num[11:0], length:(lines_per_pkt*8), + src_sid:NODE_ID, dst_sid:gen_dst_sid(traffic_patt, last_gen_sid), + timestamp:0 //TS attached later + }; + last_gen_sid = header.dst_sid; + curr_pkt_num = curr_pkt_num + 'd1; + m_chdr.push_ramp_pkt(lines_per_pkt-2, 'h0, 'h1, header); + num_samps_xferd = num_samps_xferd + lines_per_pkt; + end + running = 0; + stop_time = current_time; + end + end + + // Capture packets in a really short FIFO (for backpressure) + axi_fifo #( + .WIDTH(WIDTH+1), .SIZE(MTU + 1) + ) fifo_i ( + .clk (clk), + .reset (rst), + .clear (1'b0), + .i_tdata ({m_chdr.axis.tlast, m_chdr.axis.tdata}), + .i_tvalid (m_chdr.axis.tvalid), + .i_tready (m_chdr.axis.tready), + .o_tdata ({post_fifo.tlast, post_fifo.tdata}), + .o_tvalid (post_fifo.tvalid), + .o_tready (post_fifo.tready), + .space (), + .occupied () + ); + + // Attach timestamp after the packet leaves the FIFO after + // throttling. + + localparam [1:0] ST_HDR = 2'd0; + localparam [1:0] ST_TS = 2'd1; + localparam [1:0] ST_BODY = 2'd2; + + reg [1:0] pkt_state = ST_HDR; + always_ff @(posedge clk) begin + if (rst) begin + pkt_state <= ST_HDR; + end else if (pre_gate.tvalid & pre_gate.tready) begin + case (pkt_state) + ST_HDR: + if (~pre_gate.tlast) + pkt_state <= pre_gate.tdata[61] ? ST_TS : ST_BODY; + ST_TS: + pkt_state <= pre_gate.tlast ? ST_HDR : ST_BODY; + ST_BODY: + pkt_state <= pre_gate.tlast ? ST_HDR : ST_BODY; + default: + pkt_state <= ST_HDR; + endcase + end + end + + // Enforce injection rate by pulling from FIFO with a certain time probability + always_ff @(posedge clk) begin + throttle <= ($urandom_range(32'd99, 32'd0) > {24'h0, injection_rate}); + end + + // Insert timestamp + throttle logic + assign pre_gate.tdata = (pkt_state == ST_TS) ? current_time : post_fifo.tdata; + assign pre_gate.tlast = post_fifo.tlast; + assign pre_gate.tvalid = post_fifo.tvalid & ~throttle; + assign post_fifo.tready = pre_gate.tready & ~throttle; + + // Gate the packet to smooth out throttle-related noise. + // This also serves as a buffer for the packet in case things are backed up + axi_packet_gate #( + .WIDTH(WIDTH), .SIZE(MTU + 4), .USE_AS_BUFF(1) + ) pkt_gate_i ( + .clk (clk), + .reset (rst), + .clear (1'b0), + .i_tdata (pre_gate.tdata), + .i_tlast (pre_gate.tlast), + .i_terror (1'b0), + .i_tvalid (pre_gate.tvalid), + .i_tready (pre_gate.tready), + .o_tdata (m_axis_tdata), + .o_tlast (m_axis_tlast), + .o_tvalid (m_axis_tvalid), + .o_tready (m_axis_tready) + ); + +endmodule
\ No newline at end of file diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv new file mode 100644 index 000000000..fc9d53fe7 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv @@ -0,0 +1,428 @@ +// +// Copyright 2018 Ettus Research, A National Instruments Company +// +// SPDX-License-Identifier: LGPL-3.0-or-later + + +`timescale 1ns/1ps +`define NS_PER_TICK 1 +`define NUM_TEST_CASES 7 + +`include "sim_clks_rsts.vh" +`include "sim_exec_report.vh" +`include "sim_set_rb_lib.svh" +`include "sim_axis_lib.svh" + +`define SIM_TIMEOUT_US 1000000 // Default: 1s + +module crossbar_tb #( + parameter TEST_NAME = "crossbar_tb", + // Router parameters + parameter ROUTER_IMPL = "axi_crossbar", // Router implementation + parameter ROUTER_PORTS = 10, // # Router ports + parameter ROUTER_DWIDTH = 64, // Router datapath width + parameter MTU_LOG2 = 7, // log2 of max packet size for router + parameter NUM_MASTERS = ROUTER_PORTS, // Number of data generators in test + // Test parameters + parameter TEST_MAX_PACKETS = 50, // How many packets to stream per test case? + parameter TEST_LPP = 50, // Lines per packet + parameter TEST_MIN_INJ_RATE = 60, // Minimum injection rate to test + parameter TEST_MAX_INJ_RATE = 100, // Maximum injection rate to test + parameter TEST_INJ_RATE_INCR = 10, // Injection rate increment + parameter TEST_GEN_LL_FILES = 0 // Generate files to produce load-latency graphs? + +)( + /* no IO */ +); + `TEST_BENCH_INIT(TEST_NAME,`NUM_TEST_CASES,`NS_PER_TICK) + + //---------------------------------------------------- + // General test setup + //---------------------------------------------------- + + // Clocks and reset + `DEFINE_CLK(clk, 5.000, 50) + `DEFINE_RESET(rst, 0, 10) + + // Timekeeper (cycle counter) + logic [63:0] timestamp; + initial begin : timekeeper_blk + while (rst) @(posedge clk); + timestamp = 'd0; + while (~rst) begin + @(posedge clk); + timestamp = timestamp + 'd1; + end + end + + //---------------------------------------------------- + // Instantiate traffic generators, checkers, buses + //---------------------------------------------------- + localparam FILE_PATH = {`WORKING_DIR, "/data/", ROUTER_IMPL}; + + // Data buses + axis_t #(.DWIDTH(ROUTER_DWIDTH), .NUM_STREAMS(ROUTER_PORTS)) src2rtr_axis (.clk(clk)); + axis_t #(.DWIDTH(ROUTER_DWIDTH), .NUM_STREAMS(ROUTER_PORTS)) rtr2snk_axis (.clk(clk)); + + // Control buses + settings_bus_master #(.SR_AWIDTH(16), .SR_DWIDTH(32)) rtr_sb (.clk(clk)); + wire rtr_sb_ack; + + // Test vector source and sink instantiation + logic [7:0] set_injection_rate; + logic [15:0] set_lines_per_pkt; + logic [7:0] set_traffic_patt; + logic [31:0] set_num_pkts_to_send; + logic snk_start_stb = 0; + logic src_start_stb = 0; + + wire [63:0] session_duration [0:ROUTER_PORTS-1]; + wire [ROUTER_PORTS-1:0] src_active; + wire [31:0] src_xfer_count [0:ROUTER_PORTS-1]; + wire [31:0] src_pkt_count [0:ROUTER_PORTS-1]; + wire [ROUTER_PORTS-1:0] snk_active; + wire [31:0] snk_xfer_count [0:ROUTER_PORTS-1]; + wire [31:0] snk_pkt_count [0:ROUTER_PORTS-1]; + wire [31:0] snk_data_err_count [0:ROUTER_PORTS-1]; + wire [31:0] snk_route_err_count[0:ROUTER_PORTS-1]; + + wire deadlock_detected; + reg deadlock_detected_del = 1'b0; + always @(posedge clk) deadlock_detected_del <= deadlock_detected; + wire deadlock_re = (deadlock_detected & ~deadlock_detected_del); + wire deadlock_fe = (~deadlock_detected & deadlock_detected_del); + + genvar i; + generate for (i = 0; i < ROUTER_PORTS; i=i+1) begin: src_snk_blk + chdr_traffic_source_sim #( + .WIDTH (ROUTER_DWIDTH), + .MTU (MTU_LOG2), + .NODE_ID (i), + .NUM_NODES (ROUTER_PORTS) + ) traffic_src ( + .clk (clk), + .rst (rst), + .current_time (timestamp), + .start_stb (src_start_stb & (i < NUM_MASTERS)), + .injection_rate (set_injection_rate), + .lines_per_pkt (set_lines_per_pkt), + .traffic_patt (set_traffic_patt), + .num_pkts_to_send (set_num_pkts_to_send), + .m_axis_tdata (src2rtr_axis.tdata[((i+1)*ROUTER_DWIDTH)-1:i*ROUTER_DWIDTH]), + .m_axis_tlast (src2rtr_axis.tlast[i]), + .m_axis_tvalid (src2rtr_axis.tvalid[i]), + .m_axis_tready (src2rtr_axis.tready[i]), + .session_active (src_active[i]), + .session_duration (session_duration[i]), + .xfer_count (src_xfer_count[i]), + .pkt_count (src_pkt_count[i]) + ); + + chdr_traffic_sink_sim #( + .WIDTH (ROUTER_DWIDTH), + .MTU (MTU_LOG2), + .NODE_ID (i), + .NUM_NODES (ROUTER_PORTS), + .FILE_PATH (TEST_GEN_LL_FILES==1 ? FILE_PATH : "") + ) traffic_sink ( + .clk (clk), + .rst (rst), + .current_time (timestamp), + .start_stb (snk_start_stb), + .injection_rate (set_injection_rate), + .lines_per_pkt (set_lines_per_pkt), + .traffic_patt (set_traffic_patt), + .s_axis_tdata (rtr2snk_axis.tdata[((i+1)*ROUTER_DWIDTH)-1:i*ROUTER_DWIDTH]), + .s_axis_tlast (rtr2snk_axis.tlast[i]), + .s_axis_tvalid (rtr2snk_axis.tvalid[i]), + .s_axis_tready (rtr2snk_axis.tready[i]), + .session_active (snk_active[i]), + .xfer_count (snk_xfer_count[i]), + .pkt_count (snk_pkt_count[i]), + .data_err_count (snk_data_err_count[i]), + .route_err_count (snk_route_err_count[i]) + ); + end endgenerate + + //---------------------------------------------------- + // Instantiate DUT + //---------------------------------------------------- + generate if (ROUTER_IMPL == "FIFO") begin + for (i = 0; i < ROUTER_PORTS; i=i+1) begin + axi_fifo #( + .WIDTH(ROUTER_DWIDTH+1), .SIZE(0) + ) fifo_i ( + .clk (clk), + .reset (rst), + .clear (1'b0), + .i_tdata ({src2rtr_axis.tlast[i], src2rtr_axis.tdata[((i+1)*ROUTER_DWIDTH)-1:i*ROUTER_DWIDTH]}), + .i_tvalid (src2rtr_axis.tvalid[i]), + .i_tready (src2rtr_axis.tready[i]), + .o_tdata ({rtr2snk_axis.tlast[i], rtr2snk_axis.tdata[((i+1)*ROUTER_DWIDTH)-1:i*ROUTER_DWIDTH]}), + .o_tvalid (rtr2snk_axis.tvalid[i]), + .o_tready (rtr2snk_axis.tready[i]), + .space (), + .occupied () + ); + end + end else if (ROUTER_IMPL == "axi_crossbar") begin + axi_crossbar #( + .BASE (0), + .FIFO_WIDTH (ROUTER_DWIDTH), + .DST_WIDTH (16), + .NUM_INPUTS (ROUTER_PORTS), + .NUM_OUTPUTS (ROUTER_PORTS) + ) router_dut_i ( + // General + .clk (clk), + .reset (rst), + .clear (1'b0), + .local_addr (8'd0), + // Inputs + .i_tdata (src2rtr_axis.tdata), + .i_tlast (src2rtr_axis.tlast), + .i_tvalid (src2rtr_axis.tvalid), + .i_tready (src2rtr_axis.tready), + .pkt_present (src2rtr_axis.tvalid), + // Output + .o_tdata (rtr2snk_axis.tdata), + .o_tlast (rtr2snk_axis.tlast), + .o_tvalid (rtr2snk_axis.tvalid), + .o_tready (rtr2snk_axis.tready), + // Setting Bus + .set_stb (rtr_sb.settings_bus.set_stb), + .set_addr (rtr_sb.settings_bus.set_addr), + .set_data (rtr_sb.settings_bus.set_data), + // Readback bus + .rb_rd_stb (1'b0), + .rb_addr ({(2*$clog2(ROUTER_PORTS)){1'b0}}), + .rb_data () + ); + end else if (ROUTER_IMPL == "chdr_crossbar_nxn") begin + chdr_crossbar_nxn #( + .CHDR_W (ROUTER_DWIDTH), + .NPORTS (ROUTER_PORTS), + .DEFAULT_PORT (0), + .MTU (MTU_LOG2), + .ROUTE_TBL_SIZE (6), + .MUX_ALLOC ("ROUND-ROBIN"), + .OPTIMIZE ("AREA"), + .NPORTS_MGMT (0), + .EXT_RTCFG_PORT (1) + ) router_dut_i ( + // General + .clk (clk), + .reset (rst), + // Inputs + .s_axis_tdata (src2rtr_axis.tdata), + .s_axis_tlast (src2rtr_axis.tlast), + .s_axis_tvalid (src2rtr_axis.tvalid), + .s_axis_tready (src2rtr_axis.tready), + // Output + .m_axis_tdata (rtr2snk_axis.tdata), + .m_axis_tlast (rtr2snk_axis.tlast), + .m_axis_tvalid (rtr2snk_axis.tvalid), + .m_axis_tready (rtr2snk_axis.tready), + // External router config + .ext_rtcfg_stb (rtr_sb.settings_bus.set_stb), + .ext_rtcfg_addr (rtr_sb.settings_bus.set_addr), + .ext_rtcfg_data (rtr_sb.settings_bus.set_data), + .ext_rtcfg_ack (rtr_sb_ack) + ); + end else begin + axis_ctrl_crossbar_nxn #( + .WIDTH (ROUTER_DWIDTH), + .NPORTS (ROUTER_PORTS), + .TOPOLOGY (ROUTER_IMPL == "axis_ctrl_2d_torus" ? "TORUS" : "MESH"), + .INGRESS_BUFF_SIZE(MTU_LOG2), + .ROUTER_BUFF_SIZE (MTU_LOG2), + .ROUTING_ALLOC ("WORMHOLE"), + .SWITCH_ALLOC ("PRIO") + ) router_dut_i ( + // General + .clk (clk), + .reset (rst), + // Inputs + .s_axis_tdata (src2rtr_axis.tdata), + .s_axis_tlast (src2rtr_axis.tlast), + .s_axis_tvalid (src2rtr_axis.tvalid), + .s_axis_tready (src2rtr_axis.tready), + // Output + .m_axis_tdata (rtr2snk_axis.tdata), + .m_axis_tlast (rtr2snk_axis.tlast), + .m_axis_tvalid (rtr2snk_axis.tvalid), + .m_axis_tready (rtr2snk_axis.tready), + // Deadlock detection + .deadlock_detected(deadlock_detected) + ); + end endgenerate + + //---------------------------------------------------- + // Test routine. Runs tests and writes metrics to file + //---------------------------------------------------- + + // Constants + localparam [7:0] TRAFFIC_PATT_LOOPBACK = 8'd76; //L + localparam [7:0] TRAFFIC_PATT_NEIGHBOR = 8'd78; //N + localparam [7:0] TRAFFIC_PATT_BIT_COMPLEMENT = 8'd67; //C + localparam [7:0] TRAFFIC_PATT_SEQUENTIAL = 8'd83; //S + localparam [7:0] TRAFFIC_PATT_UNIFORM = 8'd85; //U + localparam [7:0] TRAFFIC_PATT_UNIFORM_OTHERS = 8'd79; //O + localparam [7:0] TRAFFIC_PATT_RANDOM_PERM = 8'd82; //R + + string filename; + integer node; + integer session = 0; + integer handle = 0; + logic [63:0] start_time; + integer total_pkts_recvd = 0, total_pkts_sent = 0; + + task sim_dataflow; + input [7:0] injection_rate; + input [7:0] traffic_patt; + input [15:0] lines_per_pkt; + input [31:0] num_pkts_to_send; + begin + session = session + 1; + $display("--------------- New Simulation ---------------"); + $display("- Module = %s", ROUTER_IMPL); + $display("- Nodes = %00d", ROUTER_PORTS); + $display("- Injection Rate = %00d%%", injection_rate); + $display("- Traffic Pattern = %c", traffic_patt); + $display("- Packet Size = %00d words (%00d bits)", lines_per_pkt, ROUTER_DWIDTH); + $display("- Max Packets = %00d", num_pkts_to_send); + // Configure settings + @(posedge clk); + set_injection_rate = injection_rate; + set_lines_per_pkt = lines_per_pkt; + set_traffic_patt = traffic_patt; + set_num_pkts_to_send = num_pkts_to_send; + @(posedge clk); + // Start the sink then the source + $display("Data flow starting..."); + snk_start_stb = 1; + src_start_stb = 1; + @(posedge clk); + src_start_stb = 0; + snk_start_stb = 0; + @(posedge clk); + start_time = timestamp; + // Wait for source blocks to finish generating + $display("Waiting for packets to transmit... (may take a while)"); + while (|src_active) begin + @(posedge clk); + if (deadlock_re) $display("WARNING: Deadlock detected"); + if (deadlock_fe) $display("Recovered from deadlock"); + end + // Wait for sink blocks to finish consuming + $display("All packets transmitted. Waiting to flush..."); + while (|snk_active) @(posedge clk); + // If router deadlocks then wait for it to recover + if (deadlock_detected) begin + $display("Waiting for deadlock recovery to finish..."); + while (deadlock_detected) @(posedge clk); + end + repeat(set_lines_per_pkt) @(posedge clk); + // Record summary to file and print to console + $sformat(filename, "%s/info_inj%03d_lpp%05d_traffic%c_sess%04d.csv", + FILE_PATH, injection_rate, lines_per_pkt, traffic_patt, session); + if (TEST_GEN_LL_FILES == 1) begin + handle = $fopen(filename, "w"); + if (handle == 0) begin + $error("Could not open file: %s", filename); + $finish(); + end + end + if (handle != 0) $fdisplay(handle, "Impl,Node,TxPkts,RxPkts,Duration,ErrRoute,ErrData"); + total_pkts_sent = 0; + total_pkts_recvd = 0; + for (node = 0; node < ROUTER_PORTS; node=node+1) begin + $display("- Node #%03d: TX = %5d pkts, RX = %5d pkts, Inj Rate = %3d%%. Errs = %5d route, %5d data", + node,src_pkt_count[node], snk_pkt_count[node], ((src_xfer_count[node]*100)/session_duration[node]), + snk_route_err_count[node], snk_data_err_count[node]); + if (handle != 0) $fdisplay(handle, "%s,%00d,%00d,%00d,%00d,%00d,%00d", ROUTER_IMPL, + node,src_pkt_count[node], snk_pkt_count[node], session_duration[node], + snk_route_err_count[node], snk_data_err_count[node]); + total_pkts_sent = total_pkts_sent + src_pkt_count[node]; + total_pkts_recvd = total_pkts_recvd + snk_pkt_count[node]; + `ASSERT_ERROR(snk_route_err_count[node] == 0, "Routing errors. Received packets destined to other nodes"); + `ASSERT_ERROR(snk_data_err_count[node] == 0, "Integrity errors. Received corrupted packets"); + end + $display("Finished. Elapsed = %00d cycles, TX = %00d pkts, RX = %00d pkts", + (timestamp - start_time), total_pkts_sent, total_pkts_recvd); + `ASSERT_ERROR(total_pkts_recvd == total_pkts_sent, "Total # TX packets did not match the total # RX packets"); + if (handle != 0) $fclose(handle); + $display("----------------------------------------------"); + end + endtask + + //---------------------------------------------------- + // Main test loop + //---------------------------------------------------- + + logic [31:0] MAX_PACKETS = TEST_MAX_PACKETS; + logic [15:0] LPP = TEST_LPP; + integer MIN_INJ_RATE = TEST_MIN_INJ_RATE; + integer MAX_INJ_RATE = TEST_MAX_INJ_RATE; + integer INJ_RATE_INCR = TEST_INJ_RATE_INCR; + + integer inj_rate = 0; + initial begin : tb_main + src_start_stb = 0; + snk_start_stb = 0; + rtr_sb.reset(); + while (rst) @(posedge clk); + + repeat (10) @(posedge clk); + + `TEST_CASE_START("Set up crossbar"); + for (node = 0; node < ROUTER_PORTS; node=node+1) begin + if (ROUTER_IMPL == "axi_crossbar") begin + rtr_sb.write(16'd256 + node[15:0], {16'h0, node[15:0]}); + end else if (ROUTER_IMPL == "chdr_crossbar_nxn") begin + rtr_sb.write(node[15:0], {16'h0, node[15:0]}); + while (~rtr_sb_ack) @(posedge clk); + end + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate LOOPBACK Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_LOOPBACK, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate SEQUENTIAL Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_SEQUENTIAL, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate UNIFORM Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_UNIFORM, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate UNIFORM_OTHERS Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_UNIFORM_OTHERS, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate BIT_COMPLEMENT Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_BIT_COMPLEMENT, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_CASE_START("Simulate NEIGHBOR Traffic Pattern"); + for (inj_rate = MIN_INJ_RATE; inj_rate <= MAX_INJ_RATE; inj_rate = inj_rate + INJ_RATE_INCR) begin + sim_dataflow(inj_rate, TRAFFIC_PATT_NEIGHBOR, LPP, MAX_PACKETS); + end + `TEST_CASE_DONE(1) + + `TEST_BENCH_DONE + end // initial begin + +endmodule diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/gen_load_latency_graph.py b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/gen_load_latency_graph.py new file mode 100755 index 000000000..35821c2c4 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/gen_load_latency_graph.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# +# Copyright 2018 Ettus Research, A National Instruments Company +# +# SPDX-License-Identifier: LGPL-3.0-or-later +# +# Description +# Parses the output files generated by crossbar_tb and outputs +# a load-latency graph and a expected-actual throughput graph + +import os, sys +import argparse +import time +import glob +import csv +import re +import numpy as np + +import matplotlib +#matplotlib.use('Agg') +import matplotlib.pyplot as plt + +def get_options(): + parser = argparse.ArgumentParser(description='Generate Load Latency Graphs') + parser.add_argument('datadir', type=str, default='.', help='Location of packet capture files generated by testbench') + return parser.parse_args() + +TRAFFIC_PATTERNS = {'U':'UNIFORM', 'O':'UNIFORM_OTHERS', 'N':'NEIGHBOR', 'L':'LOOPBACK', 'S':'SEQUENTIAL', 'C':'BIT_COMPLEMENT', 'R':'RANDOM_PERM'} + +class InfoFile(): + def __init__(self, filename): + # Extract test info from filename + m = re.search(r".*/info_inj([0-9]+)_lpp([0-9]+)_traffic(.)_sess([0-9]+)\.csv", filename) + if m is None: + raise ValueError('Incorrect filename format: %s'%(filename)) + self.inj_rate = int(m.group(1)) + self.lpp = int(m.group(2)) + self.traffic_patt = TRAFFIC_PATTERNS[m.group(3)] + self.session = int(m.group(4)) + + self.tx_pkts = 0 + self.rx_pkts = 0 + self.duration = 0 + self.errs = 0 + self.nodes = 0 + with open(filename, 'r') as csvfile: + reader = csv.reader(csvfile, delimiter=',') + isheader = True + for row in reader: + if isheader: + isheader = False + if row != ['Impl', 'Node', 'TxPkts', 'RxPkts', 'Duration', 'ErrRoute', 'ErrData']: + raise ValueError('Incorrect header: %s'%(filename)) + else: + self.impl = row[0] + self.tx_pkts = self.tx_pkts + int(row[2]) + self.rx_pkts = self.tx_pkts + int(row[3]) + self.duration = self.duration + int(row[4]) + self.errs = self.errs + int(row[5]) + int(row[6]) + self.nodes = self.nodes + 1 + self.real_inj_rate = (100.0 * self.tx_pkts * self.lpp) / self.duration + +class PktFile(): + def __init__(self, filename): + # Extract test info from filename + m = re.search(r".*/pkts_node([0-9]+)_inj([0-9]+)_lpp([0-9]+)_traffic(.)_sess([0-9]+)\.csv", filename) + if m is None: + raise ValueError('Incorrect filename format: %s'%(filename)) + self.node = int(m.group(1)) + self.inj_rate = int(m.group(2)) + self.lpp = int(m.group(3)) + self.traffic_patt = TRAFFIC_PATTERNS[m.group(4)] + self.session = int(m.group(5)) + + self.latencies = [] + with open(filename, 'r') as csvfile: + reader = csv.reader(csvfile, delimiter=',') + isheader = True + for row in reader: + if isheader: + isheader = False + if row != ['Src', 'Dst', 'Seqno', 'Error', 'Latency']: + raise ValueError('Incorrect header: %s'%(filename)) + else: + self.latencies.append(int(row[4])) + + +######################################################################## +# main +######################################################################## +if __name__=='__main__': + options = get_options() + + if (not os.path.isdir(options.datadir)): + print('ERROR: Data director %s does not exist'%(options.datadir)) + sys.exit(1) + + info_db = dict() + info_files = glob.glob(os.path.join(options.datadir, 'info*.csv')) + router_impl = '' + lines_per_pkt = 0 + for ifile in info_files: + print('INFO: Reading %s...'%(ifile)) + tmp = InfoFile(ifile) + router_impl = tmp.impl # Assume that all files have the same impl + lines_per_pkt = tmp.lpp # Assume that all files have the same LPP + info_db[(tmp.lpp, tmp.traffic_patt, tmp.inj_rate)] = tmp + + pkt_db = dict() + pkts_files = glob.glob(os.path.join(options.datadir, 'pkts*.csv')) + for pfile in pkts_files: + print('INFO: Reading %s...'%(pfile)) + tmp = PktFile(pfile) + config_key = (tmp.lpp, tmp.traffic_patt) + if config_key not in pkt_db: + pkt_db[config_key] = dict() + if tmp.inj_rate not in pkt_db[config_key]: + pkt_db[config_key][tmp.inj_rate] = [] + + + pkt_db[config_key][tmp.inj_rate].extend(tmp.latencies) + + # Write load-latency plots to file + actual_inj_rate_db = dict() + for config in sorted(pkt_db): + (lpp, traffic_patt) = config + ll_file = 'load-latency_%s_traffic-%s_lpp-%d.png'%(router_impl, traffic_patt, lpp) + print('INFO: Writing file ' + ll_file + '...') + percentile = [0, 25, 50, 75, 90, 95, 99, 99.9, 100] + plt.figure() + plt.title('Load Latency Graph for %s\n(Traffic: %s, LPP: %d)'%(router_impl, traffic_patt, lpp)) + for p in percentile: + plot_data = dict() + for inj_rate in pkt_db[config]: + real_inj_rate = info_db[(lpp, traffic_patt, inj_rate)].real_inj_rate + plot_data[real_inj_rate] = np.percentile(pkt_db[config][inj_rate], p) + latencies = [] + rates = [] + for inj_rate in sorted(plot_data): + rates.append(inj_rate) + latencies.append(plot_data[inj_rate]) + plt.plot(rates, latencies, label='$P_{%.1f}$'%(p)) + plt.xlabel('Load (%)') + plt.xticks(range(0, 110, 10)) + plt.ylabel('Latency (cycles)') + plt.grid(True) + plt.legend() + plt.savefig(os.path.join(options.datadir, ll_file), dpi=120) + # Generate actual inj_rate graph + real_inj_rates = [] + for inj_rate in sorted(pkt_db[config]): + real_inj_rates.append(info_db[(lpp, traffic_patt, inj_rate)].real_inj_rate) + actual_inj_rate_db[config] = (sorted(pkt_db[config]), real_inj_rates) + + # Write offered vs actual injection rate plots to file + injrate_file = 'injection-rate_%s_lpp-%d.png'%(router_impl, lines_per_pkt) + print('INFO: Writing file ' + injrate_file + '...') + plt.figure() + plt.title('Max Injection Rate Graph for %s'%(router_impl)) + for config in actual_inj_rate_db: + (x, y) = actual_inj_rate_db[config] + plt.plot(x, y, label=str(config)) + plt.xlabel('Offered Injection Rate (%)') + plt.xticks(range(0, 110, 10)) + plt.ylabel('Accepted Injection Rate (%)') + plt.yticks(range(0, 110, 10)) + plt.grid(True) + plt.legend() + plt.savefig(os.path.join(options.datadir, injrate_file), dpi=120)
\ No newline at end of file diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/run_sim_multi.py b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/run_sim_multi.py new file mode 100755 index 000000000..8e546fef9 --- /dev/null +++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/run_sim_multi.py @@ -0,0 +1,106 @@ +#!/usr/bin/python3 +# +# Copyright 2018 Ettus Research, a National Instruments Company +# +# SPDX-License-Identifier: LGPL-3.0-or-later +# +# Description +# Run the crossbar testbench (crossbar_tb) for varios parameter +# configurations and generates load-latency graphs for each run. + +import argparse +import math +import os, sys +import shutil +import glob +import subprocess + +g_tb_top_template = """ +`timescale 1ns/1ps +module crossbar_tb_auto(); + crossbar_tb #( + .TEST_NAME ("crossbar_tb_auto"), + .ROUTER_IMPL ("{rtr_impl}"), + .ROUTER_PORTS ({rtr_ports}), + .ROUTER_DWIDTH ({rtr_width}), + .MTU_LOG2 ({rtr_mtu}), + .NUM_MASTERS ({rtr_sources}), + .TEST_MAX_PACKETS ({tst_maxpkts}), + .TEST_LPP ({tst_lpp}), + .TEST_MIN_INJ_RATE ({tst_injrate_min}), + .TEST_MAX_INJ_RATE ({tst_injrate_max}), + .TEST_INJ_RATE_INCR (10), + .TEST_GEN_LL_FILES (1) + ) impl ( + /* no IO */ + ); +endmodule +""" + +g_test_params = { + 'data': {'rtr_width':64, 'rtr_mtu':7, 'tst_maxpkts':100, 'tst_lpp':100, 'tst_injrate_min':30, 'tst_injrate_max':100}, + 'ctrl': {'rtr_width':64, 'rtr_mtu':5, 'tst_maxpkts':100, 'tst_lpp':20, 'tst_injrate_min':10, 'tst_injrate_max':50}, +} + +g_xb_types = { + 'chdr_crossbar_nxn':'data', 'axi_crossbar':'data', + 'axis_ctrl_2d_torus':'ctrl', 'axis_ctrl_2d_mesh':'ctrl' +} + +def get_options(): + parser = argparse.ArgumentParser(description='Run correctness sim and generate load-latency plots') + parser.add_argument('--impl', type=str, default='chdr_crossbar_nxn', help='Implementation (CSV) [%s]'%(','.join(g_xb_types.keys()))) + parser.add_argument('--ports', type=str, default='16', help='Number of ports (CSV)') + parser.add_argument('--sources', type=str, default='16', help='Number of active data sources (masters)') + return parser.parse_args() + +def launch_run(impl, ports, sources): + run_name = '%s_ports%d_srcs%d'%(impl, ports, sources) + # Prepare a transform map to autogenerate a TB file + transform = {'rtr_impl':impl, 'rtr_ports':ports, 'rtr_sources':sources} + for k,v in g_test_params[g_xb_types[impl]].items(): + transform[k] = v + # Create crossbar_tb_auto.sv with specified parameters + with open('crossbar_tb_auto.sv', 'w') as out_file: + out_file.write(g_tb_top_template.format(**transform)) + # Create data directory for the simulation + data_dir = os.path.join('data', impl) + export_dir = os.path.join('data', run_name) + try: + os.makedirs('data') + except FileExistsError: + pass + os.makedirs(data_dir) + os.makedirs(export_dir) + # Run "make xsim" + exitcode = subprocess.Popen('make xsim TB_TOP_MODULE=crossbar_tb_auto', shell=True).wait() + if exitcode != 0: + raise RuntimeError('Error running "make xsim". Was setupenv.sh run?') + # Generate load-latency graphs + exitcode = subprocess.Popen('gen_load_latency_graph.py ' + data_dir, shell=True).wait() + if exitcode != 0: + raise RuntimeError('Error running "gen_load_latency_graph.py"') + # Copy files + os.rename('xsim.log', os.path.join(export_dir, 'xsim.log')) + for file in glob.glob(os.path.join(data_dir, '*.png')): + shutil.copy(file, export_dir) + # Cleanup outputs + subprocess.Popen('make cleanall', shell=True).wait() + try: + os.remove('crossbar_tb_auto.sv') + except FileNotFoundError: + pass + try: + shutil.rmtree(data_dir) + except OSError: + print('WARNING: Could not delete ' + data_dir) + +def main(): + args = get_options(); + for impl in args.impl.strip().split(','): + for ports in args.ports.strip().split(','): + for sources in args.sources.strip().split(','): + launch_run(impl, int(ports), min(int(ports), int(sources))) + +if __name__ == '__main__': + main() |