fpga: rfnoc: Add support for 512-bit CHDR widths

This fixes the rfnoc_null_src_sink, chdr_crossbar_nxn, and
chdr_stream_endpoint blocks so that wider CHDR widths are properly
supported. It also updates PkgChdrBfm to able to properly test these
blocks. The testbenches have been updated to test both 64 and 512-bit
widths.

6 files changed, 58 insertions, 18 deletions

diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile
index 399515640..cbeacc387 100644
--- a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/Makefile
@@ -34,9 +34,10 @@ $(RFNOC_CORE_SRCS) \
 # Testbench Specific
 #-------------------------------------------------
 # Define only one toplevel module
-SIM_TOP = chdr_crossbar_nxn_tb
+SIM_TOP = chdr_crossbar_nxn_all_tb
 
 SIM_SRCS = \
+$(abspath chdr_crossbar_nxn_all_tb.sv) \
 $(abspath chdr_crossbar_nxn_tb.sv) \
 $(abspath ../crossbar_tb.sv) \
 $(abspath ../chdr_traffic_source_sim.sv) \
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_all_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_all_tb.sv
new file mode 100644
index 000000000..9dbaa2568
--- /dev/null
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_all_tb.sv
@@ -0,0 +1,26 @@
+//
+// Copyright 2020 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+// Module: chdr_crossbar_nxn
+//
+// Description:  Testbench for chdr_crossbar_nxn that runs multiple widths
+//
+
+module chdr_crossbar_nxn_all_tb#(
+  /* no PARAM */
+)(
+  /* no IO */
+);
+
+  chdr_crossbar_nxn_tb #(.TEST_NAME("64B"),.CHDR_W(64)) CHDR64 ();
+  chdr_crossbar_nxn_tb #(.TEST_NAME("512B"),.CHDR_W(512)) CHDR512 ();
+
+  // Wait for all done
+  bit clk,rst;
+  sim_clock_gen #(100.0) clk_gen (clk, rst);  
+  always_ff@(posedge clk)
+    if (CHDR64.impl.done && CHDR512.impl.done) $finish(1);
+    
+endmodule
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv
index 1c5cace63..f5217c66f 100644
--- a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_crossbar_nxn_tb/chdr_crossbar_nxn_tb.sv
@@ -6,12 +6,17 @@
 
 `timescale 1ns/1ps
 
-module chdr_crossbar_nxn_tb();
+module chdr_crossbar_nxn_tb#(
+  parameter TEST_NAME  = "chdr_crossbar_nxn_tb",
+  parameter CHDR_W     = 64
+)(
+  /* no IO */
+);
   crossbar_tb #(
-    .TEST_NAME          ("chdr_crossbar_nxn_tb"),
+    .TEST_NAME          (TEST_NAME             ),
     .ROUTER_IMPL        ("chdr_crossbar_nxn"   ), // Router implementation
     .ROUTER_PORTS       (10                    ), // Number of ports
-    .ROUTER_DWIDTH      (64                    ), // Router datapath width
+    .ROUTER_DWIDTH      (CHDR_W                ), // Router datapath width
     .MTU_LOG2           (7                     ), // log2 of max packet size for router
     .NUM_MASTERS        (10                    ), // Number of data generators in test
     .TEST_MAX_PACKETS   (100                   ), // How many packets to stream per test case?
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv
index a9fe3ba27..c6af03582 100644
--- a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_sink_sim.sv
@@ -88,7 +88,7 @@ module chdr_traffic_sink_sim #(
     s_chdr.reset();
     while (1) begin
       // A session begins on the posedge of start_stb
-      while (~start_stb) @(posedge clk);
+      while (start_stb !== 1) @(posedge clk);
       session = session + 1;
       $sformat(filename, "%s/pkts_node%05d_inj%03d_lpp%05d_traffic%c_sess%04d.csv",
         FILE_PATH, NODE_ID, injection_rate, lines_per_pkt, traffic_patt, session);
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv
index e6cb7c5d9..8f8e2665d 100644
--- a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/chdr_traffic_source_sim.sv
@@ -107,7 +107,7 @@ module chdr_traffic_source_sim #(
     m_chdr.reset();
     while (1) begin
       // A generation session begins on the posedge of start_stb
-      while (~start_stb) @(posedge clk);
+      while (start_stb !== 1) @(posedge clk);
       curr_pkt_num = 'd0;
       m_chdr.reset();
       num_samps_xferd = 'd0;
diff --git a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv
index fc9d53fe7..33b09dfd5 100644
--- a/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv
+++ b/fpga/usrp3/lib/rfnoc/crossbar/crossbar_tb/crossbar_tb.sv
@@ -58,12 +58,18 @@ module crossbar_tb #(
   //----------------------------------------------------
   // Instantiate traffic generators, checkers, buses
   //----------------------------------------------------
-  localparam FILE_PATH = {`WORKING_DIR, "/data/", ROUTER_IMPL};
+  `ifndef WORKING_DIR
+    // For XSim
+    localparam FILE_PATH = {`WORKING_DIR, "/data/", ROUTER_IMPL};
+  `else
+    // For ModelSim
+    localparam FILE_PATH = {"./work", "/data/", ROUTER_IMPL};
+  `endif
 
   // Data buses
   axis_t #(.DWIDTH(ROUTER_DWIDTH), .NUM_STREAMS(ROUTER_PORTS)) src2rtr_axis (.clk(clk));
   axis_t #(.DWIDTH(ROUTER_DWIDTH), .NUM_STREAMS(ROUTER_PORTS)) rtr2snk_axis (.clk(clk));
-  
+
   // Control buses
   settings_bus_master #(.SR_AWIDTH(16), .SR_DWIDTH(32)) rtr_sb (.clk(clk));
   wire rtr_sb_ack;
@@ -75,6 +81,7 @@ module crossbar_tb #(
   logic [31:0]  set_num_pkts_to_send;
   logic         snk_start_stb = 0;
   logic         src_start_stb = 0;
+  bit           done = 0;
 
   wire [63:0]             session_duration   [0:ROUTER_PORTS-1];
   wire [ROUTER_PORTS-1:0] src_active;
@@ -99,7 +106,7 @@ module crossbar_tb #(
       .MTU              (MTU_LOG2),
       .NODE_ID          (i),
       .NUM_NODES        (ROUTER_PORTS)
-    ) traffic_src (     
+    ) traffic_src (
       .clk              (clk),
       .rst              (rst),
       .current_time     (timestamp),
@@ -117,14 +124,14 @@ module crossbar_tb #(
       .xfer_count       (src_xfer_count[i]),
       .pkt_count        (src_pkt_count[i])
     );
-  
+
     chdr_traffic_sink_sim #(
       .WIDTH            (ROUTER_DWIDTH),
       .MTU              (MTU_LOG2),
       .NODE_ID          (i),
       .NUM_NODES        (ROUTER_PORTS),
       .FILE_PATH        (TEST_GEN_LL_FILES==1 ? FILE_PATH : "")
-    ) traffic_sink (    
+    ) traffic_sink (
       .clk              (clk),
       .rst              (rst),
       .current_time     (timestamp),
@@ -152,8 +159,8 @@ module crossbar_tb #(
       axi_fifo #(
         .WIDTH(ROUTER_DWIDTH+1), .SIZE(0)
       ) fifo_i (
-        .clk      (clk), 
-        .reset    (rst), 
+        .clk      (clk),
+        .reset    (rst),
         .clear    (1'b0),
         .i_tdata  ({src2rtr_axis.tlast[i], src2rtr_axis.tdata[((i+1)*ROUTER_DWIDTH)-1:i*ROUTER_DWIDTH]}),
         .i_tvalid (src2rtr_axis.tvalid[i]),
@@ -178,7 +185,7 @@ module crossbar_tb #(
       .reset        (rst),
       .clear        (1'b0),
       .local_addr   (8'd0),
-      // Inputs     
+      // Inputs
       .i_tdata      (src2rtr_axis.tdata),
       .i_tlast      (src2rtr_axis.tlast),
       .i_tvalid     (src2rtr_axis.tvalid),
@@ -210,7 +217,7 @@ module crossbar_tb #(
       .NPORTS_MGMT    (0),
       .EXT_RTCFG_PORT (1)
     ) router_dut_i (
-       // General         
+       // General
       .clk            (clk),
       .reset          (rst),
       // Inputs
@@ -242,12 +249,12 @@ module crossbar_tb #(
        // General
       .clk              (clk),
       .reset            (rst),
-      // Inputs         
+      // Inputs
       .s_axis_tdata     (src2rtr_axis.tdata),
       .s_axis_tlast     (src2rtr_axis.tlast),
       .s_axis_tvalid    (src2rtr_axis.tvalid),
       .s_axis_tready    (src2rtr_axis.tready),
-      // Output         
+      // Output
       .m_axis_tdata     (rtr2snk_axis.tdata),
       .m_axis_tlast     (rtr2snk_axis.tlast),
       .m_axis_tvalid    (rtr2snk_axis.tvalid),
@@ -313,7 +320,7 @@ module crossbar_tb #(
       @(posedge clk);
       if (deadlock_re) $display("WARNING: Deadlock detected");
       if (deadlock_fe) $display("Recovered from deadlock");
-    end 
+    end
     // Wait for sink blocks to finish consuming
     $display("All packets transmitted. Waiting to flush...");
     while (|snk_active) @(posedge clk);
@@ -423,6 +430,7 @@ module crossbar_tb #(
     `TEST_CASE_DONE(1)
 
     `TEST_BENCH_DONE
+    done = 1;
   end // initial begin
 
 endmodule