RFNoC: Limit number of control packets in flight

Limit  number of unacknowledged control packets to the number of
receive frames in the transport to prevent the transport from
getting locked up or being overrun by ACK packets.

Signed-off-by: michael-west <michael.west@ettus.com>

3 files changed, 23 insertions, 19 deletions

diff --git a/host/lib/rfnoc/ctrl_iface.cpp b/host/lib/rfnoc/ctrl_iface.cpp
index e0fc03eab..a441ffbc3 100644
--- a/host/lib/rfnoc/ctrl_iface.cpp
+++ b/host/lib/rfnoc/ctrl_iface.cpp
@@ -35,9 +35,9 @@ public:
         : _xports(xports)
         , _name(name)
         , _seq_out(0)
-        , _max_outstanding_acks(
-              uhd::rfnoc::CMD_FIFO_SIZE / uhd::rfnoc::MAX_CMD_PKT_SIZE)
+        , _max_outstanding_acks(xports.recv->get_num_recv_frames())
     {
+
         UHD_ASSERT_THROW(bool(_xports.send));
         UHD_ASSERT_THROW(bool(_xports.recv));
         // Flush the response transport in case we have something over:
diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp
index cc0a610a3..bdb32b8f6 100644
--- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp
+++ b/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp
@@ -17,15 +17,6 @@ using namespace uhd::mpmd::xport;
 
 namespace {
 
-#if defined(UHD_PLATFORM_MACOS) || defined(UHD_PLATFORM_BSD)
-//! Size of the host-side socket buffer for RX
-const size_t MPMD_RX_SW_BUFF_SIZE_ETH = 0x100000; // 1Mib
-#elif defined(UHD_PLATFORM_LINUX) || defined(UHD_PLATFORM_WIN32)
-//! Size of the host-side socket buffer for RX
-// For an ~8k frame size any size >32MiB is just wasted buffer space
-const size_t MPMD_RX_SW_BUFF_SIZE_ETH = 0x2000000; // 32 MiB
-#endif
-
 //! Maximum CHDR packet size in bytes
 const size_t MPMD_10GE_DATA_FRAME_MAX_SIZE = 4000;
 
@@ -36,7 +27,7 @@ const size_t MPMD_10GE_ASYNCMSG_FRAME_MAX_SIZE = 1472;
 const size_t MPMD_ETH_NUM_FRAMES = 32;
 
 //!
-const double MPMD_BUFFER_FILL_RATE = 20.0e-3; // s
+const double MPMD_BUFFER_DEPTH = 50.0e-3; // s
 //! For MTU discovery, the time we wait for a packet before calling it
 // oversized (seconds).
 const double MPMD_MTU_DISCOVERY_TIMEOUT = 0.02;
@@ -176,10 +167,6 @@ uhd::both_xports_t mpmd_xport_ctrl_udp::make_transport(
 {
     auto xport_args = xport_args_;
 
-    if (xport_type == usrp::device3_impl::RX_DATA
-        and not xport_args.has_key("recv_buff_size")) {
-        xport_args["recv_buff_size"] = std::to_string(MPMD_RX_SW_BUFF_SIZE_ETH);
-    }
     size_t link_speed = MAX_RATE_1GIGE;
     if (xport_info.count("link_speed") == 0) {
         UHD_LOG_WARNING("MPMD",
@@ -191,10 +178,15 @@ uhd::both_xports_t mpmd_xport_ctrl_udp::make_transport(
     }
     transport::zero_copy_xport_params default_buff_args;
     // Create actual UDP transport
+    default_buff_args.num_send_frames = 1;
+    default_buff_args.num_recv_frames =
+        xport_type == usrp::device3_impl::CTRL ?
+        (uhd::rfnoc::CMD_FIFO_SIZE / uhd::rfnoc::MAX_CMD_PKT_SIZE) :
+        1;
     default_buff_args.recv_frame_size =
         xport_args.cast<size_t>("recv_frame_size", get_mtu(uhd::RX_DIRECTION));
-    default_buff_args.recv_buff_size = link_speed * MPMD_BUFFER_FILL_RATE;
-    default_buff_args.send_buff_size = link_speed * MPMD_BUFFER_FILL_RATE;
+    default_buff_args.recv_buff_size = link_speed * MPMD_BUFFER_DEPTH;
+    default_buff_args.send_buff_size = link_speed * MPMD_BUFFER_DEPTH;
     if (xport_type == usrp::device3_impl::ASYNC_MSG) {
         default_buff_args.send_frame_size = MPMD_10GE_ASYNCMSG_FRAME_MAX_SIZE;
     } else {
diff --git a/host/lib/usrp/x300/x300_impl.cpp b/host/lib/usrp/x300/x300_impl.cpp
index d912d66c3..24ce7abf4 100644
--- a/host/lib/usrp/x300/x300_impl.cpp
+++ b/host/lib/usrp/x300/x300_impl.cpp
@@ -1372,12 +1372,24 @@ uhd::both_xports_t x300_impl::make_transport(const uhd::sid_t& address,
             std::min(system_max_send_frame_size, x300::ETH_MSG_FRAME_SIZE);
         default_buff_args.recv_frame_size =
             std::min(system_max_recv_frame_size, x300::ETH_MSG_FRAME_SIZE);
+        // Buffering is done in the socket buffers, so size them relative to
+        // the link rate
         default_buff_args.send_buff_size = conn.link_rate / 50; // 20ms
         default_buff_args.recv_buff_size = std::max(conn.link_rate / 50,
             x300::ETH_MSG_NUM_FRAMES
                 * x300::ETH_MSG_FRAME_SIZE); // enough to hold greater of 20ms or number
                                              // of msg frames
-        if (xport_type == TX_DATA) {
+        // There is no need for more than 1 send and recv frame since the
+        // buffering is done in the socket buffers
+        default_buff_args.num_send_frames = 1;
+        default_buff_args.num_recv_frames = 1;
+        if (xport_type == CTRL) {
+            // Increasing number of recv frames here because ctrl_iface uses it
+            // to determine how many control packets can be in flight before it
+            // must wait for an ACK
+            default_buff_args.num_recv_frames = 
+                uhd::rfnoc::CMD_FIFO_SIZE / uhd::rfnoc::MAX_CMD_PKT_SIZE;
+        } else if (xport_type == TX_DATA) {
             size_t default_frame_size = conn.link_rate == x300::MAX_RATE_1GIGE
                                             ? x300::GE_DATA_FRAME_SEND_SIZE
                                             : x300::XGE_DATA_FRAME_SEND_SIZE;