transport,usrp: Make available packet-based flow control

DPDK provides a fixed number of fixed-size buffers for the receive
window, so it needs packet-based flow control to avoid dropping
packets. This change enables counting by packets.

Co-authored-by: Ciro Nishiguchi <ciro.nishiguchi@ni.com>

7 files changed, 61 insertions, 46 deletions

diff --git a/host/lib/include/uhdlib/transport/links.hpp b/host/lib/include/uhdlib/transport/links.hpp
index 09872b145..6fe51dbbd 100644
--- a/host/lib/include/uhdlib/transport/links.hpp
+++ b/host/lib/include/uhdlib/transport/links.hpp
@@ -20,7 +20,8 @@ using both_links_t = std::tuple<uhd::transport::send_link_if::sptr,
     size_t, // num_send_frames
     uhd::transport::recv_link_if::sptr,
     size_t, // num_recv_frames
-    bool>;  // lossy_xport
+    bool, // lossy_xport
+    bool>; // packet flow control
 
 /*!
  * Parameters for link creation.
@@ -39,4 +40,3 @@ struct link_params_t
 }} // namespace uhd::transport
 
 #endif /* INCLUDED_UHDLIB_TRANSPORT_LINKS_HPP */
-
diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp
index 00880738c..bf383f44d 100644
--- a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp
+++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp
@@ -116,8 +116,8 @@ uhd::transport::both_links_t mpmd_link_if_ctrl_liberio::get_link(const size_t li
         link_info.first, link_info.second, link_params);
     io_srv->attach_send_link(link);
     io_srv->attach_recv_link(link);
-    return std::tuple<send_link_if::sptr, size_t, recv_link_if::sptr, size_t, bool>(
-        link, link_params.send_buff_size, link, link_params.recv_buff_size, false);
+    return std::make_tuple(
+        link, link_params.send_buff_size, link, link_params.recv_buff_size, false, true);
 }
 
 size_t mpmd_link_if_ctrl_liberio::get_mtu(const uhd::direction_t /*dir*/) const
diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp
index aeb1c6436..3a2b3f59e 100644
--- a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp
+++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp
@@ -32,7 +32,7 @@ const uhd::rfnoc::chdr::chdr_packet_factory mpmd_link_if_ctrl_udp::_pkt_factory(
 namespace {
 
 //! Maximum CHDR packet size in bytes
-const size_t MPMD_10GE_DATA_FRAME_MAX_SIZE = 8000;
+const size_t MPMD_10GE_DATA_FRAME_MAX_SIZE    = 8000;
 const size_t MPMD_1GE_DATA_FRAME_MAX_SIZE     = 1472;
 const size_t MPMD_1GE_ASYNCMSG_FRAME_MAX_SIZE = 1472;
 
@@ -366,8 +366,12 @@ uhd::transport::both_links_t mpmd_link_if_ctrl_udp::get_link(const size_t link_i
     if (_mb_args.has_key("use_dpdk")) { // FIXME use constrained device args
 #ifdef HAVE_DPDK
         auto link = uhd::transport::udp_dpdk_link::make(ip_addr, udp_port, link_params);
-        return std::make_tuple(
-            link, link_params.send_buff_size, link, link_params.recv_buff_size, true);
+        return std::make_tuple(link,
+            link_params.send_buff_size,
+            link,
+            link_params.recv_buff_size,
+            true,
+            true);
 #else
         UHD_LOG_WARNING("X300", "Cannot create DPDK transport, falling back to UDP");
 #endif
@@ -378,7 +382,7 @@ uhd::transport::both_links_t mpmd_link_if_ctrl_udp::get_link(const size_t link_i
         link_params.recv_buff_size,
         link_params.send_buff_size);
     return std::make_tuple(
-        link, link_params.send_buff_size, link, link_params.recv_buff_size, true);
+        link, link_params.send_buff_size, link, link_params.recv_buff_size, true, false);
 }
 
 size_t mpmd_link_if_ctrl_udp::get_num_links() const
diff --git a/host/lib/usrp/mpmd/mpmd_mb_iface.cpp b/host/lib/usrp/mpmd/mpmd_mb_iface.cpp
index cb779f635..b474dfdc3 100644
--- a/host/lib/usrp/mpmd/mpmd_mb_iface.cpp
+++ b/host/lib/usrp/mpmd/mpmd_mb_iface.cpp
@@ -167,7 +167,7 @@ uhd::rfnoc::chdr_ctrl_xport::sptr mpmd_mboard_impl::mpmd_mb_iface::make_ctrl_tra
     const size_t link_idx = _local_device_id_map.at(local_device_id);
     uhd::transport::send_link_if::sptr send_link;
     uhd::transport::recv_link_if::sptr recv_link;
-    std::tie(send_link, std::ignore, recv_link, std::ignore, std::ignore) =
+    std::tie(send_link, std::ignore, recv_link, std::ignore, std::ignore, std::ignore) =
         _link_if_mgr->get_link(
             link_idx, uhd::transport::link_type_t::CTRL, uhd::device_addr_t());
 
@@ -209,25 +209,30 @@ mpmd_mboard_impl::mpmd_mb_iface::make_rx_data_transport(
 
     uhd::transport::send_link_if::sptr send_link;
     uhd::transport::recv_link_if::sptr recv_link;
-    bool lossy_xport;
+    bool lossy_xport, packet_fc;
     size_t recv_buff_size;
-    std::tie(send_link, std::ignore, recv_link, recv_buff_size, lossy_xport) =
+    std::tie(send_link, std::ignore, recv_link, recv_buff_size, lossy_xport, packet_fc) =
         _link_if_mgr->get_link(
             link_idx, uhd::transport::link_type_t::RX_DATA, xport_args);
 
     /* Associate local device ID with the adapter */
     _adapter_map[local_sep_addr.first] = send_link->get_send_adapter_id();
 
-    // TODO: configure this based on the transport type
-    const stream_buff_params_t recv_capacity = {
-        recv_buff_size, uhd::rfnoc::MAX_FC_CAPACITY_PKTS};
+    const uhd::rfnoc::stream_buff_params_t recv_capacity = {recv_buff_size,
+        packet_fc ? static_cast<uint32_t>(recv_link->get_num_recv_frames())
+                  : uhd::rfnoc::MAX_FC_CAPACITY_PKTS};
 
     const double ratio = 1.0 / 32;
 
-    // Configure flow control frequency to use bytes only for UDP
-    stream_buff_params_t fc_freq = {
-        static_cast<uint64_t>(std::ceil(double(recv_buff_size) * ratio)),
-        uhd::rfnoc::MAX_FC_FREQ_PKTS};
+    // Configure flow control frequency to use either bytes only or packets only
+    uhd::rfnoc::stream_buff_params_t fc_freq;
+    if (packet_fc) {
+        fc_freq = {uhd::rfnoc::MAX_FC_FREQ_BYTES,
+            static_cast<uint32_t>(std::ceil(recv_link->get_num_recv_frames() * ratio))};
+    } else {
+        fc_freq = {static_cast<uint64_t>(std::ceil(double(recv_buff_size) * ratio)),
+            uhd::rfnoc::MAX_FC_FREQ_PKTS};
+    }
 
     stream_buff_params_t fc_headroom = {0, 0};
 
@@ -293,7 +298,7 @@ mpmd_mboard_impl::mpmd_mb_iface::make_tx_data_transport(
     uhd::transport::send_link_if::sptr send_link;
     uhd::transport::recv_link_if::sptr recv_link;
     bool lossy_xport;
-    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport) =
+    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport, std::ignore) =
         _link_if_mgr->get_link(
             link_idx, uhd::transport::link_type_t::TX_DATA, xport_args);
 
diff --git a/host/lib/usrp/x300/x300_eth_mgr.cpp b/host/lib/usrp/x300/x300_eth_mgr.cpp
index fd826e484..02f9abe0f 100644
--- a/host/lib/usrp/x300/x300_eth_mgr.cpp
+++ b/host/lib/usrp/x300/x300_eth_mgr.cpp
@@ -263,25 +263,27 @@ both_links_t eth_manager::get_links(link_type_t link_type,
     link_params.num_send_frames = std::max(uhd::rfnoc::MIN_NUM_FRAMES, link_params.num_send_frames);
     link_params.num_recv_frames = std::max(uhd::rfnoc::MIN_NUM_FRAMES, link_params.num_recv_frames);
 
-    // Dummy variables for populating the return tuple
-    size_t recv_buff_size, send_buff_size;
     if (_args.get_use_dpdk()) {
 #ifdef HAVE_DPDK
-         auto link = uhd::transport::udp_dpdk_link::make(
-             conn.addr, BOOST_STRINGIZE(X300_VITA_UDP_PORT), link_params);
-         return std::make_tuple(link, send_buff_size, link, recv_buff_size, true);
+        auto link = uhd::transport::udp_dpdk_link::make(
+            conn.addr, BOOST_STRINGIZE(X300_VITA_UDP_PORT), link_params);
+        return std::make_tuple(link,
+            link_params.send_buff_size,
+            link,
+            link_params.recv_buff_size,
+            true,
+            true);
 #else
         UHD_LOG_WARNING("X300", "Cannot create DPDK transport, falling back to UDP");
 #endif
     }
-
-    // No DPDK, then return regular/kernel UDP
     auto link = uhd::transport::udp_boost_asio_link::make(conn.addr,
         BOOST_STRINGIZE(X300_VITA_UDP_PORT),
         link_params,
-        recv_buff_size,
-        send_buff_size);
-    return std::make_tuple(link, send_buff_size, link, recv_buff_size, true);
+        link_params.recv_buff_size,
+        link_params.send_buff_size);
+    return std::make_tuple(
+        link, link_params.send_buff_size, link, link_params.recv_buff_size, true, false);
 }
 
 /******************************************************************************
diff --git a/host/lib/usrp/x300/x300_mb_iface.cpp b/host/lib/usrp/x300/x300_mb_iface.cpp
index e0c426138..412b28a92 100644
--- a/host/lib/usrp/x300/x300_mb_iface.cpp
+++ b/host/lib/usrp/x300/x300_mb_iface.cpp
@@ -98,7 +98,7 @@ uhd::rfnoc::chdr_ctrl_xport::sptr x300_impl::x300_mb_iface::make_ctrl_transport(
     send_link_if::sptr send_link;
     recv_link_if::sptr recv_link;
     bool lossy_xport;
-    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport) =
+    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport, std::ignore) =
         _conn_mgr->get_links(link_type_t::CTRL,
             local_device_id,
             local_epid,
@@ -139,8 +139,8 @@ uhd::rfnoc::chdr_rx_data_xport::uptr x300_impl::x300_mb_iface::make_rx_data_tran
     send_link_if::sptr send_link;
     recv_link_if::sptr recv_link;
     size_t recv_buff_size;
-    bool lossy_xport;
-    std::tie(send_link, std::ignore, recv_link, recv_buff_size, lossy_xport) =
+    bool lossy_xport, packet_fc;
+    std::tie(send_link, std::ignore, recv_link, recv_buff_size, lossy_xport, packet_fc) =
         _conn_mgr->get_links(link_type_t::RX_DATA,
             local_sep_addr.first,
             local_epid,
@@ -150,16 +150,21 @@ uhd::rfnoc::chdr_rx_data_xport::uptr x300_impl::x300_mb_iface::make_rx_data_tran
     /* Associate local device ID with the adapter */
     _adapter_map[local_sep_addr.first] = send_link->get_send_adapter_id();
 
-    // TODO: configure this based on the transport type
-    const uhd::rfnoc::stream_buff_params_t recv_capacity = {
-        recv_buff_size, uhd::rfnoc::MAX_FC_CAPACITY_PKTS};
+    const uhd::rfnoc::stream_buff_params_t recv_capacity = {recv_buff_size,
+        packet_fc ? static_cast<uint32_t>(recv_link->get_num_recv_frames())
+                  : uhd::rfnoc::MAX_FC_CAPACITY_PKTS};
 
     const double ratio = 1.0 / 32;
 
-    // Configure flow control frequency to use bytes only for UDP
-    uhd::rfnoc::stream_buff_params_t fc_freq = {
-        static_cast<uint64_t>(std::ceil(double(recv_buff_size) * ratio)),
-        uhd::rfnoc::MAX_FC_FREQ_PKTS};
+    // Configure flow control frequency to use either bytes only or packets only
+    uhd::rfnoc::stream_buff_params_t fc_freq;
+    if (packet_fc) {
+        fc_freq = {uhd::rfnoc::MAX_FC_FREQ_BYTES,
+            static_cast<uint32_t>(std::ceil(recv_link->get_num_recv_frames() * ratio))};
+    } else {
+        fc_freq = {static_cast<uint64_t>(std::ceil(double(recv_buff_size) * ratio)),
+            uhd::rfnoc::MAX_FC_FREQ_PKTS};
+    }
 
     uhd::rfnoc::stream_buff_params_t fc_headroom = {0, 0};
 
@@ -220,7 +225,7 @@ uhd::rfnoc::chdr_tx_data_xport::uptr x300_impl::x300_mb_iface::make_tx_data_tran
     send_link_if::sptr send_link;
     recv_link_if::sptr recv_link;
     bool lossy_xport;
-    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport) =
+    std::tie(send_link, std::ignore, recv_link, std::ignore, lossy_xport, std::ignore) =
         _conn_mgr->get_links(link_type_t::TX_DATA,
             local_sep_addr.first,
             local_epid,
diff --git a/host/lib/usrp/x300/x300_pcie_mgr.cpp b/host/lib/usrp/x300/x300_pcie_mgr.cpp
index 41d9407f9..d2aad7e3c 100644
--- a/host/lib/usrp/x300/x300_pcie_mgr.cpp
+++ b/host/lib/usrp/x300/x300_pcie_mgr.cpp
@@ -351,14 +351,13 @@ both_links_t pcie_manager::get_links(link_type_t link_type,
     // Note: The nirio_link object's factory has a lot of code for sanity
     // checking the link params, and merging the link_args with the default
     // link_params, so we use that.
-    link_params_t link_params      = get_default_link_params(link_type);
+    link_params_t link_params = get_default_link_params(link_type);
 
     // PCIe: Lossless, and little endian
     size_t recv_buff_size, send_buff_size;
-    auto link = nirio_link::make(_rio_fpga_interface,
-        dma_channel_num,
-        link_params,
-        link_args);
+    auto link =
+        nirio_link::make(_rio_fpga_interface, dma_channel_num, link_params, link_args);
 
-    return std::make_tuple(link, send_buff_size, link, recv_buff_size, false /*not lossy*/);
+    return std::make_tuple(
+        link, send_buff_size, link, recv_buff_size, false /*not lossy*/, false);
 }