rfnoc: Add chdr_ctrl_xport

chdr_ctrl_xport is a dumb-pipe transport for RFNoC control transactions
and management frames.

Also remove the I/O service's check on num_recv_frames and num_send_frames.
The transports may request additional virtual channels, so the send_io_if
and recv_io_if may not reserve additional frames, as they are shared with
a previously-allocated instance.

Note: this uses a mutex to force sequentual access to the
chdr_ctrl_xport. This is supposed to go away when the multi threaded
xport is done.

4 files changed, 276 insertions, 4 deletions

diff --git a/host/lib/include/uhdlib/rfnoc/chdr_ctrl_xport.hpp b/host/lib/include/uhdlib/rfnoc/chdr_ctrl_xport.hpp
new file mode 100644
index 000000000..3365dcb23
--- /dev/null
+++ b/host/lib/include/uhdlib/rfnoc/chdr_ctrl_xport.hpp
@@ -0,0 +1,128 @@
+//
+// Copyright 2019 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+
+#ifndef INCLUDED_RFNOC_CHDR_CTRL_XPORT_HPP
+#define INCLUDED_RFNOC_CHDR_CTRL_XPORT_HPP
+
+#include <uhdlib/rfnoc/chdr_types.hpp>
+#include <uhdlib/transport/io_service.hpp>
+#include <mutex>
+
+namespace uhd { namespace rfnoc {
+
+/*!
+ * Represents a transport for CHDR control streams and management streams
+ */
+class chdr_ctrl_xport
+{
+public:
+    using io_service   = uhd::transport::io_service;
+    using frame_buff   = uhd::transport::frame_buff;
+    using send_link_if = uhd::transport::send_link_if;
+    using recv_link_if = uhd::transport::recv_link_if;
+    using send_io_if   = uhd::transport::send_io_if;
+    using recv_io_if   = uhd::transport::recv_io_if;
+
+    using sptr = std::shared_ptr<chdr_ctrl_xport>;
+
+    /*!
+     * Make a chdr_ctrl_xport
+     *
+     * \param io_srv The I/O service that send_link and recv_link are attached to
+     * \param send_link Link to use for sending packets
+     * \param recv_link Link to use for receiving packets
+     * \param my_epid The local EPID for this transport
+     * \param num_send_frames Number of frames to reserve for TX
+     * \param num_recv_frames Number of frames to reserve for RX
+     */
+    static sptr make(io_service::sptr io_srv,
+        send_link_if::sptr send_link,
+        recv_link_if::sptr recv_link,
+        sep_id_t my_epid,
+        size_t num_send_frames,
+        size_t num_recv_frames)
+    {
+        return std::make_shared<chdr_ctrl_xport>(io_srv, send_link, recv_link, my_epid,
+            num_send_frames, num_recv_frames);
+    }
+
+    /*!
+     * Make a chdr_ctrl_xport
+     *
+     * \param io_srv The I/O service that send_link and recv_link are attached to
+     * \param send_link Link to use for sending packets
+     * \param recv_link Link to use for receiving packets
+     * \param my_epid The local EPID for this transport
+     * \param num_send_frames Number of frames to reserve for TX
+     * \param num_recv_frames Number of frames to reserve for RX
+     */
+    chdr_ctrl_xport(io_service::sptr io_srv,
+        send_link_if::sptr send_link,
+        recv_link_if::sptr recv_link,
+        sep_id_t my_epid,
+        size_t num_send_frames,
+        size_t num_recv_frames);
+
+    ~chdr_ctrl_xport() = default;
+
+    /*!
+      * Get an empty frame buffer in which to write packet contents.
+      *
+      * \param timeout_ms a positive timeout value specifies the maximum number
+                          of ms to wait, a negative value specifies to block
+                          until successful, and a value of 0 specifies no wait.
+      * \return a frame buffer, or null uptr if timeout occurs
+      */
+    frame_buff::uptr get_send_buff(int32_t timeout_ms);
+
+    /*!
+     * Release a frame buffer, allowing the driver to reuse it.
+     *
+     * \param buffer frame buffer to release for reuse by the link
+     */
+    void release_send_buff(frame_buff::uptr buff);
+
+    /*!
+     * Attempt to get a frame buffer with data from the recv link.
+     *
+     * Receives packets from the control stream.
+     *
+     * \param timeout_ms a positive timeout value specifies the maximum number
+                         of ms to wait, a negative value specifies to block
+                         until successful, and a value of 0 specifies no wait.
+     * \return a frame buffer, or null uptr if timeout occurs
+     */
+    frame_buff::uptr get_recv_buff(int32_t timeout_ms);
+
+    /*!
+     * FIXME: Remove or alter when get threaded_io_service
+     *
+     * Receives packets from the management stream.
+     */
+    frame_buff::uptr get_mgmt_buff(int32_t timeout_ms);
+
+    /*!
+     * Release a frame buffer, allowing the recv link driver to reuse it.
+     *
+     * \param buffer frame buffer to release for reuse by the link
+     */
+    void release_recv_buff(frame_buff::uptr buff);
+
+private:
+    chdr_ctrl_xport(const chdr_ctrl_xport&) = delete;
+
+    sep_id_t _my_epid;
+    send_io_if::sptr _send_if;
+    recv_io_if::sptr _ctrl_recv_if;
+    recv_io_if::sptr _mgmt_recv_if;
+
+    // FIXME: Remove this when have threaded_io_service
+    std::mutex _mutex;
+};
+
+}} // namespace uhd::rfnoc
+
+#endif /* INCLUDED_UHDLIB_RFNOC_CHDR_CTRL_XPORT_HPP */
diff --git a/host/lib/rfnoc/CMakeLists.txt b/host/lib/rfnoc/CMakeLists.txt
index 0f9d62c51..93e7ff5b2 100644
--- a/host/lib/rfnoc/CMakeLists.txt
+++ b/host/lib/rfnoc/CMakeLists.txt
@@ -22,6 +22,7 @@ LIBUHD_APPEND_SOURCES(
     ${CMAKE_CURRENT_SOURCE_DIR}/block_id.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/chdr_types.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/chdr_packet.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/chdr_ctrl_xport.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/client_zero.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ctrl_iface.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/device_id.cpp
diff --git a/host/lib/rfnoc/chdr_ctrl_xport.cpp b/host/lib/rfnoc/chdr_ctrl_xport.cpp
new file mode 100644
index 000000000..72fe5bf7f
--- /dev/null
+++ b/host/lib/rfnoc/chdr_ctrl_xport.cpp
@@ -0,0 +1,147 @@
+//
+// Copyright 2019 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+
+#include <uhd/utils/log.hpp>
+#include <uhdlib/rfnoc/chdr_ctrl_xport.hpp>
+#include <uhdlib/rfnoc/chdr_types.hpp>
+
+using namespace uhd;
+using namespace uhd::rfnoc;
+using namespace uhd::rfnoc::chdr;
+using namespace uhd::transport;
+
+chdr_ctrl_xport::chdr_ctrl_xport(io_service::sptr io_srv,
+    send_link_if::sptr send_link,
+    recv_link_if::sptr recv_link,
+    sep_id_t my_epid,
+    size_t num_send_frames,
+    size_t num_recv_frames)
+    : _my_epid(my_epid)
+{
+    /* Make dumb send pipe */
+    send_io_if::send_callback_t send_cb = [this](frame_buff::uptr& buff,
+                                              send_link_if* link) {
+        link->release_send_buff(std::move(buff));
+    };
+    _send_if = io_srv->make_send_client(
+        send_link, num_send_frames, send_cb, recv_link_if::sptr(), 0, nullptr);
+
+    /* Make dumb recv pipe that matches management and control packets */
+    uhd::transport::recv_callback_t ctrl_recv_cb = [this](frame_buff::uptr& buff,
+                                                       recv_link_if* /*recv_link*/,
+                                                       send_link_if
+                                                           * /*send_link*/) -> bool {
+        uint64_t* data = (uint64_t*)buff->data();
+        auto hdr       = chdr_header(uhd::ntohx<uint64_t>(*data));
+        auto pkt_type  = hdr.get_pkt_type();
+        auto dst_epid  = hdr.get_dst_epid();
+
+        /* Check type and destination EPID */
+        if ((pkt_type == PKT_TYPE_CTRL) && (dst_epid == _my_epid)) {
+            return true;
+        } else {
+            return false;
+        }
+        return false;
+    };
+    recv_io_if::fc_callback_t release_cb =
+        [](frame_buff::uptr buff, recv_link_if* link, send_link_if* /*send_link*/) {
+            link->release_recv_buff(std::move(buff));
+        };
+
+    _ctrl_recv_if = io_srv->make_recv_client(recv_link,
+        num_recv_frames,
+        ctrl_recv_cb,
+        send_link_if::sptr(),
+        0,
+        release_cb);
+
+    uhd::transport::recv_callback_t mgmt_recv_cb = [this](frame_buff::uptr& buff,
+                                                       recv_link_if* /*recv_link*/,
+                                                       send_link_if
+                                                           * /*send_link*/) -> bool {
+        uint64_t* data = (uint64_t*)buff->data();
+        auto hdr       = chdr_header(uhd::ntohx<uint64_t>(*data));
+        auto pkt_type  = hdr.get_pkt_type();
+        auto dst_epid  = hdr.get_dst_epid();
+
+        /* Check type and destination EPID */
+        if ((pkt_type == PKT_TYPE_MGMT) && (dst_epid == _my_epid)) {
+            return true;
+        } else {
+            return false;
+        }
+        return false;
+    };
+
+    // No additional frames reserved specifically for this virtual interface
+    _mgmt_recv_if = io_srv->make_recv_client(
+        recv_link, 0, mgmt_recv_cb, send_link_if::sptr(), 0, release_cb);
+}
+
+/*!
+ * Get an empty frame buffer in which to write packet contents.
+ *
+ * \param timeout_ms a positive timeout value specifies the maximum number
+                     of ms to wait, a negative value specifies to block
+                     until successful, and a value of 0 specifies no wait.
+ * \return a frame buffer, or null uptr if timeout occurs
+ */
+frame_buff::uptr chdr_ctrl_xport::get_send_buff(int32_t timeout_ms)
+{
+    // FIXME: Remove mutex when have threaded_io_service
+    std::lock_guard<std::mutex> lock(_mutex);
+    frame_buff::uptr buff = _send_if->get_send_buff(timeout_ms);
+    if (!buff) {
+        return frame_buff::uptr();
+    }
+    return frame_buff::uptr(std::move(buff));
+}
+
+/*!
+ * Release a frame buffer, allowing the driver to reuse it.
+ *
+ * \param buffer frame buffer to release for reuse by the link
+ */
+void chdr_ctrl_xport::release_send_buff(frame_buff::uptr buff)
+{
+    // FIXME: Remove mutex when have threaded_io_service
+    std::lock_guard<std::mutex> lock(_mutex);
+    _send_if->release_send_buff(std::move(buff));
+}
+
+/*!
+ * Attempt to get a frame buffer with data from the recv link.
+ *
+ * \param timeout_ms a positive timeout value specifies the maximum number
+                     of ms to wait, a negative value specifies to block
+                     until successful, and a value of 0 specifies no wait.
+ * \return a frame buffer, or null uptr if timeout occurs
+ */
+frame_buff::uptr chdr_ctrl_xport::get_recv_buff(int32_t timeout_ms)
+{
+    // FIXME: Remove mutex when have threaded_io_service
+    std::lock_guard<std::mutex> lock(_mutex);
+    return _ctrl_recv_if->get_recv_buff(timeout_ms);
+}
+
+frame_buff::uptr chdr_ctrl_xport::get_mgmt_buff(int32_t timeout_ms)
+{
+    std::lock_guard<std::mutex> lock(_mutex);
+    return _mgmt_recv_if->get_recv_buff(timeout_ms);
+}
+
+/*!
+ * Release a frame buffer, allowing the recv link driver to reuse it.
+ *
+ * \param buffer frame buffer to release for reuse by the link
+ */
+void chdr_ctrl_xport::release_recv_buff(frame_buff::uptr buff)
+{
+    // FIXME: Remove mutex when have threaded_io_service
+    std::lock_guard<std::mutex> lock(_mutex);
+    _ctrl_recv_if->release_recv_buff(std::move(buff));
+}
diff --git a/host/lib/transport/inline_io_service.cpp b/host/lib/transport/inline_io_service.cpp
index 72acea738..942c99d5f 100644
--- a/host/lib/transport/inline_io_service.cpp
+++ b/host/lib/transport/inline_io_service.cpp
@@ -272,10 +272,8 @@ recv_io_if::sptr inline_io_service::make_recv_client(recv_link_if::sptr data_lin
     recv_io_if::fc_callback_t fc_cb)
 {
     UHD_ASSERT_THROW(data_link);
-    UHD_ASSERT_THROW(num_recv_frames > 0);
     UHD_ASSERT_THROW(cb);
     if (fc_link) {
-        UHD_ASSERT_THROW(num_send_frames > 0);
         UHD_ASSERT_THROW(fc_cb);
         connect_sender(fc_link.get(), num_send_frames);
     }
@@ -302,14 +300,12 @@ send_io_if::sptr inline_io_service::make_send_client(send_link_if::sptr send_lin
     recv_callback_t recv_cb)
 {
     UHD_ASSERT_THROW(send_link);
-    UHD_ASSERT_THROW(num_send_frames > 0);
     UHD_ASSERT_THROW(send_cb);
     connect_sender(send_link.get(), num_send_frames);
     sptr io_srv  = shared_from_this();
     auto send_io = std::make_shared<inline_send_io>(
         io_srv, send_link, num_send_frames, send_cb, recv_link, num_recv_frames, recv_cb);
     if (recv_link) {
-        UHD_ASSERT_THROW(num_recv_frames > 0);
         UHD_ASSERT_THROW(recv_cb);
         connect_receiver(recv_link.get(), send_io.get(), num_recv_frames);
     }