diff options
Diffstat (limited to 'host/lib/usrp/mpmd')
27 files changed, 1384 insertions, 1233 deletions
diff --git a/host/lib/usrp/mpmd/CMakeLists.txt b/host/lib/usrp/mpmd/CMakeLists.txt index 21511e6da..18c9c6cbd 100644 --- a/host/lib/usrp/mpmd/CMakeLists.txt +++ b/host/lib/usrp/mpmd/CMakeLists.txt @@ -21,21 +21,21 @@ if(ENABLE_MPMD) ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_impl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_mboard_impl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_mb_controller.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_mb_iface.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_prop_tree.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_xport.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_xport_mgr.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_xport_ctrl_udp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_link_if_mgr.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_link_if_ctrl_udp.cpp ) if(ENABLE_LIBERIO) LIBUHD_APPEND_SOURCES( - ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_xport_ctrl_liberio.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_link_if_ctrl_liberio.cpp ) endif(ENABLE_LIBERIO) if(ENABLE_DPDK) LIBUHD_APPEND_SOURCES( - ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_xport_ctrl_dpdk_udp.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_link_if_ctrl_dpdk_udp.cpp ) endif(ENABLE_DPDK) diff --git a/host/lib/usrp/mpmd/mpmd_find.cpp b/host/lib/usrp/mpmd/mpmd_find.cpp index 2b8e1350d..3e8bcc72f 100644 --- a/host/lib/usrp/mpmd/mpmd_find.cpp +++ b/host/lib/usrp/mpmd/mpmd_find.cpp @@ -8,6 +8,7 @@ #include "mpmd_devices.hpp" #include "mpmd_impl.hpp" +#include "mpmd_link_if_mgr.hpp" #include <uhdlib/transport/dpdk_common.hpp> #include <uhd/transport/if_addrs.hpp> #include <uhd/transport/udp_simple.hpp> @@ -89,7 +90,7 @@ device_addrs_t mpmd_find_with_addr( // Create result to return device_addr_t new_addr; - new_addr[xport::MGMT_ADDR_KEY] = recv_addr; + new_addr[MGMT_ADDR_KEY] = recv_addr; new_addr["type"] = "mpmd"; // hwd will overwrite this // remove ident string and put other informations into device_args dict result.erase(result.begin()); @@ -101,7 +102,7 @@ device_addrs_t mpmd_find_with_addr( el, [](const char& in) { return in == '='; }, boost::token_compress_on); - if (value[0] != xport::MGMT_ADDR_KEY) { + if (value[0] != MGMT_ADDR_KEY) { new_addr[value[0]] = value[1]; } } @@ -132,12 +133,12 @@ device_addrs_t mpmd_find_with_addrs(const device_addrs_t& hints) found_devices.reserve(hints.size()); for (const auto& hint : hints) { if (not(hint.has_key(xport::FIRST_ADDR_KEY) - or hint.has_key(xport::MGMT_ADDR_KEY))) { + or hint.has_key(MGMT_ADDR_KEY))) { UHD_LOG_DEBUG("MPMD FIND", "No address given in hint " << hint.to_string()); continue; } const std::string mgmt_addr = - hint.get(xport::MGMT_ADDR_KEY, hint.get(xport::FIRST_ADDR_KEY, "")); + hint.get(MGMT_ADDR_KEY, hint.get(xport::FIRST_ADDR_KEY, "")); device_addrs_t reply_addrs = mpmd_find_with_addr(mgmt_addr, hint); if (reply_addrs.size() > 1) { UHD_LOG_ERROR("MPMD", @@ -217,7 +218,7 @@ device_addrs_t mpmd_find(const device_addr_t& hint_) // Scenario 1): User gave us at least one address if (not hints.empty() and (hints[0].has_key(xport::FIRST_ADDR_KEY) - or hints[0].has_key(xport::MGMT_ADDR_KEY))) { + or hints[0].has_key(MGMT_ADDR_KEY))) { // Note: We don't try and connect to the devices in this mode, because // we only get here if the user specified addresses, and we assume she // knows what she's doing. diff --git a/host/lib/usrp/mpmd/mpmd_image_loader.cpp b/host/lib/usrp/mpmd/mpmd_image_loader.cpp index d5c7d3da9..3a285da3e 100644 --- a/host/lib/usrp/mpmd/mpmd_image_loader.cpp +++ b/host/lib/usrp/mpmd/mpmd_image_loader.cpp @@ -13,6 +13,7 @@ #include <uhd/types/component_file.hpp> #include <uhd/types/eeprom.hpp> #include <uhd/utils/paths.hpp> +#include <uhd/utils/static.hpp> #include <boost/algorithm/string.hpp> #include <boost/filesystem/convenience.hpp> #include <fstream> @@ -216,6 +217,7 @@ static bool mpmd_image_loader(const image_loader::image_loader_args_t& image_loa return true; } + }} // namespace uhd:: UHD_STATIC_BLOCK(register_mpm_image_loader) diff --git a/host/lib/usrp/mpmd/mpmd_impl.cpp b/host/lib/usrp/mpmd/mpmd_impl.cpp index 05d847060..30a3c5804 100644 --- a/host/lib/usrp/mpmd/mpmd_impl.cpp +++ b/host/lib/usrp/mpmd/mpmd_impl.cpp @@ -14,11 +14,11 @@ #include <uhd/utils/tasks.hpp> #include <uhdlib/rfnoc/radio_ctrl_impl.hpp> #include <uhdlib/rfnoc/rpc_block_ctrl.hpp> -#include <../device3/device3_impl.hpp> #include <boost/algorithm/string.hpp> #include <boost/asio.hpp> #include <boost/make_shared.hpp> #include <boost/thread.hpp> +#include <chrono> #include <future> #include <memory> #include <mutex> @@ -34,11 +34,10 @@ namespace { /************************************************************************* * Local constants ************************************************************************/ -const size_t MPMD_CROSSBAR_MAX_LADDR = 255; //! Most pessimistic time for a CHDR query to go to device and back const double MPMD_CHDR_MAX_RTT = 0.02; -//! MPM Compatibility number -const std::vector<size_t> MPM_COMPAT_NUM = {1, 2}; +//! MPM Compatibility number {MAJOR, MINOR} +const std::vector<size_t> MPM_COMPAT_NUM = {2, 0}; /************************************************************************* * Helper functions @@ -155,7 +154,7 @@ const std::string mpmd_impl::MPM_ECHO_CMD = "MPM-ECHO"; * Structors ****************************************************************************/ mpmd_impl::mpmd_impl(const device_addr_t& device_args) - : usrp::device3_impl(), _device_args(device_args) + : rfnoc_device(), _device_args(device_args) { const device_addrs_t mb_args = separate_device_addr(device_args); const size_t num_mboards = mb_args.size(); @@ -175,20 +174,12 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args) _mb.push_back(claim_and_make(mb_args[mb_i])); } - // Next figure out the number of base xport addresses. This way, we - // can run _mb[*]->init() in parallel on all the _mb. - // This can *not* be parallelized. - std::vector<size_t> base_xport_addr(num_mboards, 2); // Starts at 2 [sic] - for (size_t mb_i = 0; mb_i < num_mboards - 1; ++mb_i) { - base_xport_addr[mb_i + 1] = base_xport_addr[mb_i] + _mb[mb_i]->num_xbars; - } - if (not skip_init) { // Run the actual device initialization. This can run in parallel. for (size_t mb_i = 0; mb_i < num_mboards; ++mb_i) { // Note: This is the only place we do compat number checks. They're // effectively disabled for skip_init=1 - setup_mb(_mb[mb_i].get(), mb_i, base_xport_addr[mb_i]); + setup_mb(_mb[mb_i].get(), mb_i); } } else { UHD_LOG_DEBUG("MPMD", "Claimed device, but skipped init."); @@ -205,12 +196,6 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args) } if (not skip_init) { - // This can be parallelized, because the blocks of individual mboards - // live on different subtrees. - for (size_t mb_i = 0; mb_i < mb_args.size(); ++mb_i) { - setup_rfnoc_blocks(_mb[mb_i].get(), mb_i, mb_args[mb_i]); - } - // FIXME this section only makes sense for when the time source is external. // So, check for that, or something similar. // This section of code assumes that the prop tree is set and we have access @@ -218,12 +203,6 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args) if (device_args.has_key("sync_time")) { reset_time_synchronized(_tree); } - - auto filtered_block_args = device_args; // TODO actually filter - // Blocks will finalize their own setup in this function. They have - // (and might need) full access to the prop tree, the timekeepers, etc. - // This is already internally parallelized. - setup_rpc_blocks(filtered_block_args, serialize_init); } else { UHD_LOG_INFO("MPMD", "Claimed device without full initialization."); } @@ -231,7 +210,6 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args) mpmd_impl::~mpmd_impl() { - _rfnoc_block_ctrl.clear(); _tree.reset(); _mb.clear(); } @@ -241,7 +219,7 @@ mpmd_impl::~mpmd_impl() ****************************************************************************/ mpmd_mboard_impl::uptr mpmd_impl::claim_and_make(const uhd::device_addr_t& device_args) { - const std::string rpc_addr = device_args.get(xport::MGMT_ADDR_KEY); + const std::string rpc_addr = device_args.get(MGMT_ADDR_KEY); UHD_LOGGER_DEBUG("MPMD") << "Device args: `" << device_args.to_string() << "'. RPC address: " << rpc_addr; @@ -254,8 +232,7 @@ mpmd_mboard_impl::uptr mpmd_impl::claim_and_make(const uhd::device_addr_t& devic return mpmd_mboard_impl::make(device_args, rpc_addr); } -void mpmd_impl::setup_mb( - mpmd_mboard_impl* mb, const size_t mb_index, const size_t base_xport_addr) +void mpmd_impl::setup_mb(mpmd_mboard_impl* mb, const size_t mb_index) { assert_compat_number_throw("MPM", MPM_COMPAT_NUM, @@ -264,94 +241,8 @@ void mpmd_impl::setup_mb( UHD_LOG_DEBUG("MPMD", "Initializing mboard " << mb_index); mb->init(); - for (size_t xbar_index = 0; xbar_index < mb->num_xbars; xbar_index++) { - mb->set_xbar_local_addr(xbar_index, base_xport_addr + xbar_index); - } -} - -void mpmd_impl::setup_rfnoc_blocks(mpmd_mboard_impl* mb, - const size_t mb_index, - const uhd::device_addr_t& ctrl_xport_args) -{ - UHD_LOG_TRACE( - "MPMD", "Mboard " << mb_index << " reports " << mb->num_xbars << " crossbar(s)."); - // TODO: The args apply to all xbars, which may or may not be true - for (size_t xbar_index = 0; xbar_index < mb->num_xbars; xbar_index++) { - // Pull the number of blocks and base port from the args, if available. - // Otherwise, get the values from MPM. - const size_t num_blocks = - ctrl_xport_args.has_key("rfnoc_num_blocks") - ? ctrl_xport_args.cast<size_t>("rfnoc_num_blocks", 0) - : mb->rpc->request<size_t>("get_num_blocks", xbar_index); - const size_t base_port = - ctrl_xport_args.has_key("rfnoc_base_port") - ? ctrl_xport_args.cast<size_t>("rfnoc_base_port", 0) - : mb->rpc->request<size_t>("get_base_port", xbar_index); - const size_t local_addr = mb->get_xbar_local_addr(xbar_index); - UHD_LOGGER_TRACE("MPMD") - << "Enumerating RFNoC blocks for xbar " << xbar_index - << ". Total blocks: " << num_blocks << " Base port: " << base_port - << " Local address: " << local_addr; - if (ctrl_xport_args.has_key("rfnoc_num_blocks") - or ctrl_xport_args.has_key("rfnoc_base_port")) { - // TODO: Remove this warning once we're confident this is - // (relatively) safe and useful. Also add documentation to - // usrp_n3xx.dox - UHD_LOGGER_WARNING("MPMD") - << "Overriding default RFNoC configuration. You are using an " - << "experimental development feature, which may go away in " - << "future versions."; - } - - try { - enumerate_rfnoc_blocks(mb_index, - num_blocks, - base_port, - uhd::sid_t(0, 0, local_addr, 0), - ctrl_xport_args); - } catch (const std::exception& ex) { - UHD_LOGGER_ERROR("MPMD") << "Failure during block enumeration: " << ex.what(); - throw uhd::runtime_error("Failed to run enumerate_rfnoc_blocks()"); - } - } -} - -void mpmd_impl::setup_rpc_blocks( - const device_addr_t& block_args, const bool serialize_init) -{ - std::vector<std::future<void>> task_list; - // If we don't force async, most compilers, at least now, will default to - // deferred. - const auto launch_policy = serialize_init ? std::launch::deferred - : std::launch::async; - - // Preload all the tasks (they might start running on emplace_back) - for (const auto& block_ctrl : _rfnoc_block_ctrl) { - auto rpc_block_id = block_ctrl->get_block_id(); - if (has_block<uhd::rfnoc::rpc_block_ctrl>(rpc_block_id)) { - const size_t mboard_idx = rpc_block_id.get_device_no(); - auto rpc_block_ctrl = - get_block_ctrl<uhd::rfnoc::rpc_block_ctrl>(rpc_block_id); - auto rpc_sptr = _mb[mboard_idx]->rpc; - task_list.emplace_back(std::async( - launch_policy, [rpc_block_id, rpc_block_ctrl, &block_args, rpc_sptr]() { - UHD_LOGGER_DEBUG("MPMD") - << "Adding RPC access to block: " << rpc_block_id - << " Block args: " << block_args.to_string(); - rpc_block_ctrl->set_rpc_client(rpc_sptr, block_args); - })); - } - } - - // Execute all the calls to set_rpc_client(), either concurrently, or - // serially - for (auto& task : task_list) { - task.get(); - } -} - -size_t mpmd_impl::get_mtu(const size_t mb_index, const uhd::direction_t dir) { - return _mb[mb_index]->get_mtu(dir); + UHD_ASSERT_THROW(mb->mb_ctrl); + register_mb_controller(mb_index, mb->mb_ctrl); } /***************************************************************************** diff --git a/host/lib/usrp/mpmd/mpmd_impl.hpp b/host/lib/usrp/mpmd/mpmd_impl.hpp index bdb6bd691..e1dde49b5 100644 --- a/host/lib/usrp/mpmd/mpmd_impl.hpp +++ b/host/lib/usrp/mpmd/mpmd_impl.hpp @@ -7,14 +7,14 @@ #ifndef INCLUDED_MPMD_IMPL_HPP #define INCLUDED_MPMD_IMPL_HPP -#include "../device3/device3_impl.hpp" -#include "mpmd_xport_mgr.hpp" #include <uhd/property_tree.hpp> #include <uhd/stream.hpp> -#include <uhd/transport/muxed_zero_copy_if.hpp> #include <uhd/types/device_addr.hpp> #include <uhd/types/dict.hpp> #include <uhd/utils/tasks.hpp> +#include <uhdlib/rfnoc/rfnoc_device.hpp> +#include <uhdlib/rfnoc/clock_iface.hpp> +#include <uhdlib/usrp/common/mpmd_mb_controller.hpp> #include <uhdlib/utils/rpc.hpp> #include <boost/optional.hpp> #include <map> @@ -34,6 +34,8 @@ static constexpr size_t MPMD_DEFAULT_RPC_TIMEOUT = 2000; static constexpr size_t MPMD_SHORT_RPC_TIMEOUT = 2000; //! Claimer loop timeout value for RPC calls (ms). static constexpr size_t MPMD_CLAIMER_RPC_TIMEOUT = 10000; +//! Ethernet address for management and RPC communication +static const std::string MGMT_ADDR_KEY = "mgmt_addr"; namespace uhd { namespace mpmd { @@ -46,6 +48,11 @@ public: using uptr = std::unique_ptr<mpmd_mboard_impl>; using dev_info = std::map<std::string, std::string>; + //! MPMD-specific implementation of the mb_iface + // + // This handles the transport management + class mpmd_mb_iface; + /*** Static helper *******************************************************/ /*! Will run some checks to determine if this device can be reached from * the current UHD session @@ -73,9 +80,11 @@ public: */ static uptr make(const uhd::device_addr_t& mb_args, const std::string& addr); - /*** Init ****************************************************************/ + /*** API *****************************************************************/ void init(); + uhd::rfnoc::mb_iface& get_mb_iface(); + /*** Public attributes ***************************************************/ //! These are the args given by the user, with some filtering/preprocessing uhd::device_addr_t mb_args; @@ -88,6 +97,12 @@ public: // to be populated at all. std::vector<uhd::device_addr_t> dboard_info; + //! Reference to this motherboards mb_iface + std::unique_ptr<mpmd_mb_iface> mb_iface; + + //! Reference to this motherboards mb_controller + uhd::rfnoc::mpmd_mb_controller::sptr mb_ctrl; + /*! Reference to the RPC client for this motherboard * * We store a shared ptr, because we might share it with some of the RFNoC @@ -95,41 +110,9 @@ public: */ uhd::rpc_client::sptr rpc; - //! Number of RFNoC crossbars on this device - const size_t num_xbars; - /************************************************************************* * API ************************************************************************/ - //! Configure a crossbar to have a certain local address - void set_xbar_local_addr(const size_t xbar_index, const size_t local_addr); - - //! Return the local address of a given crossbar - size_t get_xbar_local_addr(const size_t xbar_index) const - { - return xbar_local_addrs.at(xbar_index); - } - - //! Device-specific make_transport implementation - // - // A major difference to the mpmd_impl::make_transport() is the meaning of - // the first argument (\p sid). mpmd_impl::make_transport() will add a - // source part to the SID which needs to be taken into account in this - // function. - // - // \param sid The full SID of this transport (UHD to device) - // \param xport_type Transport type (CTRL, RX_DATA, ...) - // \param args Any kind of args passed in via get_?x_stream() - uhd::both_xports_t make_transport(const sid_t& sid, - usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& args); - - size_t get_mtu(const uhd::direction_t dir) const; - - - uhd::device_addr_t get_rx_hints() const; - uhd::device_addr_t get_tx_hints() const; - /*! Setting this flag will enable a mode where a reclaim failure is * acceptable. * @@ -177,18 +160,10 @@ private: /************************************************************************* * Private attributes ************************************************************************/ - //! Stores a list of local addresses of the crossbars. The local address is - // what we use when addressing a crossbar in a CHDR header. - std::vector<size_t> xbar_local_addrs; - /*! Continuously reclaims the device. */ uhd::task::sptr _claimer_task; - uhd::mpmd::xport::mpmd_xport_mgr::uptr _xport_mgr; - uhd::device_addr_t send_args; - uhd::device_addr_t recv_args; - /*! This flag is only used within the claim() function. Go look there if you * really need to know what it does. */ @@ -208,7 +183,7 @@ private: * are taken care of by MPM itself, it is not necessary to write a specific * derived class for every single type of MPM device. */ -class mpmd_impl : public uhd::usrp::device3_impl +class mpmd_impl : public uhd::rfnoc::detail::rfnoc_device { public: //! Device arg key which will allow finding all devices, even those not @@ -242,17 +217,17 @@ public: /************************************************************************** * API ************************************************************************/ - uhd::both_xports_t make_transport(const uhd::sid_t&, - uhd::usrp::device3_impl::xport_type_t, - const uhd::device_addr_t&); - - //! get mtu - size_t get_mtu(const size_t, const uhd::direction_t); + uhd::rfnoc::mb_iface& get_mb_iface(const size_t mb_idx) + { + if (mb_idx >= _mb.size()) { + throw uhd::index_error( + std::string("Cannot get mb_iface, invalid motherboard index: ") + + std::to_string(mb_idx)); + } + return _mb.at(mb_idx)->get_mb_iface(); + } private: - uhd::device_addr_t get_rx_hints(size_t mb_index); - uhd::device_addr_t get_tx_hints(size_t mb_index); - /************************************************************************* * Private methods/helpers ************************************************************************/ @@ -272,21 +247,7 @@ private: * \param device_args Device args * */ - void setup_mb( - mpmd_mboard_impl* mb, const size_t mb_index, const size_t base_xport_addr); - - //! Setup all RFNoC blocks running on mboard \p mb_i - void setup_rfnoc_blocks( - mpmd_mboard_impl* mb, const size_t mb_i, const uhd::device_addr_t& block_args); - - //! Configure all blocks that require access to an RPC client - void setup_rpc_blocks( - const uhd::device_addr_t& block_args, const bool serialize_init); - - /*! Return the index of the motherboard given the local address of a - * crossbar - */ - size_t identify_mboard_by_xbar_addr(const size_t xbar_addr) const; + void setup_mb(mpmd_mboard_impl* mb, const size_t mb_index); /*! Initialize property tree for a single device. * @@ -297,7 +258,6 @@ private: static void init_property_tree( uhd::property_tree::sptr tree, fs_path mb_path, mpmd_mboard_impl* mb); - /************************************************************************* * Private attributes ************************************************************************/ diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_base.hpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_base.hpp new file mode 100644 index 000000000..a9cb456e5 --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_base.hpp @@ -0,0 +1,56 @@ +// +// Copyright 2017 Ettus Research, a National Instruments Company +// Copyright 2019 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_MPMD_XPORT_CTRL_BASE_HPP +#define INCLUDED_MPMD_XPORT_CTRL_BASE_HPP + +#include "mpmd_link_if_mgr.hpp" +#include <uhd/types/device_addr.hpp> +#include <uhdlib/rfnoc/chdr_packet.hpp> +#include <uhdlib/transport/links.hpp> +#include <memory> + +namespace uhd { namespace mpmd { namespace xport { + +/*! Transport manager implementation base + */ +class mpmd_link_if_ctrl_base +{ +public: + using uptr = std::unique_ptr<mpmd_link_if_ctrl_base>; + virtual ~mpmd_link_if_ctrl_base() {} + + virtual size_t get_num_links() const = 0; + + /*! Return links object + * + * \param link_idx The number of the link to use. link_idx < get_num_links() + * must hold true. link_idx is often 0. Example: When + * the underlying transport is Ethernet, and the user + * specified both addr and second_addr, then get_num_links() + * equals 2 and link_idx can also be 1. + * \param link_type CTRL, RX_DATA, or TX_DATA (for configuring the link) + * \param link_args Link-specific additional information that the underlying + * mpmd_link_if_ctrl instantiation can use + */ + virtual uhd::transport::both_links_t get_link(const size_t link_idx, + const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args) = 0; + + //! Return the underlying link's MTU in bytes + virtual size_t get_mtu(const uhd::direction_t dir) const = 0; + + //! Return the rate of the underlying link in bytes/sec + virtual double get_link_rate(const size_t link_idx) const = 0; + + //! Get the packet factory associated with this link + virtual const uhd::rfnoc::chdr::chdr_packet_factory& get_packet_factory() const = 0; +}; + +}}} /* namespace uhd::mpmd::xport */ + +#endif /* INCLUDED_MPMD_XPORT_CTRL_BASE_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_dpdk_udp.cpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_dpdk_udp.cpp index 38d295728..baf0dde3e 100644 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_dpdk_udp.cpp +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_dpdk_udp.cpp @@ -5,8 +5,8 @@ // #include "mpmd_impl.hpp" -#include "mpmd_xport_mgr.hpp" -#include "mpmd_xport_ctrl_dpdk_udp.hpp" +#include "mpmd_link_if_mgr.hpp" +#include "mpmd_link_if_ctrl_dpdk_udp.hpp" #include <uhd/transport/udp_simple.hpp> #include <uhd/transport/udp_constants.hpp> #include <uhdlib/transport/dpdk_zero_copy.hpp> @@ -161,7 +161,7 @@ size_t discover_mtu( } -mpmd_xport_ctrl_dpdk_udp::mpmd_xport_ctrl_dpdk_udp( +mpmd_link_if_ctrl_dpdk_udp::mpmd_link_if_ctrl_dpdk_udp( const uhd::device_addr_t& mb_args ) : _mb_args(mb_args) , _ctx(uhd::transport::uhd_dpdk_ctx::get()) @@ -193,8 +193,8 @@ mpmd_xport_ctrl_dpdk_udp::mpmd_xport_ctrl_dpdk_udp( } uhd::both_xports_t -mpmd_xport_ctrl_dpdk_udp::make_transport( - mpmd_xport_mgr::xport_info_t &xport_info, +mpmd_link_if_ctrl_dpdk_udp::make_transport( + mpmd_link_if_mgr::xport_info_t &xport_info, const usrp::device3_impl::xport_type_t xport_type, const uhd::device_addr_t& xport_args ) { @@ -264,14 +264,14 @@ mpmd_xport_ctrl_dpdk_udp::make_transport( return xports; } -bool mpmd_xport_ctrl_dpdk_udp::is_valid( - const mpmd_xport_mgr::xport_info_t& xport_info +bool mpmd_link_if_ctrl_dpdk_udp::is_valid( + const mpmd_link_if_mgr::xport_info_t& xport_info ) const { int dpdk_port_id = _ctx.get_route(xport_info.at("ipv4")); return (dpdk_port_id >= 0); } -size_t mpmd_xport_ctrl_dpdk_udp::get_mtu(const uhd::direction_t /*dir*/) const +size_t mpmd_link_if_ctrl_dpdk_udp::get_mtu(const uhd::direction_t /*dir*/) const { return _mtu; } diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_dpdk_udp.hpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_dpdk_udp.hpp index 943cd44d6..4423b4340 100644 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_dpdk_udp.hpp +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_dpdk_udp.hpp @@ -7,7 +7,7 @@ #ifndef INCLUDED_MPMD_XPORT_CTRL_DPDK_UDP_HPP #define INCLUDED_MPMD_XPORT_CTRL_DPDK_UDP_HPP -#include "mpmd_xport_ctrl_base.hpp" +#include "mpmd_link_if_ctrl_base.hpp" #include <uhd/types/device_addr.hpp> #include <uhdlib/transport/dpdk_zero_copy.hpp> #include "../device3/device3_impl.hpp" @@ -18,21 +18,21 @@ namespace uhd { namespace mpmd { namespace xport { * * Opens UDP sockets */ -class mpmd_xport_ctrl_dpdk_udp : public mpmd_xport_ctrl_base +class mpmd_link_if_ctrl_dpdk_udp : public mpmd_link_if_ctrl_base { public: - mpmd_xport_ctrl_dpdk_udp( + mpmd_link_if_ctrl_dpdk_udp( const uhd::device_addr_t& mb_args ); both_xports_t make_transport( - mpmd_xport_mgr::xport_info_t& xport_info, + mpmd_link_if_mgr::xport_info_t& xport_info, const usrp::device3_impl::xport_type_t xport_type, const uhd::device_addr_t& xport_args ); bool is_valid( - const mpmd_xport_mgr::xport_info_t& xport_info + const mpmd_link_if_mgr::xport_info_t& xport_info ) const; size_t get_mtu( diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp new file mode 100644 index 000000000..ed8f4395e --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.cpp @@ -0,0 +1,135 @@ +// +// Copyright 2017 Ettus Research, National Instruments Company +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include "mpmd_link_if_ctrl_liberio.hpp" +#include <uhd/rfnoc/constants.hpp> +#include <uhd/utils/byteswap.hpp> +#include <uhd/utils/log.hpp> +#include <uhdlib/transport/inline_io_service.hpp> +#include <uhdlib/transport/liberio_link.hpp> + +using namespace uhd; +using namespace uhd::transport; +using namespace uhd::mpmd::xport; + +const uhd::rfnoc::chdr::chdr_packet_factory mpmd_link_if_ctrl_liberio::_pkt_factory( + uhd::rfnoc::CHDR_W_64, ENDIANNESS_LITTLE); + +namespace { + +//! The default MTU will be this number times the page size +const size_t LIBERIO_PAGES_PER_BUF = 2; +//! The default MTU +const size_t LIBERIO_DEFAULT_MTU = LIBERIO_PAGES_PER_BUF * getpagesize(); +//! The default link_rate (8 Bytes * 200 MHz) +const double LIBERIO_DEFAULT_LINK_RATE = 200e6 * 8; +//! Number of descriptors that liberio allocates (receive) +const size_t LIBERIO_NUM_RECV_FRAMES = 128; +//! Number of descriptors that liberio allocates (send) +const size_t LIBERIO_NUM_SEND_FRAMES = 128; +//! MTU for largest non-data packet accepted (arbitrarily-determined...) +// Note: Management frames must fit here, and it determines the padded RX size +const size_t LIBERIO_MAX_NONDATA_PACKET_SIZE = 128; + +std::vector<mpmd_link_if_ctrl_liberio::liberio_link_info_t> +get_liberio_info_from_xport_info( + const mpmd_link_if_mgr::xport_info_list_t& link_info_list) +{ + std::vector<mpmd_link_if_ctrl_liberio::liberio_link_info_t> result; + for (const auto& link_info : link_info_list) { + if (!link_info.count("tx_dev")) { + UHD_LOG_ERROR("MPMD::XPORT::LIBERIO", + "Invalid response from get_chdr_link_options()! No `tx_dev' key!"); + throw uhd::runtime_error( + "Invalid response from get_chdr_link_options()! No `tx_dev' key!"); + } + if (!link_info.count("rx_dev")) { + UHD_LOG_ERROR("MPMD::XPORT::LIBERIO", + "Invalid response from get_chdr_link_options()! No `rx_dev' key!"); + throw uhd::runtime_error( + "Invalid response from get_chdr_link_options()! No `rx_dev' key!"); + } + const std::string tx_dev = link_info.at("tx_dev"); + const std::string rx_dev = link_info.at("rx_dev"); + result.emplace_back( + mpmd_link_if_ctrl_liberio::liberio_link_info_t{tx_dev, rx_dev}); + } + + return result; +} + +} // namespace + + +/****************************************************************************** + * Structors + *****************************************************************************/ +mpmd_link_if_ctrl_liberio::mpmd_link_if_ctrl_liberio(const uhd::device_addr_t& mb_args, + const mpmd_link_if_mgr::xport_info_list_t& xport_info) + : _mb_args(mb_args) + , _recv_args(filter_args(mb_args, "recv")) + , _send_args(filter_args(mb_args, "send")) + , _dma_channels(get_liberio_info_from_xport_info(xport_info)) + , _link_rate(LIBERIO_DEFAULT_LINK_RATE) // FIXME +{ + // nop +} + +/****************************************************************************** + * API + *****************************************************************************/ +uhd::transport::both_links_t mpmd_link_if_ctrl_liberio::get_link(const size_t link_idx, + const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args) +{ + UHD_ASSERT_THROW(link_idx == 0); + if (_next_channel >= _dma_channels.size()) { + UHD_LOG_ERROR( + "MPMD::XPORT::LIBERIO", "Cannot create liberio link: DMA channels exhausted"); + throw uhd::runtime_error("Cannot create liberio link: DMA channels exhausted"); + } + auto link_info = _dma_channels.at(_next_channel++); + + /* FIXME: Should have common infrastructure for creating I/O services */ + auto io_srv = uhd::transport::inline_io_service::make(); + link_params_t link_params; + if (link_type == link_type_t::RX_DATA) { + link_params.recv_frame_size = get_mtu(uhd::RX_DIRECTION); // FIXME + link_params.send_frame_size = LIBERIO_MAX_NONDATA_PACKET_SIZE; // FIXME + } else if (link_type == link_type_t::TX_DATA) { + link_params.recv_frame_size = LIBERIO_MAX_NONDATA_PACKET_SIZE; // FIXME + link_params.send_frame_size = get_mtu(uhd::TX_DIRECTION); // FIXME + } else { + link_params.recv_frame_size = LIBERIO_MAX_NONDATA_PACKET_SIZE; // FIXME + link_params.send_frame_size = LIBERIO_MAX_NONDATA_PACKET_SIZE; // FIXME + } + link_params.num_recv_frames = LIBERIO_NUM_RECV_FRAMES; // FIXME + link_params.num_send_frames = LIBERIO_NUM_SEND_FRAMES; // FIXME + + // Liberio doesn't need in-band flow control, so pretend have very large buffers + link_params.recv_buff_size = std::numeric_limits<size_t>::max(); + link_params.send_buff_size = std::numeric_limits<size_t>::max(); + auto link = uhd::transport::liberio_link::make( + link_info.first, link_info.second, link_params); + io_srv->attach_send_link(link); + io_srv->attach_recv_link(link); + return std::tuple<io_service::sptr, + send_link_if::sptr, + size_t, + recv_link_if::sptr, + size_t, + bool>(io_srv, + link, + link_params.send_buff_size, + link, + link_params.recv_buff_size, + false); +} + +size_t mpmd_link_if_ctrl_liberio::get_mtu(const uhd::direction_t /*dir*/) const +{ + return LIBERIO_DEFAULT_MTU; // FIXME +} diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.hpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.hpp new file mode 100644 index 000000000..09fb24f8e --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_liberio.hpp @@ -0,0 +1,64 @@ +// +// Copyright 2017 Ettus Research, a National Instruments Company +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_MPMD_XPORT_CTRL_LIBERIO_HPP +#define INCLUDED_MPMD_XPORT_CTRL_LIBERIO_HPP + +#include "mpmd_link_if_ctrl_base.hpp" +#include <uhd/types/device_addr.hpp> + +namespace uhd { namespace mpmd { namespace xport { + +/*! Liberio transport manager + */ +class mpmd_link_if_ctrl_liberio : public mpmd_link_if_ctrl_base +{ +public: + /* For liberio, get_chdr_link_options returns information about DMA engines. + * We assume there is only ever one liberio link available + * first = tx path + * second = rx path + */ + using liberio_link_info_t = std::pair<std::string, std::string>; + + mpmd_link_if_ctrl_liberio(const uhd::device_addr_t& mb_args, + const mpmd_link_if_mgr::xport_info_list_t& xport_info); + + size_t get_num_links() const { return 1; } + + uhd::transport::both_links_t get_link( + const size_t link_idx, const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args); + + size_t get_mtu(const uhd::direction_t) const; + + double get_link_rate(const size_t /*link_idx*/) const { return _link_rate; } + + const uhd::rfnoc::chdr::chdr_packet_factory& get_packet_factory() const + { + return _pkt_factory; + } + +private: + + const uhd::device_addr_t _mb_args; + const uhd::dict<std::string, std::string> _recv_args; + const uhd::dict<std::string, std::string> _send_args; + //! A list of DMA channels we can use for links + std::vector<liberio_link_info_t> _dma_channels; + double _link_rate; + + /*! An index representing the next DMA channel to use, for a simple + * allocation of channels. For get_link(), increment for each new link, and + * throw an exception if _next_channel > number of DMA channels. + */ + size_t _next_channel = 0; + static const uhd::rfnoc::chdr::chdr_packet_factory _pkt_factory; +}; + +}}} /* namespace uhd::mpmd::xport */ + +#endif /* INCLUDED_MPMD_XPORT_CTRL_LIBERIO_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp new file mode 100644 index 000000000..c2d746f92 --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.cpp @@ -0,0 +1,279 @@ +// +// Copyright 2017 Ettus Research, National Instruments Company +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include "mpmd_link_if_ctrl_udp.hpp" +#include "mpmd_impl.hpp" +#include "mpmd_link_if_mgr.hpp" +#include <uhd/transport/udp_constants.hpp> +#include <uhd/transport/udp_simple.hpp> +#include <uhd/transport/udp_zero_copy.hpp> +#include <uhdlib/transport/inline_io_service.hpp> +#include <uhdlib/transport/udp_boost_asio_link.hpp> +#include <uhdlib/utils/narrow.hpp> +#include <string> + +using namespace uhd; +using namespace uhd::transport; +using namespace uhd::mpmd::xport; + +const uhd::rfnoc::chdr::chdr_packet_factory mpmd_link_if_ctrl_udp::_pkt_factory( + uhd::rfnoc::CHDR_W_64, ENDIANNESS_BIG); + +namespace { + +//! Maximum CHDR packet size in bytes +const size_t MPMD_10GE_DATA_FRAME_MAX_SIZE = 8000; + +//! Maximum CHDR packet size in bytes +const size_t MPMD_10GE_ASYNCMSG_FRAME_MAX_SIZE = 1472; + +//! Number of send/recv frames +const size_t MPMD_ETH_NUM_FRAMES = 32; + +//! +const double MPMD_BUFFER_DEPTH = 20.0e-3; // s +//! For MTU discovery, the time we wait for a packet before calling it +// oversized (seconds). +const double MPMD_MTU_DISCOVERY_TIMEOUT = 0.02; + +// TODO: move these to appropriate header file for all other devices +const size_t MAX_RATE_1GIGE = 1e9 / 8; // byte/s +const size_t MAX_RATE_10GIGE = 10e9 / 8; // byte/s + + +mpmd_link_if_ctrl_udp::udp_link_info_map get_udp_info_from_xport_info( + const mpmd_link_if_mgr::xport_info_list_t& link_info_list) +{ + mpmd_link_if_ctrl_udp::udp_link_info_map result; + for (const auto& link_info : link_info_list) { + if (!link_info.count("ipv4")) { + UHD_LOG_ERROR("MPMD::XPORT::UDP", + "Invalid response from get_chdr_link_options()! No `ipv4' key!"); + throw uhd::runtime_error( + "Invalid response from get_chdr_link_options()! No `ipv4' key!"); + } + if (!link_info.count("port")) { + UHD_LOG_ERROR("MPMD::XPORT::UDP", + "Invalid response from get_chdr_link_options()! No `port' key!"); + throw uhd::runtime_error( + "Invalid response from get_chdr_link_options()! No `port' key!"); + } + const std::string udp_port = link_info.at("port"); + const size_t link_rate = link_info.count("link_rate") + ? std::stoul(link_info.at("link_rate")) + : MAX_RATE_1GIGE; + result.emplace(link_info.at("ipv4"), + mpmd_link_if_ctrl_udp::udp_link_info_t{udp_port, link_rate}); + } + + return result; +} + +std::vector<std::string> get_addrs_from_mb_args(const uhd::device_addr_t& mb_args, + const mpmd_link_if_ctrl_udp::udp_link_info_map& link_info_list) +{ + // mb_args must always include addr + if (not mb_args.has_key(FIRST_ADDR_KEY)) { + UHD_LOG_WARNING("MPMD::XPORT::UDP", + "The `" << FIRST_ADDR_KEY + << "' key must be specified in " + "device args to create an Ethernet transport to an RFNoC block"); + return {}; + } + std::vector<std::string> addrs{mb_args[FIRST_ADDR_KEY]}; + if (mb_args.has_key(SECOND_ADDR_KEY)) { + addrs.push_back(mb_args[SECOND_ADDR_KEY]); + } + // This is where in UHD we encode the knowledge about what + // get_chdr_link_options() returns to us. + for (const auto& ip_addr : addrs) { + if (link_info_list.count(ip_addr)) { + continue; + } + UHD_LOG_WARNING("MPMD::XPORT::UDP", + "Cannot create UDP link to device: The IP address `" + << ip_addr << "' is requested, but not reachable."); + return {}; + } + + return addrs; +} + +/*! Do a binary search to discover MTU + * + * Uses the MPM echo service to figure out MTU. We simply send a bunch of + * packets and see if they come back until we converged on the path MTU. + * The end result must lie between \p min_frame_size and \p max_frame_size. + * + * \param address IP address + * \param port UDP port (yeah it's a string!) + * \param min_frame_size Minimum frame size, initialize algorithm to start + * with this value + * \param max_frame_size Maximum frame size, initialize algorithm to start + * with this value + * \param echo_timeout Timeout value in seconds. For frame sizes that + * exceed the MTU, we don't expect a response, and this + * is the amount of time we'll wait before we assume + * the frame size exceeds the MTU. + */ +size_t discover_mtu(const std::string& address, + const std::string& port, + size_t min_frame_size, + size_t max_frame_size, + const double echo_timeout = 0.020) +{ + const size_t echo_prefix_offset = uhd::mpmd::mpmd_impl::MPM_ECHO_CMD.size(); + const size_t mtu_hdr_len = echo_prefix_offset + 10; + UHD_ASSERT_THROW(min_frame_size < max_frame_size); + UHD_ASSERT_THROW(min_frame_size % 4 == 0); + UHD_ASSERT_THROW(max_frame_size % 4 == 0); + UHD_ASSERT_THROW(min_frame_size >= echo_prefix_offset + mtu_hdr_len); + using namespace uhd::transport; + // The return port will probably differ from the discovery port, so we + // need a "broadcast" UDP connection; using make_connected() would + // drop packets + udp_simple::sptr udp = udp_simple::make_broadcast(address, port); + std::string send_buf(uhd::mpmd::mpmd_impl::MPM_ECHO_CMD); + send_buf.resize(max_frame_size, '#'); + UHD_ASSERT_THROW(send_buf.size() == max_frame_size); + std::vector<uint8_t> recv_buf; + recv_buf.resize(max_frame_size, ' '); + + // Little helper to check returned packets match the sent ones + auto require_bufs_match = [&recv_buf, &send_buf, mtu_hdr_len](const size_t len) { + if (len < mtu_hdr_len + or std::memcmp((void*)&recv_buf[0], (void*)&send_buf[0], mtu_hdr_len) != 0) { + throw uhd::runtime_error("Unexpected content of MTU " + "discovery return packet!"); + } + }; + UHD_LOG_TRACE("MPMD", "Determining UDP MTU... "); + size_t seq_no = 0; + while (min_frame_size < max_frame_size) { + // Only test multiples of 4 bytes! + const size_t test_frame_size = (max_frame_size / 2 + min_frame_size / 2 + 3) + & ~size_t(3); + // Encode sequence number and current size in the string, makes it + // easy to debug in code or Wireshark. Is also used for identifying + // response packets. + std::sprintf( + &send_buf[echo_prefix_offset], ";%04lu,%04lu", seq_no++, test_frame_size); + UHD_LOG_TRACE("MPMD", "Testing frame size " << test_frame_size); + udp->send(boost::asio::buffer(&send_buf[0], test_frame_size)); + + const size_t len = udp->recv(boost::asio::buffer(recv_buf), echo_timeout); + if (len == 0) { + // Nothing received, so this is probably too big + max_frame_size = test_frame_size - 4; + } else if (len >= test_frame_size) { + // Size went through, so bump the minimum + require_bufs_match(len); + min_frame_size = test_frame_size; + } else if (len < test_frame_size) { + // This is an odd case. Something must have snipped the packet + // on the way back. Still, we'll just back off and try + // something smaller. + UHD_LOG_DEBUG("MPMD", "Unexpected packet truncation during MTU discovery."); + require_bufs_match(len); + max_frame_size = len; + } + } + UHD_LOG_DEBUG("MPMD", "Path MTU for address " << address << ": " << min_frame_size); + return min_frame_size; +} + +} // namespace + + +/****************************************************************************** + * Structors + *****************************************************************************/ +mpmd_link_if_ctrl_udp::mpmd_link_if_ctrl_udp(const uhd::device_addr_t& mb_args, + const mpmd_link_if_mgr::xport_info_list_t& xport_info) + : _mb_args(mb_args) + , _recv_args(filter_args(mb_args, "recv")) + , _send_args(filter_args(mb_args, "send")) + , _udp_info(get_udp_info_from_xport_info(xport_info)) + , _mtu(MPMD_10GE_DATA_FRAME_MAX_SIZE) +{ + const std::string mpm_discovery_port = _mb_args.get( + mpmd_impl::MPM_DISCOVERY_PORT_KEY, std::to_string(mpmd_impl::MPM_DISCOVERY_PORT)); + auto discover_mtu_for_ip = [mpm_discovery_port](const std::string& ip_addr) { + return discover_mtu(ip_addr, + mpm_discovery_port, + IP_PROTOCOL_MIN_MTU_SIZE - IP_PROTOCOL_UDP_PLUS_IP_HEADER, + MPMD_10GE_DATA_FRAME_MAX_SIZE, + MPMD_MTU_DISCOVERY_TIMEOUT); + }; + + const std::vector<std::string> requested_addrs( + get_addrs_from_mb_args(mb_args, _udp_info)); + for (const auto& ip_addr : requested_addrs) { + try { + // If MTU discovery fails, we gracefully recover, but declare that + // link invalid. + _mtu = std::min(_mtu, discover_mtu_for_ip(ip_addr)); + _available_addrs.push_back(ip_addr); + } catch (const uhd::exception& ex) { + UHD_LOG_WARNING("MPMD::XPORT::UDP", + "Error during MTU discovery on address " << ip_addr << ": " << ex.what()); + } + } +} + +/****************************************************************************** + * API + *****************************************************************************/ +uhd::transport::both_links_t mpmd_link_if_ctrl_udp::get_link(const size_t link_idx, + const uhd::transport::link_type_t /*link_type*/, + const uhd::device_addr_t& /*link_args*/) +{ + UHD_ASSERT_THROW(link_idx < _available_addrs.size()); + const std::string ip_addr = _available_addrs.at(link_idx); + const std::string udp_port = _udp_info.at(ip_addr).udp_port; + + /* FIXME: Should have common infrastructure for creating I/O services */ + auto io_srv = uhd::transport::inline_io_service::make(); + link_params_t link_params; + link_params.num_recv_frames = MPMD_ETH_NUM_FRAMES; // FIXME + link_params.num_send_frames = MPMD_ETH_NUM_FRAMES; // FIXME + link_params.recv_frame_size = get_mtu(uhd::RX_DIRECTION); // FIXME + link_params.send_frame_size = get_mtu(uhd::TX_DIRECTION); // FIXME + link_params.recv_buff_size = MPMD_BUFFER_DEPTH * MAX_RATE_10GIGE; // FIXME + link_params.send_buff_size = MPMD_BUFFER_DEPTH * MAX_RATE_10GIGE; // FIXME + auto link = uhd::transport::udp_boost_asio_link::make(ip_addr, + udp_port, + link_params, + link_params.recv_buff_size, // FIXME + link_params.send_buff_size); // FIXME + io_srv->attach_send_link(link); + io_srv->attach_recv_link(link); + return std::tuple<io_service::sptr, + send_link_if::sptr, + size_t, + recv_link_if::sptr, + size_t, + bool>( + io_srv, link, link_params.send_buff_size, link, link_params.recv_buff_size, true); +} + +size_t mpmd_link_if_ctrl_udp::get_num_links() const +{ + return _available_addrs.size(); +} + +//! Return the rate of the underlying link in bytes/sec +double mpmd_link_if_ctrl_udp::get_link_rate(const size_t link_idx) const +{ + UHD_ASSERT_THROW(link_idx < get_num_links()); + return _udp_info.at(_available_addrs.at(link_idx)).link_rate; +} + +const uhd::rfnoc::chdr::chdr_packet_factory& +mpmd_link_if_ctrl_udp::get_packet_factory() const +{ + return _pkt_factory; +} diff --git a/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.hpp b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.hpp new file mode 100644 index 000000000..4c8ecade7 --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_ctrl_udp.hpp @@ -0,0 +1,61 @@ +// +// Copyright 2017 Ettus Research, a National Instruments Company +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_MPMD_XPORT_CTRL_UDP_HPP +#define INCLUDED_MPMD_XPORT_CTRL_UDP_HPP + +#include "mpmd_link_if_ctrl_base.hpp" +#include "mpmd_link_if_mgr.hpp" +#include <uhd/types/device_addr.hpp> +#include <unordered_map> + +namespace uhd { namespace mpmd { namespace xport { + +/*! UDP link interface controller + * + * Opens UDP sockets + */ +class mpmd_link_if_ctrl_udp : public mpmd_link_if_ctrl_base +{ +public: + struct udp_link_info_t + { + std::string udp_port; + size_t link_rate; + }; + + using udp_link_info_map = std::unordered_map<std::string, udp_link_info_t>; + + mpmd_link_if_ctrl_udp(const uhd::device_addr_t& mb_args, + const mpmd_link_if_mgr::xport_info_list_t& xport_info); + + size_t get_num_links() const; + uhd::transport::both_links_t get_link(const size_t link_idx, + const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args); + size_t get_mtu(const uhd::direction_t) const + { + return _mtu; + } + double get_link_rate(const size_t link_idx) const; + const uhd::rfnoc::chdr::chdr_packet_factory& get_packet_factory() const; + +private: + const uhd::device_addr_t _mb_args; + const uhd::dict<std::string, std::string> _recv_args; + const uhd::dict<std::string, std::string> _send_args; + //! + udp_link_info_map _udp_info; + //! A list of IP addresses we can connect our CHDR connections to + std::vector<std::string> _available_addrs; + //! MTU + size_t _mtu; + static const uhd::rfnoc::chdr::chdr_packet_factory _pkt_factory; +}; + +}}} /* namespace uhd::mpmd::xport */ + +#endif /* INCLUDED_MPMD_XPORT_CTRL_UDP_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_link_if_mgr.cpp b/host/lib/usrp/mpmd/mpmd_link_if_mgr.cpp new file mode 100644 index 000000000..6bb6cae3a --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_link_if_mgr.cpp @@ -0,0 +1,135 @@ +// +// Copyright 2017 Ettus Research, National Instruments Company +// Copyright 2019 Ettus Research, National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include "mpmd_link_if_mgr.hpp" +#include "mpmd_impl.hpp" +#include "mpmd_link_if_ctrl_base.hpp" +#include "mpmd_link_if_ctrl_udp.hpp" +#ifdef HAVE_LIBERIO +# include "mpmd_link_if_ctrl_liberio.hpp" +#endif +#ifdef HAVE_DPDK +# include "mpmd_link_if_ctrl_dpdk_udp.hpp" +#endif + +uhd::dict<std::string, std::string> uhd::mpmd::xport::filter_args( + const uhd::device_addr_t& args, const std::string& prefix) +{ + uhd::dict<std::string, std::string> filtered_args; + for (const std::string& key : args.keys()) { + if (key.find(prefix) != std::string::npos) { + filtered_args[key] = args[key]; + } + } + + return filtered_args; +} + +using namespace uhd::mpmd::xport; + +class mpmd_link_if_mgr_impl : public mpmd_link_if_mgr +{ +public: + mpmd_link_if_mgr_impl(const uhd::device_addr_t& mb_args) : _mb_args(mb_args) {} + + /************************************************************************** + * API (see mpmd_link_if_mgr.hpp) + *************************************************************************/ + bool connect(const std::string& link_type, const xport_info_list_t& xport_info) + { + auto link_if_ctrl = make_link_if_ctrl(link_type, xport_info); + if (!link_if_ctrl) { + UHD_LOG_WARNING( + "MPMD::XPORT", "Unable to create xport ctrl for link type " << link_type); + return false; + } + if (link_if_ctrl->get_num_links() == 0) { + UHD_LOG_TRACE("MPMD::XPORT", + "Link type " << link_type + << " has no valid links in this configuration."); + return false; + } + const size_t xport_idx = _link_if_ctrls.size(); + for (size_t link_idx = 0; link_idx < link_if_ctrl->get_num_links(); link_idx++) { + _link_link_if_ctrl_map.push_back(std::make_pair(xport_idx, link_idx)); + } + _link_if_ctrls.push_back(std::move(link_if_ctrl)); + return true; + } + + size_t get_num_links() + { + return _link_link_if_ctrl_map.size(); + } + + uhd::transport::both_links_t get_link(const size_t link_idx, + const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args) + { + const size_t link_if_ctrl_idx = _link_link_if_ctrl_map.at(link_idx).first; + const size_t xport_link_idx = _link_link_if_ctrl_map.at(link_idx).second; + return _link_if_ctrls.at(link_if_ctrl_idx) + ->get_link(xport_link_idx, link_type, link_args); + } + + size_t get_mtu(const size_t link_idx, const uhd::direction_t dir) const + { + return _link_if_ctrls.at(_link_link_if_ctrl_map.at(link_idx).first)->get_mtu(dir); + } + + const uhd::rfnoc::chdr::chdr_packet_factory& get_packet_factory( + const size_t link_idx) const + { + const size_t link_if_ctrl_idx = _link_link_if_ctrl_map.at(link_idx).first; + return _link_if_ctrls.at(link_if_ctrl_idx)->get_packet_factory(); + } + +private: + /************************************************************************** + * Private methods / helpers + *************************************************************************/ + mpmd_link_if_ctrl_base::uptr make_link_if_ctrl( + const std::string& link_type, const xport_info_list_t& xport_info) + { + // Here, we hard-code the list of available transport types + if (link_type == "udp") { +#ifdef HAVE_DPDK + // if (_mb_args.has_key("use_dpdk")) { + // return std::make_unique<mpmd_link_if_ctrl_dpdk_udp>(_mb_args, + // xport_info); + //} +#endif + return std::make_unique<mpmd_link_if_ctrl_udp>(_mb_args, xport_info); +#ifdef HAVE_LIBERIO + } else if (link_type == "liberio") { + return std::make_unique<mpmd_link_if_ctrl_liberio>(_mb_args, xport_info); +#endif + } + UHD_LOG_WARNING("MPMD", "Cannot instantiate transport medium " << link_type); + return nullptr; + } + + /************************************************************************** + * Private attributes + *************************************************************************/ + //! Cache available xport manager implementations + // + // Should only every be populated by connect() + std::vector<mpmd_link_if_ctrl_base::uptr> _link_if_ctrls; + // Maps link index to link_if_ctrl index. To look up the xport ctrl for link + // number L, do something like this: + // auto& link_if_ctrl = _link_if_ctrls.at(_link_link_if_ctrl_map.at(L).first); + std::vector<std::pair<size_t, size_t>> _link_link_if_ctrl_map; + + //! Motherboard args, can contain things like 'recv_buff_size' + const uhd::device_addr_t _mb_args; +}; + +mpmd_link_if_mgr::uptr mpmd_link_if_mgr::make(const uhd::device_addr_t& mb_args) +{ + return std::make_unique<mpmd_link_if_mgr_impl>(mb_args); +} diff --git a/host/lib/usrp/mpmd/mpmd_xport_mgr.hpp b/host/lib/usrp/mpmd/mpmd_link_if_mgr.hpp index 3d96e5ec6..4b0ba4212 100644 --- a/host/lib/usrp/mpmd/mpmd_xport_mgr.hpp +++ b/host/lib/usrp/mpmd/mpmd_link_if_mgr.hpp @@ -4,11 +4,14 @@ // SPDX-License-Identifier: GPL-3.0-or-later // -#ifndef INCLUDED_MPMD_XPORT_MGR_HPP -#define INCLUDED_MPMD_XPORT_MGR_HPP +#ifndef INCLUDED_MPMD_LINK_IF_MGR_HPP +#define INCLUDED_MPMD_LINK_IF_MGR_HPP -#include "../device3/device3_impl.hpp" +#include <uhd/types/device_addr.hpp> #include <uhd/types/dict.hpp> +#include <uhd/types/direction.hpp> +#include <uhdlib/rfnoc/chdr_packet.hpp> +#include <uhdlib/transport/links.hpp> #include <map> #include <memory> #include <string> @@ -20,8 +23,6 @@ namespace uhd { namespace mpmd { namespace xport { * Transport specifiers */ -//! Ethernet address for management and RPC communication -const std::string MGMT_ADDR_KEY = "mgmt_addr"; //! Primary Ethernet address for streaming and RFNoC communication const std::string FIRST_ADDR_KEY = "addr"; //! Secondary Ethernet address for streaming and RFNoC communication @@ -45,13 +46,13 @@ uhd::dict<std::string, std::string> filter_args( * medium. For example, if the medium is Ethernet/UDP, this class will create * sockets. */ -class mpmd_xport_mgr +class mpmd_link_if_mgr { public: - using uptr = std::unique_ptr<mpmd_xport_mgr>; + using uptr = std::unique_ptr<mpmd_link_if_mgr>; using xport_info_t = std::map<std::string, std::string>; using xport_info_list_t = std::vector<std::map<std::string, std::string>>; - virtual ~mpmd_xport_mgr() {} + virtual ~mpmd_link_if_mgr() {} /*! Return a reference to a transport manager * @@ -65,6 +66,45 @@ public: */ static uptr make(const uhd::device_addr_t& mb_args); + /*! Attempt to open a CHDR-capable link to the remote device + * + * This will compare the mb_args (passed in at construction) with + * \p xport_info to see if it can connect this way. For example, if + * \p xport_type is "udp", then it will see if it can find the `addr` key + * from mb_args in the \p xport_info. If yes, it will use that for + * connections. + * + * \param xport_type The type of xport ("udp", "liberio", ...) + * \param xport_info The available information on this transport. For + * example, if the xport_type is "udp", then this would + * contain the available IP addresses. + * \returns true on success + */ + virtual bool connect( + const std::string& xport_type, const xport_info_list_t& xport_info) = 0; + + /*! The number of available links + * + * If zero, it means that there is no valid connection to the device. + * + */ + virtual size_t get_num_links() = 0; + + /*! Return links object + * + * \param link_idx The number of the link to use. link_idx < get_num_links() + * must hold true. link_idx is often 0. Example: When + * the underlying transport is Ethernet, and the user + * specified both addr and second_addr, then get_num_links() + * equals 2 and link_idx can also be 1. + * \param link_type CTRL, RX_DATA, or TX_DATA (for configuring the link) + * \param link_args Link-specific additional information that the underlying + * mpmd_link_if_ctrl instantiation can use + */ + virtual uhd::transport::both_links_t get_link(const size_t link_idx, + const uhd::transport::link_type_t link_type, + const uhd::device_addr_t& link_args) = 0; + /*! Create a transports object * * Implementation details depend on the underlying implementation. @@ -90,16 +130,28 @@ public: * The latter needs to get sent back to MPM to complete the * transport handshake. */ - virtual both_xports_t make_transport(const xport_info_list_t& xport_info_list, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args, - xport_info_t& xport_info_out) = 0; + //virtual both_xports_t make_transport(const xport_info_list_t& xport_info_list, + //const usrp::device3_impl::xport_type_t xport_type, + //const uhd::device_addr_t& xport_args, + //xport_info_t& xport_info_out) = 0; /*! Return the path MTU for whatever this manager lets us do */ - virtual size_t get_mtu(const uhd::direction_t dir) const = 0; + virtual size_t get_mtu(const size_t link_idx, const uhd::direction_t dir) const = 0; + + /*! Get packet factory from associated link_mgr + * + * \param link_idx The number of the link to use. link_idx < get_num_links() + * must hold true. link_idx is often 0. Example: When + * the underlying transport is Ethernet, and the user + * specified both addr and second_addr, then get_num_links() + * equals 2 and link_idx can also be 1. + * \return a CHDR packet factory + */ + virtual const uhd::rfnoc::chdr::chdr_packet_factory& get_packet_factory( + const size_t link_idx) const = 0; }; }}} /* namespace uhd::mpmd::xport */ -#endif /* INCLUDED_MPMD_XPORT_MGR_HPP */ +#endif /* INCLUDED_MPMD_LINK_IF_MGR_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_mb_controller.cpp b/host/lib/usrp/mpmd/mpmd_mb_controller.cpp index 6c2954fb8..e9310d01d 100644 --- a/host/lib/usrp/mpmd/mpmd_mb_controller.cpp +++ b/host/lib/usrp/mpmd/mpmd_mb_controller.cpp @@ -4,11 +4,34 @@ // SPDX-License-Identifier: GPL-3.0-or-later // -#include "mpmd_mb_controller.hpp" +#include <uhdlib/usrp/common/mpmd_mb_controller.hpp> using namespace uhd::rfnoc; +using namespace uhd; +namespace { + //! Default timeout value for tRPC calls that we know can take long (ms) + constexpr size_t MPMD_DEFAULT_LONG_TIMEOUT = 12000; // ms +} + +mpmd_mb_controller::mpmd_mb_controller( + uhd::rpc_client::sptr rpcc, uhd::device_addr_t device_info) + : _rpc(rpcc), _device_info(device_info) +{ + const size_t num_tks = _rpc->request_with_token<size_t>("get_num_timekeepers"); + for (size_t tk_idx = 0; tk_idx < num_tks; tk_idx++) { + register_timekeeper(tk_idx, std::make_shared<mpmd_timekeeper>(tk_idx, _rpc)); + } + + auto sensor_list = + _rpc->request_with_token<std::vector<std::string>>("get_mb_sensors"); + UHD_LOG_DEBUG("MPMD", "Found " << sensor_list.size() << " motherboard sensors."); + _sensor_names.insert(sensor_list.cbegin(), sensor_list.cend()); +} +/****************************************************************************** + * Timekeeper API + *****************************************************************************/ uint64_t mpmd_mb_controller::mpmd_timekeeper::get_ticks_now() { return _rpc->request_with_token<uint64_t>("get_timekeeper_time", _tk_idx, false); @@ -34,3 +57,120 @@ void mpmd_mb_controller::mpmd_timekeeper::set_period(const uint64_t period_ns) _rpc->notify_with_token("set_tick_period", _tk_idx, period_ns); } +void mpmd_mb_controller::mpmd_timekeeper::update_tick_rate(const double tick_rate) +{ + set_tick_rate(tick_rate); +} + +/****************************************************************************** + * Motherboard Control API (see mb_controller.hpp) + *****************************************************************************/ +std::string mpmd_mb_controller::get_mboard_name() const +{ + return _device_info.get("name", "UNKNOWN"); +} + +void mpmd_mb_controller::set_time_source(const std::string& source) +{ + _rpc->notify_with_token(MPMD_DEFAULT_LONG_TIMEOUT, "set_time_source", source); +} + +std::string mpmd_mb_controller::get_time_source() const +{ + return _rpc->request_with_token<std::string>("get_time_source"); +} + +std::vector<std::string> mpmd_mb_controller::get_time_sources() const +{ + return _rpc->request_with_token<std::vector<std::string>>("get_time_sources"); +} + +void mpmd_mb_controller::set_clock_source(const std::string& source) +{ + _rpc->notify_with_token(MPMD_DEFAULT_LONG_TIMEOUT, "set_clock_source", source); +} + +std::string mpmd_mb_controller::get_clock_source() const +{ + return _rpc->request_with_token<std::string>("get_clock_source"); +} + +std::vector<std::string> mpmd_mb_controller::get_clock_sources() const +{ + return _rpc->request_with_token<std::vector<std::string>>("get_clock_sources"); +} + +void mpmd_mb_controller::set_sync_source(const std::string& clock_source, const std::string& time_source) +{ + uhd::device_addr_t sync_source; + sync_source["clock_source"] = clock_source; + sync_source["time_source"] = time_source; + set_sync_source(sync_source); +} + +void mpmd_mb_controller::set_sync_source(const device_addr_t& sync_source) +{ + std::map<std::string, std::string> sync_source_map; + for (const auto& key : sync_source.keys()) { + sync_source_map[key] = sync_source.get(key); + } + _rpc->notify_with_token( + MPMD_DEFAULT_LONG_TIMEOUT, "set_clock_source", sync_source_map); +} + +device_addr_t mpmd_mb_controller::get_sync_source() const +{ + const auto sync_source_map = + _rpc->request_with_token<std::map<std::string, std::string>>("get_sync_source"); + return device_addr_t(sync_source_map); +} + +std::vector<device_addr_t> mpmd_mb_controller::get_sync_sources() +{ + std::vector<device_addr_t> result; + const auto sync_sources = + _rpc->request_with_token<std::vector<std::map<std::string, std::string>>>( + "get_sync_sources"); + for (auto& sync_source : sync_sources) { + result.push_back(device_addr_t(sync_source)); + } + + return result; +} + +void mpmd_mb_controller::set_clock_source_out(const bool /*enb*/) +{ + throw uhd::not_implemented_error( + "set_clock_source_out() not implemented on this device!"); +} + +void mpmd_mb_controller::set_time_source_out(const bool /*enb*/) +{ + throw uhd::not_implemented_error( + "set_time_source_out() not implemented on this device!"); +} + +sensor_value_t mpmd_mb_controller::get_sensor(const std::string& name) +{ + if (!_sensor_names.count(name)) { + throw uhd::key_error(std::string("Invalid motherboard sensor name: ") + name); + } + return sensor_value_t( + _rpc->request_with_token<sensor_value_t::sensor_map_t>("get_mb_sensor", name)); +} + +std::vector<std::string> mpmd_mb_controller::get_sensor_names() +{ + std::vector<std::string> sensor_names(_sensor_names.cbegin(), _sensor_names.cend()); + return sensor_names; +} + +uhd::usrp::mboard_eeprom_t mpmd_mb_controller::get_eeprom() +{ + auto mb_eeprom = + _rpc->request_with_token<std::map<std::string, std::string>>("get_mb_eeprom"); + uhd::usrp::mboard_eeprom_t mb_eeprom_dict( + mb_eeprom.cbegin(), mb_eeprom.cend()); + return mb_eeprom_dict; +} + diff --git a/host/lib/usrp/mpmd/mpmd_mb_controller.hpp b/host/lib/usrp/mpmd/mpmd_mb_controller.hpp deleted file mode 100644 index 65e5dc468..000000000 --- a/host/lib/usrp/mpmd/mpmd_mb_controller.hpp +++ /dev/null @@ -1,73 +0,0 @@ -// -// Copyright 2019 Ettus Research, a National Instruments Brand -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#ifndef INCLUDED_LIBUHD_MPMD_MB_CONTROLLER_HPP -#define INCLUDED_LIBUHD_MPMD_MB_CONTROLLER_HPP - -#include <uhd/rfnoc/mb_controller.hpp> -#include <uhdlib/utils/rpc.hpp> - -namespace uhd { namespace rfnoc { - -/*! X300-Specific version of the mb_controller - * - * Reminder: There is one of these per motherboard. - */ -class mpmd_mb_controller : public mb_controller -{ -public: - - - //! Return reference to the RPC client - uhd::rpc_client::sptr get_rpc_client() { return _rpc; } - - //! X300-specific version of the timekeeper controls - class mpmd_timekeeper : public mb_controller::timekeeper - { - public: - mpmd_timekeeper(const size_t tk_idx, uhd::rpc_client::sptr rpc_client) - : _tk_idx(tk_idx), _rpc(rpc_client) - { - // nop - } - - uint64_t get_ticks_now(); - - uint64_t get_ticks_last_pps(); - - void set_ticks_now(const uint64_t ticks); - - void set_ticks_next_pps(const uint64_t ticks); - - void set_period(const uint64_t period_ns); - - private: - /*! Shorthand to perform an RPC request. Saves some typing. - */ - template <typename return_type, typename... Args> - return_type request(std::string const& func_name, Args&&... args) - { - UHD_LOG_TRACE("X300MBCTRL", "[RPC] Calling " << func_name); - return _rpc->request_with_token<return_type>( - func_name, std::forward<Args>(args)...); - }; - - const size_t _tk_idx; - - uhd::rpc_client::sptr _rpc; - }; - -private: - /************************************************************************** - * Attributes - *************************************************************************/ - //! Reference to RPC interface - uhd::rpc_client::sptr _rpc; -}; - -}} // namespace uhd::rfnoc - -#endif /* INCLUDED_LIBUHD_MPMD_MB_CONTROLLER_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_mb_iface.cpp b/host/lib/usrp/mpmd/mpmd_mb_iface.cpp new file mode 100644 index 000000000..e713cc7a3 --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_mb_iface.cpp @@ -0,0 +1,301 @@ +// +// Copyright 2018 Ettus Research, a National Instruments Company +// Copyright 2019 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include "mpmd_mb_iface.hpp" +#include "mpmd_link_if_mgr.hpp" +#include <uhd/exception.hpp> +#include <uhd/utils/log.hpp> +#include <uhdlib/rfnoc/device_id.hpp> + +using namespace uhd::rfnoc; +using namespace uhd::mpmd; + +mpmd_mboard_impl::mpmd_mb_iface::mpmd_mb_iface( + const uhd::device_addr_t& mb_args, uhd::rpc_client::sptr rpc) + : _mb_args(mb_args), _rpc(rpc), _link_if_mgr(xport::mpmd_link_if_mgr::make(mb_args)) +{ + _remote_device_id = allocate_device_id(); + UHD_LOG_TRACE("MPMD::MB_IFACE", "Assigning device_id " << _remote_device_id); + _rpc->notify_with_token("set_device_id", _remote_device_id); +} + +/****************************************************************************** + * mpmd_mb_iface API + *****************************************************************************/ +void mpmd_mboard_impl::mpmd_mb_iface::init() +{ + UHD_LOG_TRACE("MPMD::MB_IFACE", "Requesting clock ifaces..."); + auto clock_ifaces = _rpc->request_with_token<clock_iface_list_t>("get_clocks"); + for (auto& clock : clock_ifaces) { + auto iface = std::make_shared<uhd::rfnoc::clock_iface>( + clock.at("name"), std::stod(clock.at("freq")), clock.count("mutable")); + _clock_ifaces[clock.at("name")] = iface; + iface->set_running(true); + UHD_LOG_DEBUG("MPMD::MB_IFACE", + "Adding clock iface `" + << clock.at("name") << "`, frequency: " << (iface->get_freq() / 1e6) + << " MHz, mutable: " << (iface->is_mutable() ? "Yes" : "No")); + } + UHD_LOG_TRACE("MPMD::MB_IFACE", "Requesting CHDR link types..."); + auto chdr_link_types = + _rpc->request_with_token<std::vector<std::string>>("get_chdr_link_types"); + UHD_LOG_TRACE( + "MPMD::MB_IFACE", "Found " << chdr_link_types.size() << " link type(s)"); + for (const auto& type : chdr_link_types) { + UHD_LOG_TRACE("MPMD::MB_IFACE", "Trying link type `" << type << "'"); + const auto xport_info = + _rpc->request_with_token<xport::mpmd_link_if_mgr::xport_info_list_t>( + "get_chdr_link_options", type); + // User may have specified: addr=192.168.10.2, second_addr= + // MPM may have said: "my addresses are 192.168.10.2 and 192.168.20.2" + if (_link_if_mgr->connect(type, xport_info)) { + UHD_LOG_TRACE("MPMD::MB_IFACE", "Link type " << type << " successful."); + } + } + + if (_link_if_mgr->get_num_links() == 0) { + UHD_LOG_ERROR("MPMD::MB_IFACE", "No CHDR connection available!"); + throw uhd::runtime_error("No CHDR connection available!"); + } + + for (size_t link_idx = 0; link_idx < _link_if_mgr->get_num_links(); link_idx++) { + _local_device_id_map.emplace(allocate_device_id(), link_idx); + } +} + +/****************************************************************************** + * mb_iface API + *****************************************************************************/ +uint16_t mpmd_mboard_impl::mpmd_mb_iface::get_proto_ver() +{ + return _rpc->request_with_token<uint16_t>("get_proto_ver"); +} + +uhd::rfnoc::chdr_w_t mpmd_mboard_impl::mpmd_mb_iface::get_chdr_w() +{ + const auto chdr_w_bits = _rpc->request_with_token<size_t>("get_chdr_width"); + switch (chdr_w_bits) { + case 512: + return CHDR_W_512; + case 256: + return CHDR_W_256; + case 128: + return CHDR_W_128; + case 64: + return CHDR_W_64; + } + throw uhd::runtime_error(std::string("Device reporting invalid CHDR width: ") + + std::to_string(chdr_w_bits)); +} + +uhd::endianness_t mpmd_mboard_impl::mpmd_mb_iface::get_endianness( + const uhd::rfnoc::device_id_t local_device_id) +{ + uhd::rfnoc::device_id_t lookup_id = local_device_id; + if (lookup_id == NULL_DEVICE_ID) { + for (auto& ids : _local_device_id_map) { + lookup_id = ids.first; + break; + } + } + const size_t link_idx = _local_device_id_map.at(lookup_id); + auto& pkt_factory = _link_if_mgr->get_packet_factory(link_idx); + return pkt_factory.get_endianness(); +} + +uhd::rfnoc::device_id_t mpmd_mboard_impl::mpmd_mb_iface::get_remote_device_id() +{ + return _remote_device_id; +} + +std::vector<device_id_t> mpmd_mboard_impl::mpmd_mb_iface::get_local_device_ids() +{ + std::vector<device_id_t> device_ids; + for (auto& local_dev_id_pair : _local_device_id_map) { + device_ids.push_back(local_dev_id_pair.first); + } + return device_ids; +} + +uhd::transport::adapter_id_t mpmd_mboard_impl::mpmd_mb_iface::get_adapter_id( + const uhd::rfnoc::device_id_t local_device_id) +{ + return _adapter_map.at(local_device_id); +} + +void mpmd_mboard_impl::mpmd_mb_iface::reset_network() +{ + // FIXME +} + +uhd::rfnoc::clock_iface::sptr mpmd_mboard_impl::mpmd_mb_iface::get_clock_iface( + const std::string& clock_name) +{ + if (_clock_ifaces.count(clock_name)) { + return _clock_ifaces.at(clock_name); + } else { + UHD_LOG_ERROR("MPMD::MB_IFACE", "Invalid timebase clock name: " + clock_name); + throw uhd::key_error( + "[MPMD_MB::IFACE] Invalid timebase clock name: " + clock_name); + } +} + +uhd::rfnoc::chdr_ctrl_xport::sptr mpmd_mboard_impl::mpmd_mb_iface::make_ctrl_transport( + uhd::rfnoc::device_id_t local_device_id, const uhd::rfnoc::sep_id_t& local_epid) +{ + if (!_local_device_id_map.count(local_device_id)) { + throw uhd::key_error(std::string("[MPMD::MB_IFACE] Cannot create control " + "transport: Unknown local device ID ") + + std::to_string(local_device_id)); + } + const size_t link_idx = _local_device_id_map.at(local_device_id); + uhd::transport::io_service::sptr io_srv; + uhd::transport::send_link_if::sptr send_link; + uhd::transport::recv_link_if::sptr recv_link; + std::tie(io_srv, send_link, std::ignore, recv_link, std::ignore, std::ignore) = + _link_if_mgr->get_link( + link_idx, uhd::transport::link_type_t::CTRL, uhd::device_addr_t()); + + /* Associate local device ID with the adapter */ + _adapter_map[local_device_id] = send_link->get_send_adapter_id(); + + auto pkt_factory = _link_if_mgr->get_packet_factory(link_idx); + auto xport = uhd::rfnoc::chdr_ctrl_xport::make(io_srv, + send_link, + recv_link, + pkt_factory, + local_epid, + send_link->get_num_send_frames(), + recv_link->get_num_recv_frames()); + return xport; +} + +uhd::rfnoc::chdr_rx_data_xport::uptr +mpmd_mboard_impl::mpmd_mb_iface::make_rx_data_transport( + uhd::rfnoc::mgmt::mgmt_portal& mgmt_portal, + const uhd::rfnoc::sep_addr_pair_t& addrs, + const uhd::rfnoc::sep_id_pair_t& epids, + const uhd::rfnoc::sw_buff_t pyld_buff_fmt, + const uhd::rfnoc::sw_buff_t mdata_buff_fmt, + const uhd::device_addr_t& xport_args) +{ + const uhd::rfnoc::sep_addr_t local_sep_addr = addrs.second; + + if (!_local_device_id_map.count(local_sep_addr.first)) { + throw uhd::key_error(std::string("[MPMD::MB_IFACE] Cannot create RX data " + "transport: Unknown local device ID ") + + std::to_string(local_sep_addr.first)); + } + const size_t link_idx = _local_device_id_map.at(local_sep_addr.first); + + uhd::transport::io_service::sptr io_srv; + uhd::transport::send_link_if::sptr send_link; + uhd::transport::recv_link_if::sptr recv_link; + bool lossy_xport; + size_t recv_buff_size; + std::tie(io_srv, send_link, std::ignore, recv_link, recv_buff_size, lossy_xport) = + _link_if_mgr->get_link( + link_idx, uhd::transport::link_type_t::RX_DATA, xport_args); + + /* Associate local device ID with the adapter */ + _adapter_map[local_sep_addr.first] = send_link->get_send_adapter_id(); + + // TODO: configure this based on the transport type + const stream_buff_params_t recv_capacity = { + recv_buff_size, uhd::rfnoc::MAX_FC_CAPACITY_PKTS}; + + const double ratio = 1.0 / 32; + + // Configure flow control frequency to use bytes only for UDP + stream_buff_params_t fc_freq = { + static_cast<uint64_t>(std::ceil(double(recv_buff_size) * ratio)), + uhd::rfnoc::MAX_FC_FREQ_PKTS}; + + stream_buff_params_t fc_headroom = {0, 0}; + + // Create the data transport + auto pkt_factory = _link_if_mgr->get_packet_factory(link_idx); + auto fc_params = chdr_rx_data_xport::configure_sep(io_srv, + recv_link, + send_link, + pkt_factory, + mgmt_portal, + epids, + pyld_buff_fmt, + mdata_buff_fmt, + recv_capacity, + fc_freq, + fc_headroom, + lossy_xport); + auto rx_xport = std::make_unique<chdr_rx_data_xport>(io_srv, + recv_link, + send_link, + pkt_factory, + epids, + recv_link->get_num_recv_frames(), + fc_params); + + return rx_xport; +} + +uhd::rfnoc::chdr_tx_data_xport::uptr +mpmd_mboard_impl::mpmd_mb_iface::make_tx_data_transport( + uhd::rfnoc::mgmt::mgmt_portal& mgmt_portal, + const uhd::rfnoc::sep_addr_pair_t& addrs, + const uhd::rfnoc::sep_id_pair_t& epids, + const uhd::rfnoc::sw_buff_t pyld_buff_fmt, + const uhd::rfnoc::sw_buff_t mdata_buff_fmt, + const uhd::device_addr_t& xport_args) +{ + const uhd::rfnoc::sep_addr_t local_sep_addr = addrs.first; + + if (!_local_device_id_map.count(local_sep_addr.first)) { + throw uhd::key_error(std::string("[MPMD::MB_IFACE] Cannot create TX data " + "transport: Unknown local device ID ") + + std::to_string(local_sep_addr.first)); + } + const size_t link_idx = _local_device_id_map.at(local_sep_addr.first); + + uhd::transport::io_service::sptr io_srv; + uhd::transport::send_link_if::sptr send_link; + uhd::transport::recv_link_if::sptr recv_link; + bool lossy_xport; + std::tie(io_srv, send_link, std::ignore, recv_link, std::ignore, lossy_xport) = + _link_if_mgr->get_link( + link_idx, uhd::transport::link_type_t::TX_DATA, xport_args); + + /* Associate local device ID with the adapter */ + _adapter_map[local_sep_addr.first] = send_link->get_send_adapter_id(); + + // TODO: configure this based on the transport type + const double fc_freq_ratio = 1.0 / 8; + const double fc_headroom_ratio = 0; + + auto pkt_factory = _link_if_mgr->get_packet_factory(link_idx); + const auto buff_capacity = chdr_tx_data_xport::configure_sep(io_srv, + recv_link, + send_link, + pkt_factory, + mgmt_portal, + epids, + pyld_buff_fmt, + mdata_buff_fmt, + fc_freq_ratio, + fc_headroom_ratio); + + // Create the data transport + auto tx_xport = std::make_unique<chdr_tx_data_xport>(io_srv, + recv_link, + send_link, + pkt_factory, + epids, + send_link->get_num_send_frames(), + buff_capacity); + + + return tx_xport; +} diff --git a/host/lib/usrp/mpmd/mpmd_mb_iface.hpp b/host/lib/usrp/mpmd/mpmd_mb_iface.hpp new file mode 100644 index 000000000..4e47dd35a --- /dev/null +++ b/host/lib/usrp/mpmd/mpmd_mb_iface.hpp @@ -0,0 +1,68 @@ +// +// Copyright 2019 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_MPMD_MB_IFACE_HPP +#define INCLUDED_MPMD_MB_IFACE_HPP + +#include "mpmd_impl.hpp" +#include "mpmd_link_if_mgr.hpp" +#include <uhdlib/rfnoc/mb_iface.hpp> +#include <map> +#include <unordered_map> + +namespace uhd { namespace mpmd { + +class mpmd_mboard_impl::mpmd_mb_iface : public uhd::rfnoc::mb_iface +{ +public: + using uptr = std::unique_ptr<mpmd_mb_iface>; + using clock_iface_list_t = std::vector<std::map<std::string, std::string>>; + mpmd_mb_iface(const uhd::device_addr_t& mb_args, uhd::rpc_client::sptr rpc); + ~mpmd_mb_iface() = default; + + /*** mpmd_mb_iface API calls *****************************************/ + //! Initialize transports + void init(); + + /*** mb_iface API calls **********************************************/ + uint16_t get_proto_ver(); + uhd::rfnoc::chdr_w_t get_chdr_w(); + uhd::endianness_t get_endianness(const uhd::rfnoc::device_id_t local_device_id); + uhd::rfnoc::device_id_t get_remote_device_id(); + std::vector<uhd::rfnoc::device_id_t> get_local_device_ids(); + uhd::transport::adapter_id_t get_adapter_id(const uhd::rfnoc::device_id_t local_device_id); + void reset_network(); + uhd::rfnoc::clock_iface::sptr get_clock_iface(const std::string& clock_name); + uhd::rfnoc::chdr_ctrl_xport::sptr make_ctrl_transport( + uhd::rfnoc::device_id_t local_device_id, const uhd::rfnoc::sep_id_t& local_epid); + uhd::rfnoc::chdr_rx_data_xport::uptr make_rx_data_transport( + uhd::rfnoc::mgmt::mgmt_portal& mgmt_portal, + const uhd::rfnoc::sep_addr_pair_t& addrs, + const uhd::rfnoc::sep_id_pair_t& epids, + const uhd::rfnoc::sw_buff_t pyld_buff_fmt, + const uhd::rfnoc::sw_buff_t mdata_buff_fmt, + const uhd::device_addr_t& xport_args); + uhd::rfnoc::chdr_tx_data_xport::uptr make_tx_data_transport( + uhd::rfnoc::mgmt::mgmt_portal& mgmt_portal, + const uhd::rfnoc::sep_addr_pair_t& addrs, + const uhd::rfnoc::sep_id_pair_t& epids, + const uhd::rfnoc::sw_buff_t pyld_buff_fmt, + const uhd::rfnoc::sw_buff_t mdata_buff_fmt, + const uhd::device_addr_t& xport_args); + +private: + uhd::device_addr_t _mb_args; + uhd::rpc_client::sptr _rpc; + xport::mpmd_link_if_mgr::uptr _link_if_mgr; + uhd::rfnoc::device_id_t _remote_device_id; + std::map<uhd::rfnoc::device_id_t, size_t> _local_device_id_map; + std::unordered_map<uhd::rfnoc::device_id_t, uhd::transport::adapter_id_t> _adapter_map; + std::map<std::string, uhd::rfnoc::clock_iface::sptr> _clock_ifaces; +}; + +}} /* namespace uhd::mpmd */ + +#endif /* INCLUDED_MPMD_MB_IFACE_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp index 83b47b485..5c8fd5485 100644 --- a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp +++ b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp @@ -6,11 +6,13 @@ // #include "mpmd_impl.hpp" +#include "mpmd_mb_iface.hpp" #include <uhd/transport/udp_simple.hpp> #include <uhd/utils/log.hpp> #include <uhd/utils/safe_call.hpp> #include <chrono> #include <thread> +#include <memory> namespace { /************************************************************************* @@ -177,8 +179,8 @@ boost::optional<device_addr_t> mpmd_mboard_impl::is_device_reachable( { UHD_LOG_TRACE( "MPMD", "Checking accessibility of device `" << device_addr.to_string() << "'"); - UHD_ASSERT_THROW(device_addr.has_key(xport::MGMT_ADDR_KEY)); - const std::string rpc_addr = device_addr.get(xport::MGMT_ADDR_KEY); + UHD_ASSERT_THROW(device_addr.has_key(MGMT_ADDR_KEY)); + const std::string rpc_addr = device_addr.get(MGMT_ADDR_KEY); const size_t rpc_port = device_addr.cast<size_t>(mpmd_impl::MPM_RPC_PORT_KEY, mpmd_impl::MPM_RPC_PORT); // 1) Read back device info @@ -258,15 +260,10 @@ mpmd_mboard_impl::mpmd_mboard_impl( const device_addr_t& mb_args_, const std::string& rpc_server_addr) : mb_args(mb_args_) , rpc(make_mpm_rpc_client(rpc_server_addr, mb_args_)) - , num_xbars(rpc->request<size_t>("get_num_xbars")) , _claim_rpc(make_mpm_rpc_client(rpc_server_addr, mb_args, MPMD_CLAIMER_RPC_TIMEOUT)) - // xbar_local_addrs is not yet valid after this! - , xbar_local_addrs(num_xbars, 0xFF) - , _xport_mgr(xport::mpmd_xport_mgr::make(mb_args)) { UHD_LOGGER_TRACE("MPMD") << "Initializing mboard, connecting to RPC server address: " - << rpc_server_addr << " mboard args: " << mb_args.to_string() - << " number of crossbars: " << num_xbars; + << rpc_server_addr << " mboard args: " << mb_args.to_string(); _claimer_task = claim_device_and_make_task(); if (mb_args_.has_key(MPMD_MEAS_LATENCY_KEY)) { @@ -278,7 +275,7 @@ mpmd_mboard_impl::mpmd_mboard_impl( for (const auto& info_pair : device_info_dict) { device_info[info_pair.first] = info_pair.second; } - UHD_LOGGER_TRACE("MPMD") << "MPM reports device info: " << device_info.to_string(); + UHD_LOG_DEBUG("MPMD", "MPM reports device info: " << device_info.to_string()); /// Get dboard info const auto dboards_info = rpc->request<std::vector<dev_info>>("get_dboard_info"); UHD_ASSERT_THROW(this->dboard_info.size() == 0); @@ -293,19 +290,16 @@ mpmd_mboard_impl::mpmd_mboard_impl( this->dboard_info.push_back(this_db_info); } - for (const std::string& key : mb_args_.keys()) { - if (key.find("recv") != std::string::npos) - recv_args[key] = mb_args_[key]; - if (key.find("send") != std::string::npos) - send_args[key] = mb_args_[key]; - } + // Initialize mb_iface and mb_controller + mb_iface = std::make_unique<mpmd_mb_iface>(mb_args, rpc); + mb_ctrl = std::make_shared<rfnoc::mpmd_mb_controller>(rpc, device_info); } mpmd_mboard_impl::~mpmd_mboard_impl() { // Destroy the claimer task to avoid spurious asynchronous reclaim call // after the unclaim. - UHD_SAFE_CALL(dump_logs(); _claimer_task.reset(); _xport_mgr.reset(); + UHD_SAFE_CALL(dump_logs(); _claimer_task.reset(); if (not rpc->request_with_token<bool>("unclaim")) { UHD_LOG_WARNING("MPMD", "Failure to ack unclaim!"); }); @@ -317,79 +311,15 @@ mpmd_mboard_impl::~mpmd_mboard_impl() void mpmd_mboard_impl::init() { init_device(rpc, mb_args); - // RFNoC block clocks are now on. Noc-IDs can be read back. + mb_iface->init(); } /***************************************************************************** * API ****************************************************************************/ -void mpmd_mboard_impl::set_xbar_local_addr( - const size_t xbar_index, const size_t local_addr) -{ - UHD_ASSERT_THROW( - rpc->request_with_token<bool>("set_xbar_local_addr", xbar_index, local_addr)); - UHD_ASSERT_THROW(xbar_index < xbar_local_addrs.size()); - xbar_local_addrs.at(xbar_index) = local_addr; -} - -uhd::both_xports_t mpmd_mboard_impl::make_transport(const sid_t& sid, - usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args) -{ - const std::string xport_type_str = [xport_type]() { - switch (xport_type) { - case mpmd_impl::CTRL: - return "CTRL"; - case mpmd_impl::ASYNC_MSG: - return "ASYNC_MSG"; - case mpmd_impl::RX_DATA: - return "RX_DATA"; - case mpmd_impl::TX_DATA: - return "TX_DATA"; - default: - UHD_THROW_INVALID_CODE_PATH(); - }; - }(); - - UHD_LOGGER_TRACE("MPMD") << __func__ - << "(): Creating new transport of type: " << xport_type_str; - - using namespace uhd::mpmd::xport; - const auto xport_info_list = - rpc->request_with_token<mpmd_xport_mgr::xport_info_list_t>( - "request_xport", sid.get_dst(), sid.get_src(), xport_type_str); - UHD_LOGGER_TRACE("MPMD") << __func__ << "(): request_xport() gave us " - << xport_info_list.size() << " option(s)."; - if (xport_info_list.empty()) { - UHD_LOG_ERROR("MPMD", "No viable transport path found!"); - throw uhd::runtime_error("No viable transport path found!"); - } - - xport::mpmd_xport_mgr::xport_info_t xport_info_out; - auto xports = _xport_mgr->make_transport( - xport_info_list, xport_type, xport_args, xport_info_out); - - if (not rpc->request_with_token<bool>("commit_xport", xport_info_out)) { - UHD_LOG_ERROR("MPMD", "Failed to create UDP transport!"); - throw uhd::runtime_error("commit_xport() failed!"); - } - - return xports; -} - -size_t mpmd_mboard_impl::get_mtu(const uhd::direction_t dir) const -{ - return _xport_mgr->get_mtu(dir); -} - -uhd::device_addr_t mpmd_mboard_impl::get_rx_hints() const -{ - return recv_args; -} - -uhd::device_addr_t mpmd_mboard_impl::get_tx_hints() const +uhd::rfnoc::mb_iface& mpmd_mboard_impl::get_mb_iface() { - return send_args; + return *(mb_iface.get()); } /***************************************************************************** diff --git a/host/lib/usrp/mpmd/mpmd_prop_tree.cpp b/host/lib/usrp/mpmd/mpmd_prop_tree.cpp index 51b88b0e6..dc559f91b 100644 --- a/host/lib/usrp/mpmd/mpmd_prop_tree.cpp +++ b/host/lib/usrp/mpmd/mpmd_prop_tree.cpp @@ -194,16 +194,4 @@ void mpmd_impl::init_property_tree( return _get_component_info(comp_name, mb); }); // Done adding component to property tree } - - /*** MTUs ***********************************************************/ - tree->create<size_t>(mb_path / "mtu/recv") - .add_coerced_subscriber([](const size_t) { - throw uhd::runtime_error("Attempting to write read-only value (MTU)!"); - }) - .set_publisher([mb]() { return mb->get_mtu(uhd::RX_DIRECTION); }); - tree->create<size_t>(mb_path / "mtu/send") - .add_coerced_subscriber([](const size_t) { - throw uhd::runtime_error("Attempting to write read-only value (MTU)!"); - }) - .set_publisher([mb]() { return mb->get_mtu(uhd::TX_DIRECTION); }); } diff --git a/host/lib/usrp/mpmd/mpmd_xport.cpp b/host/lib/usrp/mpmd/mpmd_xport.cpp deleted file mode 100644 index 3ef6a074c..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// -// Copyright 2017 Ettus Research, National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -// make_transport logic for mpmd_impl. Note that mpmd_xport_mgr.* has most of -// the actual transport logic, this for transport-related APIs. - -#include "mpmd_impl.hpp" -#include "mpmd_xport_mgr.hpp" - -using namespace uhd; -using namespace uhd::mpmd; - -uhd::device_addr_t mpmd_impl::get_rx_hints(size_t mb_index) -{ - return _mb.at(mb_index)->get_rx_hints(); -} - -uhd::device_addr_t mpmd_impl::get_tx_hints(size_t mb_index) -{ - return _mb.at(mb_index)->get_tx_hints(); -} - -size_t mpmd_impl::identify_mboard_by_xbar_addr(const size_t xbar_addr) const -{ - for (size_t mb_index = 0; mb_index < _mb.size(); mb_index++) { - for (size_t xbar_index = 0; xbar_index < _mb[mb_index]->num_xbars; xbar_index++) { - if (_mb.at(mb_index)->get_xbar_local_addr(xbar_index) == xbar_addr) { - return mb_index; - } - } - } - throw uhd::lookup_error( - str(boost::format("Cannot identify mboard for crossbar address %d") % xbar_addr)); -} - -both_xports_t mpmd_impl::make_transport(const sid_t& dst_address, - usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& args) -{ - const size_t mb_index = identify_mboard_by_xbar_addr(dst_address.get_dst_addr()); - - const sid_t sid(0, - 0, // Not actually an address, more of an 'ignore me' value - dst_address.get_dst_addr(), - dst_address.get_dst_endpoint()); - UHD_LOGGER_TRACE("MPMD") << "Creating new transport to mboard: " << mb_index - << " SID: " << sid.to_pp_string_hex() - << " User-defined xport args: " << args.to_string(); - - both_xports_t xports = _mb[mb_index]->make_transport(sid, xport_type, args); - UHD_LOGGER_TRACE("MPMD") << "xport info: send_sid==" - << xports.send_sid.to_pp_string_hex() - << " recv_sid==" << xports.recv_sid.to_pp_string_hex() - << " endianness==" - << (xports.endianness == uhd::ENDIANNESS_BIG ? "BE" : "LE") - << " recv_buff_size==" << xports.recv_buff_size - << " send_buff_size==" << xports.send_buff_size; - return xports; -} diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_base.hpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_base.hpp deleted file mode 100644 index a7fff9262..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_base.hpp +++ /dev/null @@ -1,44 +0,0 @@ -// -// Copyright 2017 Ettus Research, a National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#ifndef INCLUDED_MPMD_XPORT_CTRL_BASE_HPP -#define INCLUDED_MPMD_XPORT_CTRL_BASE_HPP - -#include "../device3/device3_impl.hpp" -#include "mpmd_xport_mgr.hpp" -#include <uhd/types/device_addr.hpp> -#include <memory> - -namespace uhd { namespace mpmd { namespace xport { - -/*! Transport manager implementation base - */ -class mpmd_xport_ctrl_base -{ -public: - using uptr = std::unique_ptr<mpmd_xport_ctrl_base>; - virtual ~mpmd_xport_ctrl_base() {} - - /*! This is the final step of a make_transport() sequence - * - * \param xport_info Contains all necessary transport info. The - * implementation may update this! - * \param xport_type CTRL, ASYNC_MSG, ... (see xport_type_t) - * \param xport_args Additional arguments. These can come from the user. - */ - virtual both_xports_t make_transport(mpmd_xport_mgr::xport_info_t& xport_info, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args) = 0; - - //! Assert if an xport_info is even valid/feasible/available - virtual bool is_valid(const mpmd_xport_mgr::xport_info_t& xport_info) const = 0; - - virtual size_t get_mtu(const uhd::direction_t dir) const = 0; -}; - -}}} /* namespace uhd::mpmd::xport */ - -#endif /* INCLUDED_MPMD_XPORT_CTRL_BASE_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.cpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.cpp deleted file mode 100644 index c53eb97a1..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.cpp +++ /dev/null @@ -1,194 +0,0 @@ -// -// Copyright 2017 Ettus Research, National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#include "mpmd_xport_ctrl_liberio.hpp" -#include "../transport/liberio_zero_copy.hpp" -#include <uhd/transport/udp_zero_copy.hpp> -#include <uhd/utils/byteswap.hpp> -#include <uhd/rfnoc/constants.hpp> - -using namespace uhd; -using namespace uhd::mpmd::xport; - -namespace { - -//! Max frame size of a control packet in bytes -const size_t LIBERIO_CTRL_FRAME_MAX_SIZE = 128; -//! Max frame size of an async message packet in bytes -const size_t LIBERIO_ASYNC_FRAME_MAX_SIZE = 256; -//! Max frame size of a flow control packet in bytes -const size_t LIBERIO_FC_FRAME_MAX_SIZE = 64; -//! The max MTU will be this number times the page size -const size_t LIBERIO_PAGES_PER_BUF = 2; -//! Number of descriptors that liberio allocates (receive) -const size_t LIBERIO_NUM_RECV_FRAMES = 128; -//! Number of descriptors that liberio allocates (send) -const size_t LIBERIO_NUM_SEND_FRAMES = 128; - -uint32_t extract_sid_from_pkt(void* pkt, size_t) -{ - return uhd::sid_t(uhd::wtohx(static_cast<const uint32_t*>(pkt)[1])).get_dst(); -} - -} // namespace - -mpmd_xport_ctrl_liberio::mpmd_xport_ctrl_liberio(const uhd::device_addr_t& mb_args) - : _mb_args(mb_args) - , _recv_args(filter_args(mb_args, "recv")) - , _send_args(filter_args(mb_args, "send")) -{ - // nop -} - - -uhd::both_xports_t mpmd_xport_ctrl_liberio::make_transport( - mpmd_xport_mgr::xport_info_t& xport_info, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args_) -{ - auto xport_args = (xport_type == usrp::device3_impl::CTRL) ? - uhd::device_addr_t() : xport_args_; - - // Constrain by this transport's MTU and the MTU passed in - const size_t send_mtu = std::min(get_mtu(uhd::TX_DIRECTION), - xport_args.cast<size_t>("mtu", get_mtu(uhd::TX_DIRECTION))); - const size_t recv_mtu = std::min(get_mtu(uhd::RX_DIRECTION), - xport_args.cast<size_t>("mtu", get_mtu(uhd::RX_DIRECTION))); - size_t send_frame_size = xport_args.cast<size_t>("send_frame_size", send_mtu); - size_t recv_frame_size = xport_args.cast<size_t>("recv_frame_size", recv_mtu); - - // Check any user supplied frame sizes and constrain to MTU - if (xport_args.has_key("send_frame_size") and - xport_type == usrp::device3_impl::TX_DATA) - { - if (send_frame_size > send_mtu) { - UHD_LOGGER_WARNING("MPMD") - << boost::format("Requested send_frame_size of %d exceeds the " - "maximum supported by the hardware. Using %d.") - % send_frame_size % send_mtu; - send_frame_size = send_mtu; - } - } - if (xport_args.has_key("recv_frame_size") and - xport_type == usrp::device3_impl::RX_DATA) - { - size_t recv_frame_size = xport_args.cast<size_t>("recv_frame_size", recv_mtu); - if (recv_frame_size > recv_mtu) { - UHD_LOGGER_WARNING("MPMD") - << boost::format("Requested recv_frame_size of %d exceeds the " - "maximum supported by the hardware. Using %d.") - % recv_frame_size % recv_mtu; - recv_frame_size = recv_mtu; - } - } - - transport::zero_copy_xport_params default_buff_args; - /* default ones for RX / TX, override below */ - - default_buff_args.send_frame_size = send_mtu; - default_buff_args.recv_frame_size = recv_mtu; - default_buff_args.num_recv_frames = LIBERIO_NUM_RECV_FRAMES; - default_buff_args.num_send_frames = LIBERIO_NUM_SEND_FRAMES; - - if (xport_type == usrp::device3_impl::CTRL) { - default_buff_args.send_frame_size = LIBERIO_CTRL_FRAME_MAX_SIZE; - default_buff_args.recv_frame_size = LIBERIO_CTRL_FRAME_MAX_SIZE; - default_buff_args.num_recv_frames = uhd::rfnoc::CMD_FIFO_SIZE / - uhd::rfnoc::MAX_CMD_PKT_SIZE; - default_buff_args.num_send_frames = uhd::rfnoc::CMD_FIFO_SIZE / - uhd::rfnoc::MAX_CMD_PKT_SIZE; - } else if (xport_type == usrp::device3_impl::ASYNC_MSG) { - default_buff_args.send_frame_size = LIBERIO_ASYNC_FRAME_MAX_SIZE; - default_buff_args.recv_frame_size = LIBERIO_ASYNC_FRAME_MAX_SIZE; - } else if (xport_type == usrp::device3_impl::RX_DATA) { - default_buff_args.recv_frame_size = recv_frame_size; - default_buff_args.send_frame_size = LIBERIO_FC_FRAME_MAX_SIZE; - } else { - default_buff_args.recv_frame_size = LIBERIO_FC_FRAME_MAX_SIZE; - default_buff_args.send_frame_size = send_frame_size; - } - - const std::string tx_dev = xport_info["tx_dev"]; - const std::string rx_dev = xport_info["rx_dev"]; - - both_xports_t xports; - xports.lossless = true; - xports.endianness = uhd::ENDIANNESS_LITTLE; - xports.send_sid = sid_t(xport_info["send_sid"]); - xports.recv_sid = xports.send_sid.reversed(); - - if (xport_type == usrp::device3_impl::CTRL) { - UHD_ASSERT_THROW(xport_info["muxed"] == "True"); - if (not _ctrl_dma_xport) { - _ctrl_dma_xport = - make_muxed_liberio_xport(tx_dev, rx_dev, default_buff_args, - uhd::rfnoc::MAX_NUM_BLOCKS * uhd::rfnoc::MAX_NUM_PORTS); - } - - UHD_LOGGER_TRACE("MPMD") - << "Making (muxed) stream with num " << xports.recv_sid.get_dst(); - xports.recv = _ctrl_dma_xport->make_stream(xports.recv_sid.get_dst()); - } else if (xport_type == usrp::device3_impl::ASYNC_MSG) { - UHD_ASSERT_THROW(xport_info["muxed"] == "True"); - if (not _async_msg_dma_xport) { - _async_msg_dma_xport = - make_muxed_liberio_xport(tx_dev, rx_dev, default_buff_args, - uhd::rfnoc::MAX_NUM_BLOCKS * uhd::rfnoc::MAX_NUM_PORTS); - } - - UHD_LOGGER_TRACE("MPMD") - << "making (muxed) stream with num " << xports.recv_sid.get_dst(); - xports.recv = _async_msg_dma_xport->make_stream(xports.recv_sid.get_dst()); - } else { - // Create muxed transport in case of less DMA channels - if (xport_info["muxed"] == "True") { - if (not _data_dma_xport) { - _data_dma_xport = - make_muxed_liberio_xport(tx_dev, rx_dev, default_buff_args, - uhd::rfnoc::MAX_NUM_BLOCKS * uhd::rfnoc::MAX_NUM_PORTS); - } - - UHD_LOGGER_TRACE("MPMD") - << "Making (muxed) stream with num " << xports.recv_sid.get_dst(); - xports.recv = _data_dma_xport->make_stream(xports.recv_sid.get_dst()); - } - else { - xports.recv = - transport::liberio_zero_copy::make(tx_dev, rx_dev, default_buff_args); - } - } - - // Finish both_xports_t object and return: - xports.recv_buff_size = default_buff_args.recv_frame_size * - default_buff_args.num_recv_frames; - xports.send_buff_size = default_buff_args.send_frame_size * - default_buff_args.num_send_frames; - xports.send = xports.recv; - return xports; -} - -bool mpmd_xport_ctrl_liberio::is_valid( - const mpmd_xport_mgr::xport_info_t& xport_info) const -{ - return xport_info.at("type") == "liberio"; -} - -size_t mpmd_xport_ctrl_liberio::get_mtu(const uhd::direction_t /*dir*/) const -{ - return LIBERIO_PAGES_PER_BUF * getpagesize(); -} - -uhd::transport::muxed_zero_copy_if::sptr -mpmd_xport_ctrl_liberio::make_muxed_liberio_xport(const std::string& tx_dev, - const std::string& rx_dev, - const uhd::transport::zero_copy_xport_params& buff_args, - const size_t max_muxed_ports) -{ - auto base_xport = transport::liberio_zero_copy::make(tx_dev, rx_dev, buff_args); - - return uhd::transport::muxed_zero_copy_if::make( - base_xport, extract_sid_from_pkt, max_muxed_ports); -} diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.hpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.hpp deleted file mode 100644 index 36f02fe46..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_liberio.hpp +++ /dev/null @@ -1,56 +0,0 @@ -// -// Copyright 2017 Ettus Research, a National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#ifndef INCLUDED_MPMD_XPORT_ctrl_liberio_HPP -#define INCLUDED_MPMD_XPORT_ctrl_liberio_HPP - -#include "../device3/device3_impl.hpp" -#include "mpmd_xport_ctrl_base.hpp" -#include <uhd/transport/muxed_zero_copy_if.hpp> -#include <uhd/types/device_addr.hpp> - -namespace uhd { namespace mpmd { namespace xport { - -/*! Liberio transport manager - */ -class mpmd_xport_ctrl_liberio : public mpmd_xport_ctrl_base -{ -public: - mpmd_xport_ctrl_liberio(const uhd::device_addr_t& mb_args); - - /*! Open DMA interface to kernel (and thus to FPGA DMA engine) - */ - both_xports_t make_transport(mpmd_xport_mgr::xport_info_t& xport_info, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args); - - bool is_valid(const mpmd_xport_mgr::xport_info_t& xport_info) const; - - size_t get_mtu(const uhd::direction_t dir) const; - -private: - /*! Create a muxed liberio transport for control packets */ - uhd::transport::muxed_zero_copy_if::sptr make_muxed_liberio_xport( - const std::string& tx_dev, - const std::string& rx_dev, - const uhd::transport::zero_copy_xport_params& buff_args, - const size_t max_muxed_ports); - - const uhd::device_addr_t _mb_args; - const uhd::dict<std::string, std::string> _recv_args; - const uhd::dict<std::string, std::string> _send_args; - - //! Control transport for one liberio connection - uhd::transport::muxed_zero_copy_if::sptr _ctrl_dma_xport; - //! Data transport for one liberio connection - uhd::transport::muxed_zero_copy_if::sptr _data_dma_xport; - //! Control transport for one liberio connection - uhd::transport::muxed_zero_copy_if::sptr _async_msg_dma_xport; -}; - -}}} /* namespace uhd::mpmd::xport */ - -#endif /* INCLUDED_MPMD_XPORT_ctrl_liberio_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp deleted file mode 100644 index 20b94899c..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.cpp +++ /dev/null @@ -1,265 +0,0 @@ -// -// Copyright 2017 Ettus Research, National Instruments Company -// Copyright 2019 Ettus Research, National Instruments Brand -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#include "mpmd_xport_ctrl_udp.hpp" -#include "mpmd_impl.hpp" -#include "mpmd_xport_mgr.hpp" -#include <uhd/transport/udp_constants.hpp> -#include <uhd/transport/udp_simple.hpp> -#include <uhd/transport/udp_zero_copy.hpp> - - -using namespace uhd; -using namespace uhd::mpmd::xport; - -namespace { - -//! Maximum CHDR packet size in bytes -const size_t MPMD_10GE_DATA_FRAME_MAX_SIZE = 4000; - -//! Default number of send frames -const size_t MPMD_UDP_DEFAULT_NUM_SEND_FRAMES = 1; -//! Default number of recv frames -const size_t MPMD_UDP_DEFAULT_NUM_RECV_FRAMES = 1; -//! Default message frame size -const size_t MPMD_UDP_MSG_FRAME_SIZE = 256; -//! Default 1GbE send frame size -const size_t MPMD_UDP_1GE_DEFAULT_SEND_FRAME_SIZE = 1472; -//! Default 1GbE receive frame size -const size_t MPMD_UDP_1GE_DEFAULT_RECV_FRAME_SIZE = 1472; -//! Default 10GbE send frame size -const size_t MPMD_UDP_10GE_DEFAULT_SEND_FRAME_SIZE = 4000; -//! Default 10GbE receive frame size -const size_t MPMD_UDP_10GE_DEFAULT_RECV_FRAME_SIZE = 4000; - -//! -const double MPMD_BUFFER_DEPTH = 50.0e-3; // s -//! For MTU discovery, the time we wait for a packet before calling it -// oversized (seconds). -const double MPMD_MTU_DISCOVERY_TIMEOUT = 0.02; - -// TODO: move these to appropriate header file for all other devices -const double MAX_RATE_1GIGE = 1e9 / 8; // byte/s -const double MAX_RATE_10GIGE = 10e9 / 8; // byte/s - -std::vector<std::string> get_addrs_from_mb_args(const uhd::device_addr_t& mb_args) -{ - // mb_args must always include addr - if (not mb_args.has_key(FIRST_ADDR_KEY)) { - throw uhd::runtime_error( - "The " + FIRST_ADDR_KEY - + " key must be specified in " - "device args to create an Ethernet transport to an RFNoC block"); - } - std::vector<std::string> addrs{mb_args[FIRST_ADDR_KEY]}; - if (mb_args.has_key(SECOND_ADDR_KEY)) { - addrs.push_back(mb_args[SECOND_ADDR_KEY]); - } - return addrs; -} - -/*! Do a binary search to discover MTU - * - * Uses the MPM echo service to figure out MTU. We simply send a bunch of - * packets and see if they come back until we converged on the path MTU. - * The end result must lie between \p min_frame_size and \p max_frame_size. - * - * \param address IP address - * \param port UDP port (yeah it's a string!) - * \param min_frame_size Minimum frame size, initialize algorithm to start - * with this value - * \param max_frame_size Maximum frame size, initialize algorithm to start - * with this value - * \param echo_timeout Timeout value in seconds. For frame sizes that - * exceed the MTU, we don't expect a response, and this - * is the amount of time we'll wait before we assume - * the frame size exceeds the MTU. - */ -size_t discover_mtu(const std::string& address, - const std::string& port, - size_t min_frame_size, - size_t max_frame_size, - const double echo_timeout = 0.020) -{ - const size_t echo_prefix_offset = uhd::mpmd::mpmd_impl::MPM_ECHO_CMD.size(); - const size_t mtu_hdr_len = echo_prefix_offset + 10; - UHD_ASSERT_THROW(min_frame_size < max_frame_size); - UHD_ASSERT_THROW(min_frame_size % 4 == 0); - UHD_ASSERT_THROW(max_frame_size % 4 == 0); - UHD_ASSERT_THROW(min_frame_size >= echo_prefix_offset + mtu_hdr_len); - using namespace uhd::transport; - // The return port will probably differ from the discovery port, so we - // need a "broadcast" UDP connection; using make_connected() would - // drop packets - udp_simple::sptr udp = udp_simple::make_broadcast(address, port); - std::string send_buf(uhd::mpmd::mpmd_impl::MPM_ECHO_CMD); - send_buf.resize(max_frame_size, '#'); - UHD_ASSERT_THROW(send_buf.size() == max_frame_size); - std::vector<uint8_t> recv_buf; - recv_buf.resize(max_frame_size, ' '); - - // Little helper to check returned packets match the sent ones - auto require_bufs_match = [&recv_buf, &send_buf, mtu_hdr_len](const size_t len) { - if (len < mtu_hdr_len - or std::memcmp((void*)&recv_buf[0], (void*)&send_buf[0], mtu_hdr_len) != 0) { - throw uhd::runtime_error("Unexpected content of MTU " - "discovery return packet!"); - } - }; - UHD_LOG_TRACE("MPMD", "Determining UDP MTU... "); - size_t seq_no = 0; - while (min_frame_size < max_frame_size) { - // Only test multiples of 4 bytes! - const size_t test_frame_size = (max_frame_size / 2 + min_frame_size / 2 + 3) - & ~size_t(3); - // Encode sequence number and current size in the string, makes it - // easy to debug in code or Wireshark. Is also used for identifying - // response packets. - std::sprintf( - &send_buf[echo_prefix_offset], ";%04lu,%04lu", seq_no++, test_frame_size); - UHD_LOG_TRACE("MPMD", "Testing frame size " << test_frame_size); - udp->send(boost::asio::buffer(&send_buf[0], test_frame_size)); - - const size_t len = udp->recv(boost::asio::buffer(recv_buf), echo_timeout); - if (len == 0) { - // Nothing received, so this is probably too big - max_frame_size = test_frame_size - 4; - } else if (len >= test_frame_size) { - // Size went through, so bump the minimum - require_bufs_match(len); - min_frame_size = test_frame_size; - } else if (len < test_frame_size) { - // This is an odd case. Something must have snipped the packet - // on the way back. Still, we'll just back off and try - // something smaller. - UHD_LOG_DEBUG("MPMD", "Unexpected packet truncation during MTU discovery."); - require_bufs_match(len); - max_frame_size = len; - } - } - UHD_LOG_DEBUG("MPMD", "Path MTU for address " << address << ": " << min_frame_size); - return min_frame_size; -} - -} // namespace - - -mpmd_xport_ctrl_udp::mpmd_xport_ctrl_udp(const uhd::device_addr_t& mb_args) - : _mb_args(mb_args) - , _recv_args(filter_args(mb_args, "recv")) - , _send_args(filter_args(mb_args, "send")) - , _available_addrs(get_addrs_from_mb_args(mb_args)) - , _mtu(MPMD_10GE_DATA_FRAME_MAX_SIZE) -{ - const std::string mpm_discovery_port = _mb_args.get( - mpmd_impl::MPM_DISCOVERY_PORT_KEY, std::to_string(mpmd_impl::MPM_DISCOVERY_PORT)); - auto discover_mtu_for_ip = [mpm_discovery_port](const std::string& ip_addr) { - return discover_mtu(ip_addr, - mpm_discovery_port, - IP_PROTOCOL_MIN_MTU_SIZE - IP_PROTOCOL_UDP_PLUS_IP_HEADER, - MPMD_10GE_DATA_FRAME_MAX_SIZE, - MPMD_MTU_DISCOVERY_TIMEOUT); - }; - - for (const auto& ip_addr : _available_addrs) { - _mtu = std::min(_mtu, discover_mtu_for_ip(ip_addr)); - } -} - -uhd::both_xports_t mpmd_xport_ctrl_udp::make_transport( - mpmd_xport_mgr::xport_info_t& xport_info, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args) -{ - - double link_speed = MAX_RATE_1GIGE; - if (xport_info.count("link_speed") == 0) { - UHD_LOG_WARNING("MPMD", - "Could not determine link speed; using 1GibE max speed of " - << MAX_RATE_1GIGE); - } else { - link_speed = xport_info.at("link_speed") == "10000" ? MAX_RATE_10GIGE - : MAX_RATE_1GIGE; - } - - // Constrain by this transport's MTU and the MTU in the xport_args - const size_t send_mtu = std::min(get_mtu(uhd::TX_DIRECTION), - xport_args.cast<size_t>("mtu", get_mtu(uhd::TX_DIRECTION))); - const size_t recv_mtu = std::min(get_mtu(uhd::RX_DIRECTION), - xport_args.cast<size_t>("mtu", get_mtu(uhd::RX_DIRECTION))); - - // Create actual UDP transport - transport::zero_copy_xport_params default_buff_args; - default_buff_args.num_send_frames = MPMD_UDP_DEFAULT_NUM_SEND_FRAMES; - default_buff_args.num_recv_frames = MPMD_UDP_DEFAULT_NUM_RECV_FRAMES; - default_buff_args.recv_frame_size = MPMD_UDP_MSG_FRAME_SIZE; - default_buff_args.send_frame_size = MPMD_UDP_MSG_FRAME_SIZE; - default_buff_args.recv_buff_size = link_speed * MPMD_BUFFER_DEPTH; - default_buff_args.send_buff_size = link_speed * MPMD_BUFFER_DEPTH; - if (xport_type == usrp::device3_impl::CTRL) { - default_buff_args.num_recv_frames = - uhd::rfnoc::CMD_FIFO_SIZE / uhd::rfnoc::MAX_CMD_PKT_SIZE; - } else if (xport_type == usrp::device3_impl::TX_DATA) { - const size_t default_frame_size = (link_speed == MAX_RATE_10GIGE ? - MPMD_UDP_10GE_DEFAULT_SEND_FRAME_SIZE : - MPMD_UDP_1GE_DEFAULT_SEND_FRAME_SIZE); - default_buff_args.send_frame_size = - xport_args.cast<size_t>("send_frame_size", - std::min(default_frame_size, send_mtu)); - default_buff_args.num_send_frames = - xport_args.cast<size_t>("num_send_frames", - default_buff_args.num_send_frames); - default_buff_args.send_buff_size = - xport_args.cast<size_t>("send_buff_size", - default_buff_args.send_buff_size); - } else if (xport_type == usrp::device3_impl::RX_DATA) { - const size_t default_frame_size = (link_speed == MAX_RATE_10GIGE ? - MPMD_UDP_10GE_DEFAULT_RECV_FRAME_SIZE : - MPMD_UDP_1GE_DEFAULT_RECV_FRAME_SIZE); - default_buff_args.recv_frame_size = - xport_args.cast<size_t>("recv_frame_size", - std::min(default_frame_size, recv_mtu)); - default_buff_args.num_recv_frames = - xport_args.cast<size_t>("num_recv_frames", - default_buff_args.num_recv_frames); - default_buff_args.recv_buff_size = - xport_args.cast<size_t>("recv_buff_size", - default_buff_args.recv_buff_size); - } - transport::udp_zero_copy::buff_params buff_params; - auto recv = transport::udp_zero_copy::make(xport_info["ipv4"], - xport_info["port"], - default_buff_args, - buff_params); - const uint16_t port = recv->get_local_port(); - const std::string src_ip_addr = recv->get_local_addr(); - xport_info["src_port"] = std::to_string(port); - xport_info["src_ipv4"] = src_ip_addr; - - // Create both_xports_t object and finish: - both_xports_t xports; - xports.endianness = uhd::ENDIANNESS_BIG; - xports.send_sid = sid_t(xport_info["send_sid"]); - xports.recv_sid = xports.send_sid.reversed(); - xports.recv_buff_size = buff_params.recv_buff_size; - xports.send_buff_size = buff_params.send_buff_size; - xports.recv = recv; // Note: This is a type cast! - xports.send = recv; // This too - return xports; -} - -bool mpmd_xport_ctrl_udp::is_valid(const mpmd_xport_mgr::xport_info_t& xport_info) const -{ - return std::find( - _available_addrs.cbegin(), _available_addrs.cend(), xport_info.at("ipv4")) - != _available_addrs.cend(); -} - -size_t mpmd_xport_ctrl_udp::get_mtu(const uhd::direction_t /*dir*/) const -{ - return _mtu; -} diff --git a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.hpp b/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.hpp deleted file mode 100644 index 86301bb2a..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_ctrl_udp.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// -// Copyright 2017 Ettus Research, a National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#ifndef INCLUDED_MPMD_XPORT_ctrl_udp_HPP -#define INCLUDED_MPMD_XPORT_ctrl_udp_HPP - -#include "../device3/device3_impl.hpp" -#include "mpmd_xport_ctrl_base.hpp" -#include <uhd/types/device_addr.hpp> - -namespace uhd { namespace mpmd { namespace xport { - -/*! UDP transport manager - * - * Opens UDP sockets - */ -class mpmd_xport_ctrl_udp : public mpmd_xport_ctrl_base -{ -public: - mpmd_xport_ctrl_udp(const uhd::device_addr_t& mb_args); - - both_xports_t make_transport(mpmd_xport_mgr::xport_info_t& xport_info, - const usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args); - - bool is_valid(const mpmd_xport_mgr::xport_info_t& xport_info) const; - - size_t get_mtu(const uhd::direction_t dir) const; - -private: - const uhd::device_addr_t _mb_args; - const uhd::dict<std::string, std::string> _recv_args; - const uhd::dict<std::string, std::string> _send_args; - //! A list of IP addresses we can connect our CHDR connections to - const std::vector<std::string> _available_addrs; - //! MTU - size_t _mtu; -}; - -}}} /* namespace uhd::mpmd::xport */ - -#endif /* INCLUDED_MPMD_XPORT_ctrl_udp_HPP */ diff --git a/host/lib/usrp/mpmd/mpmd_xport_mgr.cpp b/host/lib/usrp/mpmd/mpmd_xport_mgr.cpp deleted file mode 100644 index d3023e3af..000000000 --- a/host/lib/usrp/mpmd/mpmd_xport_mgr.cpp +++ /dev/null @@ -1,173 +0,0 @@ -// -// Copyright 2017 Ettus Research, National Instruments Company -// -// SPDX-License-Identifier: GPL-3.0-or-later -// - -#include "mpmd_xport_mgr.hpp" -#include "mpmd_impl.hpp" -#include "mpmd_xport_ctrl_base.hpp" -#include "mpmd_xport_ctrl_udp.hpp" -#ifdef HAVE_LIBERIO -# include "mpmd_xport_ctrl_liberio.hpp" -#endif -#ifdef HAVE_DPDK -# include "mpmd_xport_ctrl_dpdk_udp.hpp" -#endif - -uhd::dict<std::string, std::string> uhd::mpmd::xport::filter_args( - const uhd::device_addr_t& args, const std::string& prefix) -{ - uhd::dict<std::string, std::string> filtered_args; - for (const std::string& key : args.keys()) { - if (key.find(prefix) != std::string::npos) { - filtered_args[key] = args[key]; - } - } - - return filtered_args; -} - -using namespace uhd::mpmd::xport; - -class mpmd_xport_mgr_impl : public mpmd_xport_mgr -{ -public: - mpmd_xport_mgr_impl(const uhd::device_addr_t& mb_args) : _mb_args(mb_args) - { - // nop - } - - /************************************************************************** - * API (see mpmd_xport_mgr.hpp) - *************************************************************************/ - uhd::both_xports_t make_transport(const xport_info_list_t& xport_info_list, - const uhd::usrp::device3_impl::xport_type_t xport_type, - const uhd::device_addr_t& xport_args, - xport_info_t& xport_info_out) - { - for (const auto& xport_info : xport_info_list) { - require_xport_mgr(xport_info.at("type")); - } - - // Run our incredibly smart selection algorithm - xport_info_out = select_xport_option(xport_info_list); - const std::string xport_medium = xport_info_out.at("type"); - UHD_LOG_TRACE("MPMD", __func__ << "(): xport medium is " << xport_medium); - - UHD_ASSERT_THROW(_xport_ctrls.count(xport_medium) > 0); - UHD_ASSERT_THROW(_xport_ctrls.at(xport_medium)); - // When we've picked our preferred option, pass it to the transport - // implementation for execution: - return _xport_ctrls.at(xport_medium) - ->make_transport(xport_info_out, xport_type, xport_args); - } - - size_t get_mtu(const uhd::direction_t dir) const - { - if (_xport_ctrls.empty()) { - UHD_LOG_WARNING("MPMD", - "Cannot determine MTU, no transport controls have been " - "established!"); - return 0; - } - - size_t mtu = ~size_t(0); - for (const auto& xport_ctrl_pair : _xport_ctrls) { - mtu = std::min(mtu, xport_ctrl_pair.second->get_mtu(dir)); - } - - return mtu; - } - - -private: - /************************************************************************** - * Private methods / helpers - *************************************************************************/ - /*! Picks a transport option based on available data - * - * \param xport_info_list List of available options, they all need to be - * valid choices. - * - * \returns One element of \p xport_info_list based on a selection - * algorithm. - */ - xport_info_t select_xport_option(const xport_info_list_t& xport_info_list) const - { - for (const auto& xport_info : xport_info_list) { - const std::string xport_medium = xport_info.at("type"); - if (_xport_ctrls.count(xport_medium) != 0 and _xport_ctrls.at(xport_medium) - and _xport_ctrls.at(xport_medium)->is_valid(xport_info)) { - return xport_info; - } - } - - throw uhd::runtime_error( - "Could not select a transport option! " - "Either a transport hint was not specified or the specified " - "hint does not support communication with RFNoC blocks."); - } - - //! Create an instance of an xport manager implementation - // - // \param xport_medium "UDP" or "liberio" - // \param mb_args Device args - mpmd_xport_ctrl_base::uptr make_mgr_impl( - const std::string& xport_medium, const uhd::device_addr_t& mb_args) const - { - if (xport_medium == "UDP") { -#ifdef HAVE_DPDK - if (mb_args.has_key("use_dpdk")) { - return mpmd_xport_ctrl_base::uptr(new mpmd_xport_ctrl_dpdk_udp(mb_args)); - } -#endif - return mpmd_xport_ctrl_base::uptr(new mpmd_xport_ctrl_udp(mb_args)); -#ifdef HAVE_LIBERIO - } else if (xport_medium == "liberio") { - return mpmd_xport_ctrl_base::uptr(new mpmd_xport_ctrl_liberio(mb_args)); -#endif - } else { - UHD_LOG_WARNING( - "MPMD", "Cannot instantiate transport medium " << xport_medium); - return nullptr; - } - } - - //! This will try to make _xport_ctrls contain a valid transport manager - // for \p xport_medium - // - // When this function returns, it will be possible to access - // this->_xport_ctrls[xport_medium]. - // - // \param xport_medium Type of transport, e.g. "UDP", "liberio", ... - // - // \throws uhd::key_error if \p xport_medium is not known or registered - void require_xport_mgr(const std::string& xport_medium) - { - if (_xport_ctrls.count(xport_medium) == 0) { - UHD_LOG_TRACE( - "MPMD", "Instantiating transport manager `" << xport_medium << "'"); - auto mgr_impl = make_mgr_impl(xport_medium, _mb_args); - if (mgr_impl) { - _xport_ctrls[xport_medium] = std::move(mgr_impl); - } - } - } - - /************************************************************************** - * Private attributes - *************************************************************************/ - //! Cache available xport manager implementations - // - // Should only every be populated by require_xport_mgr() - std::unordered_map<std::string, mpmd_xport_ctrl_base::uptr> _xport_ctrls; - - //! Motherboard args, can contain things like 'recv_buff_size' - const uhd::device_addr_t _mb_args; -}; - -mpmd_xport_mgr::uptr mpmd_xport_mgr::make(const uhd::device_addr_t& mb_args) -{ - return mpmd_xport_mgr::uptr(new mpmd_xport_mgr_impl(mb_args)); -} |