mpm/mpmd: Further improvements on multi-device operations

- Crossbar local addr is now written during initialization of UHD
  session
- Support for multiple crossbars per device (at least on UHD side)
- Crossbars are now sequentially given a unique address

3 files changed, 168 insertions, 51 deletions

diff --git a/host/lib/usrp/mpmd/mpmd_impl.cpp b/host/lib/usrp/mpmd/mpmd_impl.cpp
index 353258aeb..efc89f86e 100644
--- a/host/lib/usrp/mpmd/mpmd_impl.cpp
+++ b/host/lib/usrp/mpmd/mpmd_impl.cpp
@@ -36,17 +36,20 @@
 
 using namespace uhd;
 
+namespace {
+    const size_t MPMD_CROSSBAR_MAX_LADDR = 255;
+}
+
 /*****************************************************************************
  * Structors
  ****************************************************************************/
 mpmd_impl::mpmd_impl(const device_addr_t& device_args)
     : usrp::device3_impl()
-    , _device_addr(device_args)
+    , _device_args(device_args)
     , _sid_framer(0)
 {
     UHD_LOGGER_INFO("MPMD")
-        << "MPMD initialization sequence. Device args: "
-        << device_args.to_string();
+        << "Initializing device with args: " << device_args.to_string();
 
     for (const std::string& key : device_args.keys()) {
         if (key.find("recv") != std::string::npos) {
@@ -59,8 +62,17 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args)
 
     const device_addrs_t mb_args = separate_device_addr(device_args);
     _mb.reserve(mb_args.size());
+
+    // This can theoretically be parallelized, but then we want to make sure
+    // we're distributing crossbar local addresses in some orderly fashion.
+    // At the very least, _xbar_local_addr_ctr needs to become atomic.
     for (size_t mb_i = 0; mb_i < mb_args.size(); ++mb_i) {
         _mb.push_back(setup_mb(mb_i, mb_args[mb_i]));
+    }
+
+    //! This might be parallelized. std::tasks would probably be a good way to
+    // do that if we want to.
+    for (size_t mb_i = 0; mb_i < mb_args.size(); ++mb_i) {
         setup_rfnoc_blocks(mb_i, mb_args[mb_i]);
     }
 
@@ -71,7 +83,10 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args)
     setup_rpc_blocks(filtered_block_args);
 }
 
-mpmd_impl::~mpmd_impl() {}
+mpmd_impl::~mpmd_impl()
+{
+    /* nop */
+}
 
 /*****************************************************************************
  * Private methods
@@ -80,13 +95,20 @@ mpmd_mboard_impl::uptr mpmd_impl::setup_mb(
     const size_t mb_index,
     const uhd::device_addr_t& device_args
 ) {
-    UHD_LOG_DEBUG("MPMD",
-        "Initializing mboard " << mb_index << ". Device args: "
-        << device_args.to_string()
+    UHD_LOGGER_DEBUG("MPMD")
+        << "Initializing mboard " << mb_index
+        << ". Device args: " << device_args.to_string()
+    ;
+
+    auto mb = mpmd_mboard_impl::make(
+        device_args,
+        device_args["addr"]
     );
+    for (size_t xbar_index = 0; xbar_index < mb->num_xbars; xbar_index++) {
+        mb->set_xbar_local_addr(xbar_index, allocate_xbar_local_addr());
+    }
 
     const fs_path mb_path = fs_path("/mboards") / mb_index;
-    auto mb = mpmd_mboard_impl::make(device_args["addr"]);
     _tree->create<std::string>(mb_path / "name")
         .set(mb->device_info.get("type", "UNKNOWN"));
     _tree->create<std::string>(mb_path / "serial")
@@ -117,39 +139,46 @@ void mpmd_impl::setup_rfnoc_blocks(
     const size_t mb_index,
     const uhd::device_addr_t& ctrl_xport_args
 ) {
-    auto mb = _mb[mb_index].get();
+    auto &mb = _mb[mb_index];
     mb->num_xbars = mb->rpc->call<size_t>("get_num_xbars");
-    UHD_ASSERT_THROW(mb->num_xbars >= 1);
-    if (mb->num_xbars > 1) {
-        UHD_LOG_WARNING("MPMD", "Only using first crossbar");
-    }
-    // TODO loop over all xbars
-    const size_t xbar_index = 0;
-    const size_t num_blocks = mb->rpc->call<size_t>("get_num_blocks", xbar_index);
-    const size_t base_port = mb->rpc->call<size_t>("get_base_port", xbar_index);
-    UHD_LOG_TRACE("MPMD",
-        "Enumerating RFNoC blocks for xbar " << xbar_index <<
-        ". Total blocks: " << num_blocks <<
-        " Base port: " << base_port
+    UHD_LOG_TRACE("MPM",
+        "Mboard " << mb_index << " reports " << mb->num_xbars << " crossbar(s)."
     );
-    try {
-        enumerate_rfnoc_blocks(
-          mb_index,
-          num_blocks,
-          base_port,
-          uhd::sid_t(0x0200), // TODO don't hardcode
-          ctrl_xport_args
-        );
-    } catch (const std::exception &ex) {
-        UHD_LOGGER_ERROR("MPMD")
-            << "Failure during block enumeration: "
-            << ex.what();
-        throw uhd::runtime_error("Failed to run enumerate_rfnoc_blocks()");
+
+    for (size_t xbar_index = 0; xbar_index < mb->num_xbars; xbar_index++) {
+        const size_t num_blocks =
+            mb->rpc->call<size_t>("get_num_blocks", xbar_index);
+        const size_t base_port =
+            mb->rpc->call<size_t>("get_base_port", xbar_index);
+        const size_t local_addr = mb->get_xbar_local_addr(xbar_index);
+        UHD_LOGGER_TRACE("MPMD")
+            << "Enumerating RFNoC blocks for xbar " << xbar_index
+            << ". Total blocks: " << num_blocks
+            << " Base port: " << base_port
+            << " Local address: " << local_addr
+        ;
+        try {
+            enumerate_rfnoc_blocks(
+              mb_index,
+              num_blocks,
+              base_port,
+              uhd::sid_t(0, 0, local_addr, 0),
+              ctrl_xport_args
+            );
+        } catch (const std::exception &ex) {
+            UHD_LOGGER_ERROR("MPMD")
+                << "Failure during block enumeration: "
+                << ex.what();
+            throw uhd::runtime_error("Failed to run enumerate_rfnoc_blocks()");
+        }
     }
 }
 
 void mpmd_impl::setup_rpc_blocks(const device_addr_t &block_args)
 {
+    // This could definitely be parallelized. Blocks may do all sorts of stuff
+    // inside set_rpc_client(), and it can take any amount of time (I mean,
+    // like, seconds).
     for (const auto &block_ctrl: _rfnoc_block_ctrl) {
         auto rpc_block_id = block_ctrl->get_block_id();
         if (has_block<uhd::rfnoc::rpc_block_ctrl>(block_ctrl->get_block_id())) {
@@ -164,6 +193,15 @@ void mpmd_impl::setup_rpc_blocks(const device_addr_t &block_args)
     }
 }
 
+size_t mpmd_impl::allocate_xbar_local_addr()
+{
+    const size_t new_local_addr = _xbar_local_addr_ctr++;
+    if (new_local_addr > MPMD_CROSSBAR_MAX_LADDR) {
+        throw uhd::runtime_error("Too many crossbars.");
+    }
+
+    return new_local_addr;
+}
 
 
 /*****************************************************************************
@@ -219,7 +257,7 @@ both_xports_t mpmd_impl::make_transport(const sid_t& address,
     std::cout << address.get_dst_addr() << std::endl;
     */
 
-    std::string interface_addr = _device_addr["addr"];
+    std::string interface_addr = _device_args["addr"];
     const uint32_t xbar_src_addr = address.get_src_addr();
     const uint32_t xbar_src_dst = 0;
 
diff --git a/host/lib/usrp/mpmd/mpmd_impl.hpp b/host/lib/usrp/mpmd/mpmd_impl.hpp
index 6af2e4385..3bf8293cd 100644
--- a/host/lib/usrp/mpmd/mpmd_impl.hpp
+++ b/host/lib/usrp/mpmd/mpmd_impl.hpp
@@ -52,15 +52,29 @@ class mpmd_mboard_impl
     using dev_info = std::map<std::string, std::string>;
 
     /*** Structors ***********************************************************/
-    mpmd_mboard_impl(const std::string& addr);
+    /*!
+     * \param mb_args Device args that pertain to this motherboard
+     * \param ip_addr RPC client will attempt to connect to this IP address
+     */
+    mpmd_mboard_impl(
+            const uhd::device_addr_t &mb_args,
+            const std::string& ip_addr
+    );
     ~mpmd_mboard_impl();
 
     /*** Factory *************************************************************/
-    static uptr make(const std::string& addr);
+    /*!
+     * \param mb_args Device args that pertain to this motherboard
+     * \param ip_addr RPC client will attempt to connect to this IP address
+     */
+    static uptr make(
+            const uhd::device_addr_t &mb_args,
+            const std::string& addr
+    );
 
     /*** Public attributes ***************************************************/
     //! Device information is read back via MPM and stored here.
-    uhd::dict<std::string, std::string> device_info;
+    uhd::device_addr_t device_info;
 
     //! Number of RFNoC crossbars on this device
     size_t num_xbars = 0;
@@ -81,7 +95,23 @@ class mpmd_mboard_impl
                             const uint32_t xbar_src_addr,
                             const uint32_t xbar_src_dst);
 
+    //! Configure a crossbar to have a certain local address
+    void set_xbar_local_addr(const size_t xbar_index, const size_t local_addr);
+
+    //! Return the local address of a given crossbar
+    size_t get_xbar_local_addr(const size_t xbar_index) {
+        return xbar_local_addrs.at(xbar_index);
+    }
+
   private:
+    /*************************************************************************
+     * Private attributes
+     ************************************************************************/
+    //! Stores a list of local addresses of the crossbars. The local address is
+    // what we use when addressing a crossbar in a CHDR header.
+    std::vector<size_t> xbar_local_addrs;
+
+
     /*! Renew the claim onto the device.
      *
      * This is meant to be called repeatedly, e.g., using a UHD task.
@@ -111,14 +141,20 @@ class mpmd_impl : public uhd::usrp::device3_impl
                                       const uhd::device_addr_t&);
 
   private:
+    /*! Initialize a single motherboard
+     *
+     * - See mpmd_mboard_impl ctor for details
+     * - Also allocates the local crossbar addresses
+     */
     mpmd_mboard_impl::uptr setup_mb(
             const size_t mb_i,
             const uhd::device_addr_t& dev_addr
     );
 
+    //! Setup all RFNoC blocks running on mboard \p mb_i
     void setup_rfnoc_blocks(
             const size_t mb_i,
-            const uhd::device_addr_t& dev_addr
+            const uhd::device_addr_t& block_args
     );
 
     //! Configure all blocks that require access to an RPC client
@@ -126,11 +162,26 @@ class mpmd_impl : public uhd::usrp::device3_impl
 
     uhd::device_addr_t get_rx_hints(size_t mb_index);
 
+    /*! Returns a valid local address for a crossbar
+     *
+     * \returns Valid local address
+     * \throws uhd::runtime_error if there are no more local addresses
+     */
+    size_t allocate_xbar_local_addr();
+
+
     uhd::dict<std::string, std::string> recv_args;
     uhd::dict<std::string, std::string> send_args;
 
-    uhd::device_addr_t _device_addr;
+    //! Stores the args with which the device was originally initialized
+    uhd::device_addr_t _device_args;
+    //! Stores a list of mboard references
     std::vector<mpmd_mboard_impl::uptr> _mb;
+
+    //! A counter for distributing local addresses to crossbars
+    // No-one touches this except allocate_xbar_local_addr(), gotcha?
+    size_t  _xbar_local_addr_ctr = 2;
+
     size_t _sid_framer;
 };
 
diff --git a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
index 46e8d23a9..934663e29 100644
--- a/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
+++ b/host/lib/usrp/mpmd/mpmd_mboard_impl.cpp
@@ -21,6 +21,8 @@
 
 namespace {
     const size_t MPMD_RECLAIM_INTERVAL_MS = 1000;
+
+    const std::string MPMD_DEFAULT_SESSION_ID = "UHD";
 }
 
 using namespace uhd;
@@ -28,15 +30,26 @@ using namespace uhd;
 /*****************************************************************************
  * Structors
  ****************************************************************************/
-mpmd_mboard_impl::mpmd_mboard_impl(const std::string& ip_addr)
-    : rpc(uhd::rpc_client::make(ip_addr, MPM_RPC_PORT))
+mpmd_mboard_impl::mpmd_mboard_impl(
+        const device_addr_t &mb_args,
+        const std::string& ip_addr
+) : rpc(uhd::rpc_client::make(ip_addr, MPM_RPC_PORT))
 {
-    UHD_LOG_TRACE("MPMD", "Initializing mboard, IP address: " << ip_addr);
-    auto _dev_info = rpc->call<dev_info>("get_device_info");
-    device_info =
-        dict<std::string, std::string>(_dev_info.begin(), _dev_info.end());
-    // Get initial claim on mboard
-    auto rpc_token = rpc->call<std::string>("claim", "UHD - Session 01"); // TODO make this configurable with device_addr, and provide better defaults
+    UHD_LOGGER_TRACE("MPMD")
+        << "Initializing mboard, connecting to IP address: " << ip_addr
+        << " mboard args: " << mb_args.to_string()
+    ;
+    auto device_info_dict = rpc->call<dev_info>("get_device_info");
+    for (const auto &info_pair: device_info_dict) {
+        device_info[info_pair.first] = info_pair.second;
+    }
+    UHD_LOGGER_TRACE("MPMD")
+        << "MPM reports device info: " << device_info.to_string();
+
+    // Claim logic
+    auto rpc_token = rpc->call<std::string>("claim",
+        mb_args.get("session_id", MPMD_DEFAULT_SESSION_ID)
+    );
     if (rpc_token.empty()) {
         throw uhd::value_error("mpmd device claiming failed!");
     }
@@ -50,6 +63,11 @@ mpmd_mboard_impl::mpmd_mboard_impl(const std::string& ip_addr)
         );
     });
 
+    // Initialize properties
+    this->num_xbars = rpc->call<size_t>("get_num_xbars");
+    // Local addresses are not yet valid after this!
+    this->xbar_local_addrs.resize(this->num_xbars, 0xFF);
+
     // std::vector<std::string> data_ifaces =
     //     rpc.call<std::vector<std::string>>("get_interfaces", rpc_token);
 
@@ -89,6 +107,14 @@ uhd::sid_t mpmd_mboard_impl::allocate_sid(const uint16_t port,
     return uhd::sid_t(sid);
 }
 
+void mpmd_mboard_impl::set_xbar_local_addr(
+        const size_t xbar_index,
+        const size_t local_addr
+) {
+    rpc->call_with_token<void>("set_xbar_local_addr", xbar_index, local_addr);
+    UHD_ASSERT_THROW(xbar_index < xbar_local_addrs.size());
+    xbar_local_addrs.at(xbar_index) = local_addr;
+}
 
 /*****************************************************************************
  * Private methods
@@ -101,10 +127,12 @@ bool mpmd_mboard_impl::claim()
 /*****************************************************************************
  * Factory
  ****************************************************************************/
-mpmd_mboard_impl::uptr mpmd_mboard_impl::make(const std::string& addr)
-{
+mpmd_mboard_impl::uptr mpmd_mboard_impl::make(
+    const uhd::device_addr_t &mb_args,
+    const std::string& addr
+) {
     mpmd_mboard_impl::uptr mb =
-        mpmd_mboard_impl::uptr(new mpmd_mboard_impl(addr));
+        mpmd_mboard_impl::uptr(new mpmd_mboard_impl(mb_args, addr));
     // implicit move
     return mb;
 }