rfnoc: Allow MB controllers to init after blocks have initialized

This allows mb_controller childs to implement an init() call which
rfnoc_graph will call after the block initialization is complete.

rfnoc: graph/mb_controller: Add synchronization routine

This adds two new API calls:
* rfnoc_graph::synchronize_devices() and
* mb_controller::synchronize().

The purpose is to synchronize devices in time and/or phase, depending on
device capabilities. mb_controller childs can override or extend the
default implementation, which is to simply set time next PPS and verify
(similar to the set_time_unknown_pps() call in multi_usrp).

rfnoc: mb_controller: Add gpio_src API

Adds new API calls (get_gpio_src, get_gpio_srcs, set_gpio_src,
get_gpio_banks) to mb_controllers

2 files changed, 164 insertions, 6 deletions

diff --git a/host/lib/rfnoc/mb_controller.cpp b/host/lib/rfnoc/mb_controller.cpp
index 10d5ebe47..62ec1f886 100644
--- a/host/lib/rfnoc/mb_controller.cpp
+++ b/host/lib/rfnoc/mb_controller.cpp
@@ -4,12 +4,110 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 //
 
-#include <uhd/rfnoc/mb_controller.hpp>
 #include <uhd/exception.hpp>
+#include <uhd/rfnoc/mb_controller.hpp>
+#include <uhd/utils/algorithm.hpp>
+#include <uhd/utils/log.hpp>
 #include <atomic>
+#include <chrono>
+#include <thread>
 
 using namespace uhd::rfnoc;
+using namespace std::chrono_literals;
 
+namespace {
+const std::vector<std::string> SYNCHRONIZABLE_REF_SOURCES = {"gpsdo", "external"};
+}
+
+bool mb_controller::synchronize(std::vector<mb_controller::sptr>& mb_controllers,
+    const uhd::time_spec_t& time_spec,
+    const bool quiet)
+{
+    if (mb_controllers.empty()) {
+        return false;
+    }
+    if (mb_controllers.size() == 1) {
+        UHD_LOG_TRACE("MB_CTRL", "Skipping time synchronization of a single USRP.");
+        mb_controllers.at(0)->get_timekeeper(0)->set_time_now(time_spec);
+        return true;
+    }
+    // Verify that all devices share a time reference, and that it is a common
+    // one
+    const std::string time_source = mb_controllers.at(0)->get_time_source();
+    if (!uhd::has(SYNCHRONIZABLE_REF_SOURCES, time_source)) {
+        if (!quiet) {
+            UHD_LOG_WARNING("MB_CTRL",
+                "The selected time source "
+                    << time_source << " does not allow synchronization between devices.");
+        }
+        return false;
+    }
+    for (auto& mbc : mb_controllers) {
+        if (mbc->get_time_source() != time_source) {
+            if (!quiet) {
+                UHD_LOG_WARNING("MB_CTRL",
+                    "Motherboards do not share a time source, and thus cannot be "
+                    "synchronized!");
+            }
+            return false;
+        }
+    }
+
+    // Get a reference to all timekeepers
+    std::vector<timekeeper::sptr> timekeepers;
+    timekeepers.reserve(mb_controllers.size());
+    for (auto& mbc : mb_controllers) {
+        // If we also want to sync other timekeepers, this would be the place to
+        // do that
+        timekeepers.push_back(mbc->get_timekeeper(0));
+    }
+
+    if (!quiet) {
+        UHD_LOGGER_INFO("MB_CTRL") << "    1) catch time transition at pps edge";
+    }
+    const auto end_time                   = std::chrono::steady_clock::now() + 1100ms;
+    const time_spec_t time_start_last_pps = timekeepers.front()->get_time_last_pps();
+    while (time_start_last_pps == timekeepers.front()->get_time_last_pps()) {
+        if (std::chrono::steady_clock::now() > end_time) {
+            // This is always bad, and we'll throw regardless of quiet
+            throw uhd::runtime_error("Board 0 may not be getting a PPS signal!\n"
+                                     "No PPS detected within the time interval.\n"
+                                     "See the application notes for your device.\n");
+        }
+        std::this_thread::sleep_for(1ms);
+    }
+
+    if (!quiet) {
+        UHD_LOGGER_INFO("MB_CTRL") << "    2) set times next pps (synchronously)";
+    }
+
+    for (auto& timekeeper : timekeepers) {
+        timekeeper->set_time_next_pps(time_spec);
+    }
+    std::this_thread::sleep_for(1s);
+
+    // verify that the time registers are read to be within a few RTT
+    size_t m = 0;
+    for (auto& timekeeper : timekeepers) {
+        time_spec_t time_0 = timekeepers.front()->get_time_now();
+        time_spec_t time_i = timekeeper->get_time_now();
+        // 10 ms: greater than RTT but not too big
+        constexpr double MAX_DEVIATION = 0.01;
+        if (time_i < time_0 or (time_i - time_0) > time_spec_t(MAX_DEVIATION)) {
+            if (!quiet) {
+                UHD_LOGGER_WARNING("MULTI_USRP")
+                    << boost::format(
+                           "Detected time deviation between board %d and board 0.\n"
+                           "Board 0 time is %f seconds.\n"
+                           "Board %d time is %f seconds.\n")
+                           % m % time_0.get_real_secs() % m % time_i.get_real_secs();
+            }
+            return false;
+        }
+        m++;
+    }
+    return true;
+}
 
 /******************************************************************************
  * Timekeeper API
@@ -71,3 +169,24 @@ void mb_controller::register_timekeeper(const size_t idx, timekeeper::sptr tk)
 {
     _timekeepers.emplace(idx, std::move(tk));
 }
+
+std::vector<std::string> mb_controller::get_gpio_banks() const
+{
+    return {};
+}
+
+std::vector<std::string> mb_controller::get_gpio_srcs(const std::string&) const
+{
+    throw uhd::not_implemented_error(
+        "get_gpio_srcs() not supported on this motherboard!");
+}
+
+std::vector<std::string> mb_controller::get_gpio_src(const std::string&)
+{
+    throw uhd::not_implemented_error("get_gpio_src() not supported on this motherboard!");
+}
+
+void mb_controller::set_gpio_src(const std::string&, const std::vector<std::string>&)
+{
+    throw uhd::not_implemented_error("set_gpio_src() not supported on this motherboard!");
+}
diff --git a/host/lib/rfnoc/rfnoc_graph.cpp b/host/lib/rfnoc/rfnoc_graph.cpp
index 12f1cb2c4..85e403cbe 100644
--- a/host/lib/rfnoc/rfnoc_graph.cpp
+++ b/host/lib/rfnoc/rfnoc_graph.cpp
@@ -27,6 +27,14 @@ using namespace uhd::rfnoc;
 
 namespace {
 const std::string LOG_ID("RFNOC::GRAPH");
+
+//! Which blocks are actually stored at a given port on the crossbar
+struct block_xbar_info
+{
+    size_t xbar_port;
+    noc_id_t noc_id;
+    size_t inst_num;
+};
 }
 
 class rfnoc_graph_impl : public rfnoc_graph
@@ -41,6 +49,7 @@ public:
           _num_mboards(_tree->list("/mboards").size()),
           _block_registry(std::make_unique<detail::block_container_t>()),
           _graph(std::make_unique<uhd::rfnoc::detail::graph_t>()) {
+        _mb_controllers.reserve(_num_mboards);
         // Now initialize all subsystems:
         _init_mb_controllers();
         _init_gsm(); // Graph Stream Manager
@@ -59,6 +68,9 @@ public:
         _block_registry->init_props();
         _init_sep_map();
         _init_static_connections();
+        _init_mbc();
+        // Start with time set to zero, but don't complain if sync fails
+        rfnoc_graph_impl::synchronize_devices(uhd::time_spec_t(0.0), true);
     } catch (const std::exception& ex) {
         UHD_LOG_ERROR(LOG_ID, "Caught exception while initializing graph: " << ex.what());
         throw uhd::runtime_error("Failure to create rfnoc_graph.");
@@ -319,9 +331,14 @@ public:
         return boost::make_shared<rfnoc_tx_streamer>(num_chans, args);
     }
 
+    size_t get_num_mboards() const
+    {
+        return _num_mboards;
+    }
+
     std::shared_ptr<mb_controller> get_mb_controller(const size_t mb_index = 0)
     {
-        if (!_mb_controllers.count(mb_index)) {
+        if (_mb_controllers.size() <= mb_index) {
             throw uhd::index_error(
                 std::string("Could not get mb controller for motherboard index ")
                 + std::to_string(mb_index));
@@ -329,10 +346,23 @@ public:
         return _mb_controllers.at(mb_index);
     }
 
+    bool synchronize_devices(const uhd::time_spec_t& time_spec, const bool quiet)
+    {
+        auto mb_controllers_copy = _mb_controllers;
+        bool result =
+            _mb_controllers.at(0)->synchronize(mb_controllers_copy, time_spec, quiet);
+        if (mb_controllers_copy.size() != _mb_controllers.size()) {
+            // This shouldn't happen until we allow different device types in a
+            // rfnoc_graph
+            UHD_LOG_ERROR(LOG_ID, "Some devices wouldn't be sync'd!");
+            return false;
+        }
+        return result;
+    }
 
-    size_t get_num_mboards() const
+    uhd::property_tree::sptr get_tree(void) const
     {
-        return _num_mboards;
+        return _tree;
     }
 
     std::vector<uhd::transport::adapter_id_t> enumerate_adapters_to_dst(
@@ -430,7 +460,7 @@ private:
     {
         UHD_LOG_TRACE(LOG_ID, "Initializing MB controllers...");
         for (size_t i = 0; i < _num_mboards; ++i) {
-            _mb_controllers.emplace(i, _device->get_mb_controller(i));
+            _mb_controllers.push_back(_device->get_mb_controller(i));
         }
     }
 
@@ -625,6 +655,15 @@ private:
         }
     }
 
+    //! Initialize the motherboard controllers, if they require it
+    void _init_mbc()
+    {
+        for (size_t i = 0; i < _mb_controllers.size(); ++i) {
+            UHD_LOG_TRACE(LOG_ID, "Calling MBC init for motherboard " << i);
+            _mb_controllers.at(i)->init();
+        }
+    }
+
     /**************************************************************************
      * Helpers
      *************************************************************************/
@@ -811,7 +850,7 @@ private:
     std::unique_ptr<detail::graph_t> _graph;
 
     //! Stash a list of motherboard controllers
-    std::unordered_map<size_t, mb_controller::sptr> _mb_controllers;
+    std::vector<mb_controller::sptr> _mb_controllers;
 
     //! Stash of the client zeros for all motherboards
     std::unordered_map<size_t, detail::client_zero::sptr> _client_zeros;