rfnoc: Allow MB controllers to init after blocks have initialized

This allows mb_controller childs to implement an init() call which
rfnoc_graph will call after the block initialization is complete.

rfnoc: graph/mb_controller: Add synchronization routine

This adds two new API calls:
* rfnoc_graph::synchronize_devices() and
* mb_controller::synchronize().

The purpose is to synchronize devices in time and/or phase, depending on
device capabilities. mb_controller childs can override or extend the
default implementation, which is to simply set time next PPS and verify
(similar to the set_time_unknown_pps() call in multi_usrp).

rfnoc: mb_controller: Add gpio_src API

Adds new API calls (get_gpio_src, get_gpio_srcs, set_gpio_src,
get_gpio_banks) to mb_controllers

4 files changed, 240 insertions, 16 deletions

diff --git a/host/include/uhd/rfnoc/mb_controller.hpp b/host/include/uhd/rfnoc/mb_controller.hpp
index a1c0f21aa..0269301ff 100644
--- a/host/include/uhd/rfnoc/mb_controller.hpp
+++ b/host/include/uhd/rfnoc/mb_controller.hpp
@@ -135,6 +135,10 @@ public:
     /**************************************************************************
      * Motherboard Control
      *************************************************************************/
+    /*! Run initializations of this motherboard that have to occur post-block init
+     */
+    virtual void init() {}
+
     /*! Get canonical name for this USRP motherboard
      *
      * \return a string representing the name
@@ -333,6 +337,51 @@ public:
      */
     virtual uhd::usrp::mboard_eeprom_t get_eeprom() = 0;
 
+    /*! Synchronize a list of motherboards
+     *
+     * \param mb_controllers A list of motherboard controllers to synchronize.
+     *                       Any motherboard controllers that could not be
+     *                       synchronized because they're incompatible with this
+     *                       motherboard controller are removed from the list.
+     *                       On return, the list should be (ideally) identical
+     *                       to its value at call time.
+     * \param quiet If true, don't print any errors or warnings if
+     *              synchronization fails.
+     * \returns true if all motherboards that were removed from \p mb_controllers
+     *          could be synchronized.
+     */
+    virtual bool synchronize(std::vector<mb_controller::sptr>& mb_controllers,
+        const uhd::time_spec_t& time_spec = uhd::time_spec_t(0.0),
+        const bool quiet                  = false);
+
+    /*! Return the list of GPIO banks that are controlled by this MB controller
+     *
+     * Note that this list may be empty. Only if the MB controller has any
+     * control over GPIOs, do the get listed here.
+     */
+    virtual std::vector<std::string> get_gpio_banks() const;
+
+    /*! Return a list of possible sources to drive GPIOs
+     *
+     * Sources can be "PS", for when an embedded device can drive the pins from
+     * software, "Radio#0", if a radio block can drive them, and so on.
+     */
+    virtual std::vector<std::string> get_gpio_srcs(const std::string& bank) const;
+
+    /*! Return the current sources for a given GPIO bank
+     */
+    virtual std::vector<std::string> get_gpio_src(const std::string& bank);
+
+    /*! Set the source for GPIO pins on a given bank.
+     *
+     * \throws uhd::key_error if the bank does not exist
+     * \throws uhd::value_error if the source does not exist
+     * \throws uhd::not_implemented_error if the current motherboard does not
+     *         support this feature
+     */
+    virtual void set_gpio_src(
+        const std::string& bank, const std::vector<std::string>& src);
+
 protected:
     /*! Stash away a timekeeper. This needs to be called by the implementer of
      * mb_controller.
diff --git a/host/include/uhd/rfnoc_graph.hpp b/host/include/uhd/rfnoc_graph.hpp
index 8a56d1c80..34091ddbd 100644
--- a/host/include/uhd/rfnoc_graph.hpp
+++ b/host/include/uhd/rfnoc_graph.hpp
@@ -14,6 +14,7 @@
 #include <uhd/stream.hpp>
 #include <uhd/transport/adapter_id.hpp>
 #include <uhd/types/device_addr.hpp>
+#include <uhd/types/time_spec.hpp>
 #include <uhd/utils/noncopyable.hpp>
 #include <boost/units/detail/utility.hpp> // for demangle
 #include <memory>
@@ -39,16 +40,6 @@ public:
 
     virtual ~rfnoc_graph() {}
 
-
-    //! Stuct to store information about which blocks are actually stored at a given port
-    //! on the crossbar
-    struct block_xbar_info
-    {
-        size_t xbar_port;
-        noc_id_t noc_id;
-        size_t inst_num;
-    };
-
     /******************************************
      * Factory
      ******************************************/
@@ -316,6 +307,32 @@ public:
     // See also uhd::rfnoc::mb_controller
     virtual std::shared_ptr<mb_controller> get_mb_controller(
         const size_t mb_index = 0) = 0;
+
+    /*! Run any routines necessary to synchronize devices
+     *
+     * The specific implementation of this call are device-specific. In all
+     * cases, it will set the time to a common value.
+     *
+     * Any application that requires any kind of phase or time alignment (if
+     * supported by the hardware) must call this before operation.
+     *
+     * \param time_spec The timestamp to be used to sync the devices. It will be
+     *                  an input to set_time_next_pps() on the motherboard
+     *                  controllers.
+     * \param quiet If true, there will be no errors or warnings printed if the
+     *              synchronization happens. This call will always be called
+     *              during initialization, but preconditions might not yet be
+     *              met (e.g., the time and reference sources might still be
+     *              internal), and will fail quietly in that case.
+     *
+     * \returns the success status of this call (true means devices are now
+     *          synchronized)
+     */
+    virtual bool synchronize_devices(
+        const uhd::time_spec_t& time_spec, const bool quiet) = 0;
+
+    //! Return a reference to the property tree
+    virtual uhd::property_tree::sptr get_tree(void) const = 0;
 }; // class rfnoc_graph
 
 }}; // namespace uhd::rfnoc
diff --git a/host/lib/rfnoc/mb_controller.cpp b/host/lib/rfnoc/mb_controller.cpp
index 10d5ebe47..62ec1f886 100644
--- a/host/lib/rfnoc/mb_controller.cpp
+++ b/host/lib/rfnoc/mb_controller.cpp
@@ -4,12 +4,110 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 //
 
-#include <uhd/rfnoc/mb_controller.hpp>
 #include <uhd/exception.hpp>
+#include <uhd/rfnoc/mb_controller.hpp>
+#include <uhd/utils/algorithm.hpp>
+#include <uhd/utils/log.hpp>
 #include <atomic>
+#include <chrono>
+#include <thread>
 
 using namespace uhd::rfnoc;
+using namespace std::chrono_literals;
 
+namespace {
+const std::vector<std::string> SYNCHRONIZABLE_REF_SOURCES = {"gpsdo", "external"};
+}
+
+bool mb_controller::synchronize(std::vector<mb_controller::sptr>& mb_controllers,
+    const uhd::time_spec_t& time_spec,
+    const bool quiet)
+{
+    if (mb_controllers.empty()) {
+        return false;
+    }
+    if (mb_controllers.size() == 1) {
+        UHD_LOG_TRACE("MB_CTRL", "Skipping time synchronization of a single USRP.");
+        mb_controllers.at(0)->get_timekeeper(0)->set_time_now(time_spec);
+        return true;
+    }
+    // Verify that all devices share a time reference, and that it is a common
+    // one
+    const std::string time_source = mb_controllers.at(0)->get_time_source();
+    if (!uhd::has(SYNCHRONIZABLE_REF_SOURCES, time_source)) {
+        if (!quiet) {
+            UHD_LOG_WARNING("MB_CTRL",
+                "The selected time source "
+                    << time_source << " does not allow synchronization between devices.");
+        }
+        return false;
+    }
+    for (auto& mbc : mb_controllers) {
+        if (mbc->get_time_source() != time_source) {
+            if (!quiet) {
+                UHD_LOG_WARNING("MB_CTRL",
+                    "Motherboards do not share a time source, and thus cannot be "
+                    "synchronized!");
+            }
+            return false;
+        }
+    }
+
+    // Get a reference to all timekeepers
+    std::vector<timekeeper::sptr> timekeepers;
+    timekeepers.reserve(mb_controllers.size());
+    for (auto& mbc : mb_controllers) {
+        // If we also want to sync other timekeepers, this would be the place to
+        // do that
+        timekeepers.push_back(mbc->get_timekeeper(0));
+    }
+
+    if (!quiet) {
+        UHD_LOGGER_INFO("MB_CTRL") << "    1) catch time transition at pps edge";
+    }
+    const auto end_time                   = std::chrono::steady_clock::now() + 1100ms;
+    const time_spec_t time_start_last_pps = timekeepers.front()->get_time_last_pps();
+    while (time_start_last_pps == timekeepers.front()->get_time_last_pps()) {
+        if (std::chrono::steady_clock::now() > end_time) {
+            // This is always bad, and we'll throw regardless of quiet
+            throw uhd::runtime_error("Board 0 may not be getting a PPS signal!\n"
+                                     "No PPS detected within the time interval.\n"
+                                     "See the application notes for your device.\n");
+        }
+        std::this_thread::sleep_for(1ms);
+    }
+
+    if (!quiet) {
+        UHD_LOGGER_INFO("MB_CTRL") << "    2) set times next pps (synchronously)";
+    }
+
+    for (auto& timekeeper : timekeepers) {
+        timekeeper->set_time_next_pps(time_spec);
+    }
+    std::this_thread::sleep_for(1s);
+
+    // verify that the time registers are read to be within a few RTT
+    size_t m = 0;
+    for (auto& timekeeper : timekeepers) {
+        time_spec_t time_0 = timekeepers.front()->get_time_now();
+        time_spec_t time_i = timekeeper->get_time_now();
+        // 10 ms: greater than RTT but not too big
+        constexpr double MAX_DEVIATION = 0.01;
+        if (time_i < time_0 or (time_i - time_0) > time_spec_t(MAX_DEVIATION)) {
+            if (!quiet) {
+                UHD_LOGGER_WARNING("MULTI_USRP")
+                    << boost::format(
+                           "Detected time deviation between board %d and board 0.\n"
+                           "Board 0 time is %f seconds.\n"
+                           "Board %d time is %f seconds.\n")
+                           % m % time_0.get_real_secs() % m % time_i.get_real_secs();
+            }
+            return false;
+        }
+        m++;
+    }
+    return true;
+}
 
 /******************************************************************************
  * Timekeeper API
@@ -71,3 +169,24 @@ void mb_controller::register_timekeeper(const size_t idx, timekeeper::sptr tk)
 {
     _timekeepers.emplace(idx, std::move(tk));
 }
+
+std::vector<std::string> mb_controller::get_gpio_banks() const
+{
+    return {};
+}
+
+std::vector<std::string> mb_controller::get_gpio_srcs(const std::string&) const
+{
+    throw uhd::not_implemented_error(
+        "get_gpio_srcs() not supported on this motherboard!");
+}
+
+std::vector<std::string> mb_controller::get_gpio_src(const std::string&)
+{
+    throw uhd::not_implemented_error("get_gpio_src() not supported on this motherboard!");
+}
+
+void mb_controller::set_gpio_src(const std::string&, const std::vector<std::string>&)
+{
+    throw uhd::not_implemented_error("set_gpio_src() not supported on this motherboard!");
+}
diff --git a/host/lib/rfnoc/rfnoc_graph.cpp b/host/lib/rfnoc/rfnoc_graph.cpp
index 12f1cb2c4..85e403cbe 100644
--- a/host/lib/rfnoc/rfnoc_graph.cpp
+++ b/host/lib/rfnoc/rfnoc_graph.cpp
@@ -27,6 +27,14 @@ using namespace uhd::rfnoc;
 
 namespace {
 const std::string LOG_ID("RFNOC::GRAPH");
+
+//! Which blocks are actually stored at a given port on the crossbar
+struct block_xbar_info
+{
+    size_t xbar_port;
+    noc_id_t noc_id;
+    size_t inst_num;
+};
 }
 
 class rfnoc_graph_impl : public rfnoc_graph
@@ -41,6 +49,7 @@ public:
           _num_mboards(_tree->list("/mboards").size()),
           _block_registry(std::make_unique<detail::block_container_t>()),
           _graph(std::make_unique<uhd::rfnoc::detail::graph_t>()) {
+        _mb_controllers.reserve(_num_mboards);
         // Now initialize all subsystems:
         _init_mb_controllers();
         _init_gsm(); // Graph Stream Manager
@@ -59,6 +68,9 @@ public:
         _block_registry->init_props();
         _init_sep_map();
         _init_static_connections();
+        _init_mbc();
+        // Start with time set to zero, but don't complain if sync fails
+        rfnoc_graph_impl::synchronize_devices(uhd::time_spec_t(0.0), true);
     } catch (const std::exception& ex) {
         UHD_LOG_ERROR(LOG_ID, "Caught exception while initializing graph: " << ex.what());
         throw uhd::runtime_error("Failure to create rfnoc_graph.");
@@ -319,9 +331,14 @@ public:
         return boost::make_shared<rfnoc_tx_streamer>(num_chans, args);
     }
 
+    size_t get_num_mboards() const
+    {
+        return _num_mboards;
+    }
+
     std::shared_ptr<mb_controller> get_mb_controller(const size_t mb_index = 0)
     {
-        if (!_mb_controllers.count(mb_index)) {
+        if (_mb_controllers.size() <= mb_index) {
             throw uhd::index_error(
                 std::string("Could not get mb controller for motherboard index ")
                 + std::to_string(mb_index));
@@ -329,10 +346,23 @@ public:
         return _mb_controllers.at(mb_index);
     }
 
+    bool synchronize_devices(const uhd::time_spec_t& time_spec, const bool quiet)
+    {
+        auto mb_controllers_copy = _mb_controllers;
+        bool result =
+            _mb_controllers.at(0)->synchronize(mb_controllers_copy, time_spec, quiet);
+        if (mb_controllers_copy.size() != _mb_controllers.size()) {
+            // This shouldn't happen until we allow different device types in a
+            // rfnoc_graph
+            UHD_LOG_ERROR(LOG_ID, "Some devices wouldn't be sync'd!");
+            return false;
+        }
+        return result;
+    }
 
-    size_t get_num_mboards() const
+    uhd::property_tree::sptr get_tree(void) const
     {
-        return _num_mboards;
+        return _tree;
     }
 
     std::vector<uhd::transport::adapter_id_t> enumerate_adapters_to_dst(
@@ -430,7 +460,7 @@ private:
     {
         UHD_LOG_TRACE(LOG_ID, "Initializing MB controllers...");
         for (size_t i = 0; i < _num_mboards; ++i) {
-            _mb_controllers.emplace(i, _device->get_mb_controller(i));
+            _mb_controllers.push_back(_device->get_mb_controller(i));
         }
     }
 
@@ -625,6 +655,15 @@ private:
         }
     }
 
+    //! Initialize the motherboard controllers, if they require it
+    void _init_mbc()
+    {
+        for (size_t i = 0; i < _mb_controllers.size(); ++i) {
+            UHD_LOG_TRACE(LOG_ID, "Calling MBC init for motherboard " << i);
+            _mb_controllers.at(i)->init();
+        }
+    }
+
     /**************************************************************************
      * Helpers
      *************************************************************************/
@@ -811,7 +850,7 @@ private:
     std::unique_ptr<detail::graph_t> _graph;
 
     //! Stash a list of motherboard controllers
-    std::unordered_map<size_t, mb_controller::sptr> _mb_controllers;
+    std::vector<mb_controller::sptr> _mb_controllers;
 
     //! Stash of the client zeros for all motherboards
     std::unordered_map<size_t, detail::client_zero::sptr> _client_zeros;