X300: Add retry to DAC synchronization

Reviewed-by: Martin Braun <martin.braun@ettus.com>
Reviewed-by: Ashish Chaudhari <ashish.chaudhari@ettus.com>

3 files changed, 90 insertions, 38 deletions

diff --git a/host/lib/usrp/x300/x300_dac_ctrl.cpp b/host/lib/usrp/x300/x300_dac_ctrl.cpp
index 162eeb143..597d6351a 100644
--- a/host/lib/usrp/x300/x300_dac_ctrl.cpp
+++ b/host/lib/usrp/x300/x300_dac_ctrl.cpp
@@ -80,6 +80,33 @@ public:
         _sleep_mode(false);
     }
 
+    void sync()
+    {
+        try {
+            // Just return if PLL is locked and backend is synchronized
+            _check_pll();
+            _check_dac_sync();
+            return;
+        } catch (...) {}
+
+        std::string err_str;
+
+        // Try 3 times to sync before giving up
+        for (size_t retries = 0; retries < 3; retries++)
+        {
+            try {
+                _sleep_mode(true);
+                _init();
+                _backend_sync();
+                _sleep_mode(false);
+                return;
+            } catch (const uhd::runtime_error &e) {
+                err_str = e.what();
+            }
+        }
+        throw uhd::runtime_error(err_str);
+    }
+
     void verify_sync()
     {
         _check_pll();
@@ -148,7 +175,6 @@ public:
     void _backend_sync(void)
     {
         write_ad9146_reg(0x10, 0x40);   // Disable SYNC mode to reset state machines.
-        write_ad9146_reg(0x06, 0x30);   // Clear Sync event flags
 
         //SYNC Settings:
         //- SYNC = Enabled
@@ -176,13 +202,10 @@ public:
         write_ad9146_reg(0x17, 0x05);
 
         // We are requesting a soft FIFO align just to put the FIFO
-        // in a known state. The FRAME will actually do sync the
+        // in a known state. The FRAME will actually sync the
         // FIFO correctly when a stream is created
         write_ad9146_reg(0x18, 0x02); // Request soft FIFO align
         write_ad9146_reg(0x18, 0x00); // (See above)
-
-        //Verify the FIFO thermometer
-        _check_frontend_sync(false); //FIFO sanity check
     }
 
     //
@@ -190,6 +213,9 @@ public:
     //
     void _check_pll()
     {
+        //Clear PLL event flags
+        write_ad9146_reg(0x06, 0xC0);
+
         // Verify PLL is Locked. 1 sec timeout.
         // NOTE: Data sheet inconsistent about which pins give PLL lock status. FIXME!
         const time_spec_t exit_time = time_spec_t::get_system_time() + time_spec_t(1.0);
@@ -199,7 +225,7 @@ public:
             const size_t reg_6 = read_ad9146_reg(0x06); // Event Flags (Expect bit 7 = 0 and bit 6 = 1)
             if ((((reg_e >> 7) & 0x1) == 0x1) && (((reg_6 >> 6) & 0x3) == 0x1))
                 break;
-            if (exit_time < time_spec_t::get_system_time())
+            if (time_spec_t::get_system_time() > exit_time)
                 throw uhd::runtime_error("x300_dac_ctrl: timeout waiting for DAC PLL to lock");
             if (reg_6 & (1 << 7))               // Lock lost?
                 write_ad9146_reg(0x06, 0xC0);   // Clear PLL event flags
@@ -212,6 +238,10 @@ public:
     //
     void _check_dac_sync()
     {
+        // Clear Sync event flags
+        write_ad9146_reg(0x06, 0x30);
+        write_ad9146_reg(0x12, 0x00);
+
         const time_spec_t exit_time = time_spec_t::get_system_time() + time_spec_t(1.0);
         while (true)
         {
@@ -220,13 +250,15 @@ public:
             const size_t reg_6 = read_ad9146_reg(0x06);     // Event Flags (Expect bit 5 = 0 and bit 4 = 1)
             if ((((reg_12 >> 6) & 0x3) == 0x1) && (((reg_6 >> 4) & 0x3) == 0x1))
                 break;
-            if (exit_time < time_spec_t::get_system_time())
+            if (time_spec_t::get_system_time() > exit_time)
                 throw uhd::runtime_error("x300_dac_ctrl: timeout waiting for backend synchronization");
             if (reg_6 & (1 << 5))
                 write_ad9146_reg(0x06, 0x30);   // Clear Sync event flags
 #ifdef X300_DAC_RETRY_BACKEND_SYNC
-            if (reg_12 & (1 << 7))              // Sync acquired and lost?
+            if (reg_12 & (1 << 7)) {            // Sync acquired and lost?
                 write_ad9146_reg(0x10, 0xC7);   // Enable SYNC mode. Falling edge sync. Averaging set to 128.
+                write_ad9146_reg(0x12, 0x00);   // Clear Sync event flags
+            }
 #endif
         }
     }
@@ -239,7 +271,7 @@ public:
         // Register 0x19 has a thermometer indicator of the FIFO depth
         const size_t reg_19 = read_ad9146_reg(0x19);
         if ((reg_19 & 0xFF) != 0xF) {
-            std::string msg((boost::format("x300_dac_ctrl: front-end sync failed. unexpected FIFO depth [0x%x]\n") % (reg_19 & 0xFF)).str());
+            std::string msg((boost::format("x300_dac_ctrl: front-end sync failed. unexpected FIFO depth [0x%x]") % (reg_19 & 0xFF)).str());
             if (failure_is_fatal) {
                 throw uhd::runtime_error(msg);
             } else {
diff --git a/host/lib/usrp/x300/x300_dac_ctrl.hpp b/host/lib/usrp/x300/x300_dac_ctrl.hpp
index ca47a90e7..d208f2c2e 100644
--- a/host/lib/usrp/x300/x300_dac_ctrl.hpp
+++ b/host/lib/usrp/x300/x300_dac_ctrl.hpp
@@ -40,6 +40,9 @@ public:
     // ! Reset the DAC
     virtual void reset(void) = 0;
 
+    // ! Sync the DAC
+    virtual void sync(void) = 0;
+
     // ! Check for successful backend and frontend sync
     virtual void verify_sync(void) = 0;
 };
diff --git a/host/lib/usrp/x300/x300_radio_ctrl_impl.cpp b/host/lib/usrp/x300/x300_radio_ctrl_impl.cpp
index a462ceea4..c7ff99066 100644
--- a/host/lib/usrp/x300/x300_radio_ctrl_impl.cpp
+++ b/host/lib/usrp/x300/x300_radio_ctrl_impl.cpp
@@ -595,11 +595,6 @@ void x300_radio_ctrl_impl::synchronize_dacs(const std::vector<x300_radio_ctrl_im
     //to a common reference. Currently, this function is called in get_tx_stream
     //which also has the same precondition.
 
-    //Reinitialize and resync all DACs
-    for (size_t i = 0; i < radios.size(); i++) {
-        radios[i]->_dac->reset();
-    }
-
     //Get a rough estimate of the cumulative command latency
     boost::posix_time::ptime t_start = boost::posix_time::microsec_clock::local_time();
     for (size_t i = 0; i < radios.size(); i++) {
@@ -608,37 +603,59 @@ void x300_radio_ctrl_impl::synchronize_dacs(const std::vector<x300_radio_ctrl_im
     boost::posix_time::time_duration t_elapsed =
         boost::posix_time::microsec_clock::local_time() - t_start;
 
-    //Set tick rate and make sure FRAMEP/N is 0
-    for (size_t i = 0; i < radios.size(); i++) {
-        radios[i]->set_command_tick_rate(radios[i]->_radio_clk_rate, IO_MASTER_RADIO);
-        radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 0);
-    }
-
     //Add 100% of headroom + uncertainty to the command time
     uint64_t t_sync_us = (t_elapsed.total_microseconds() * 2) + 16000 /*Scheduler latency*/;
 
-    //Pick radios[0] as the time reference.
-    uhd::time_spec_t sync_time =
-        radios[0]->_time64->get_time_now() + uhd::time_spec_t(((double)t_sync_us)/1e6);
+    std::string err_str;
+    //Try to sync 3 times before giving up
+    for (size_t attempt = 0; attempt < 3; attempt++)
+    {
+        try
+        {
+            //Reinitialize and resync all DACs
+            for (size_t i = 0; i < radios.size(); i++) {
+                radios[i]->_dac->sync();
+            }
 
-    //Send the sync command
-    for (size_t i = 0; i < radios.size(); i++) {
-        radios[i]->set_command_time(sync_time, IO_MASTER_RADIO);
-        //Arm FRAMEP/N sync pulse by asserting a rising edge
-        radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 1);
-        radios[i]->set_command_time(uhd::time_spec_t(0.0), IO_MASTER_RADIO);
-    }
+            //Set tick rate and make sure FRAMEP/N is 0
+            for (size_t i = 0; i < radios.size(); i++) {
+                radios[i]->set_command_tick_rate(radios[i]->_radio_clk_rate, IO_MASTER_RADIO);
+                radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 0);
+            }
 
-    //Reset FRAMEP/N to 0
-    for (size_t i = 0; i < radios.size(); i++) {
-        radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 0);
-    }
+            //Pick radios[0] as the time reference.
+            uhd::time_spec_t sync_time =
+                radios[0]->_time64->get_time_now() + uhd::time_spec_t(((double)t_sync_us)/1e6);
 
-    //Wait and check status
-    boost::this_thread::sleep(boost::posix_time::microseconds(t_sync_us));
-    for (size_t i = 0; i < radios.size(); i++) {
-        radios[i]->_dac->verify_sync();
+            //Send the sync command
+            for (size_t i = 0; i < radios.size(); i++) {
+                radios[i]->set_command_time(sync_time, IO_MASTER_RADIO);
+                //Arm FRAMEP/N sync pulse by asserting a rising edge
+                radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 1);
+            }
+
+            //Reset FRAMEP/N to 0 after 2 clock cycles
+            for (size_t i = 0; i < radios.size(); i++) {
+                radios[i]->set_command_time(sync_time + (2.0 / radios[i]->_radio_clk_rate), IO_MASTER_RADIO);
+                radios[i]->_regs->misc_outs_reg.write(radio_regmap_t::misc_outs_reg_t::DAC_SYNC, 0);
+                radios[i]->set_command_time(uhd::time_spec_t(0.0), IO_MASTER_RADIO);
+            }
+
+            //Wait and check status
+            boost::this_thread::sleep(boost::posix_time::microseconds(t_sync_us));
+            for (size_t i = 0; i < radios.size(); i++) {
+                radios[i]->_dac->verify_sync();
+            }
+
+            return;
+        }
+        catch (const uhd::runtime_error &e)
+        {
+            err_str = e.what();
+            UHD_LOGV(rarely) << "X300 RADIO: Retrying DAC synchronization: " << err_str;
+        }
     }
+    throw uhd::runtime_error(err_str);
 }
 
 double x300_radio_ctrl_impl::self_cal_adc_xfer_delay(