uhd: fixed RX and TX DSP cores for 3rd generation products

2 files changed, 52 insertions, 8 deletions

diff --git a/host/lib/usrp/cores/rx_dsp_core_3000.cpp b/host/lib/usrp/cores/rx_dsp_core_3000.cpp
index fc7c58da8..6ce3c1d32 100644
--- a/host/lib/usrp/cores/rx_dsp_core_3000.cpp
+++ b/host/lib/usrp/cores/rx_dsp_core_3000.cpp
@@ -197,14 +197,36 @@ public:
         if (std::abs(freq) > _tick_rate/2.0)
             freq -= boost::math::sign(freq)*_tick_rate;
 
-        //calculate the freq register word (signed)
+        //confirm that the target frequency is within range of the CORDIC
         UHD_ASSERT_THROW(std::abs(freq) <= _tick_rate/2.0);
+
+        /* Now calculate the frequency word. It is possible for this calculation
+         * to cause an overflow. As the requested DSP frequency approaches the
+         * master clock rate, that ratio multiplied by the scaling factor (2^32)
+         * will generally overflow within the last few kHz of tunable range.
+         * Thus, we check to see if the operation will overflow before doing it,
+         * and if it will, we set it to the integer min or max of this system.
+         */
+        boost::int32_t freq_word = 0;
+
         static const double scale_factor = std::pow(2.0, 32);
-        const boost::int32_t freq_word = boost::int32_t(boost::math::round((freq / _tick_rate) * scale_factor));
+        static const boost::int32_t int_max = boost::numeric::bounds<boost::int32_t>::highest();
+        static const boost::int32_t int_min = boost::numeric::bounds<boost::int32_t>::lowest();
+        if((freq / _tick_rate) >= (int_max / scale_factor)) {
+            /* Operation would have caused a positive overflow of int32. */
+            freq_word = boost::numeric::bounds<boost::int32_t>::highest();
 
-        //update the actual frequency
-        const double actual_freq = (double(freq_word) / scale_factor) * _tick_rate;
+        } else if((freq / _tick_rate) <= (int_min / scale_factor)) {
+            /* Operation would have caused a negative overflow of int32. */
+            freq_word = boost::numeric::bounds<boost::int32_t>::lowest();
 
+        } else {
+            /* The operation is safe. Perform normally. */
+            freq_word = boost::int32_t(boost::math::round((freq / _tick_rate) * scale_factor));
+        }
+
+        //program the frequency word into the device DSP
+        const double actual_freq = (double(freq_word) / scale_factor) * _tick_rate;
         _iface->poke32(REG_DSP_RX_FREQ, boost::uint32_t(freq_word));
 
         return actual_freq;
diff --git a/host/lib/usrp/cores/tx_dsp_core_3000.cpp b/host/lib/usrp/cores/tx_dsp_core_3000.cpp
index 9c2b98ffb..0dc19f2c8 100644
--- a/host/lib/usrp/cores/tx_dsp_core_3000.cpp
+++ b/host/lib/usrp/cores/tx_dsp_core_3000.cpp
@@ -134,14 +134,36 @@ public:
         if (std::abs(freq) > _tick_rate/2.0)
             freq -= boost::math::sign(freq)*_tick_rate;
 
-        //calculate the freq register word (signed)
+        //confirm that the target frequency is within range of the CORDIC
         UHD_ASSERT_THROW(std::abs(freq) <= _tick_rate/2.0);
+
+        /* Now calculate the frequency word. It is possible for this calculation
+         * to cause an overflow. As the requested DSP frequency approaches the
+         * master clock rate, that ratio multiplied by the scaling factor (2^32)
+         * will generally overflow within the last few kHz of tunable range.
+         * Thus, we check to see if the operation will overflow before doing it,
+         * and if it will, we set it to the integer min or max of this system.
+         */
+        boost::int32_t freq_word = 0;
+
         static const double scale_factor = std::pow(2.0, 32);
-        const boost::int32_t freq_word = boost::int32_t(boost::math::round((freq / _tick_rate) * scale_factor));
+        static const boost::int32_t int_max = boost::numeric::bounds<boost::int32_t>::highest();
+        static const boost::int32_t int_min = boost::numeric::bounds<boost::int32_t>::lowest();
+        if((freq / _tick_rate) >= (int_max / scale_factor)) {
+            /* Operation would have caused a positive overflow of int32. */
+            freq_word = boost::numeric::bounds<boost::int32_t>::highest();
+
+        } else if((freq / _tick_rate) <= (int_min / scale_factor)) {
+            /* Operation would have caused a negative overflow of int32. */
+            freq_word = boost::numeric::bounds<boost::int32_t>::lowest();
+
+        } else {
+            /* The operation is safe. Perform normally. */
+            freq_word = boost::int32_t(boost::math::round((freq / _tick_rate) * scale_factor));
+        }
 
-        //update the actual frequency
+        //program the frequency word into the device DSP
         const double actual_freq = (double(freq_word) / scale_factor) * _tick_rate;
-
         _iface->poke32(REG_DSP_TX_FREQ, boost::uint32_t(freq_word));
 
         return actual_freq;