rfnoc: Clarify usage of MTU vs. max payload size, remove DEFAULT_SPP

These two values where being mixed up in the code. To summarize:
- The MTU is the max CHDR packet size, including header & timestamp.
- The max payload is the total number of bytes regular payload plus
  metadata that can be fit into into a CHDR packet. It is strictly
  smaller than the MTU. For example, for 64-bit CHDR widths, if
  a timestamp is desired, the max payload is 16 bytes smaller than
  the MTU.

The other issue was that we were using a magic constant (DEFAULT_SPP)
which was causing conflicts with MTUs and max payloads.
This constant was harmful in multiple ways:
- The explanatory comment was incorrect (it stated it would cap packets
  to 1500 bytes, which it didn't)
- It imposed random, hardcoded values that interfered with an 'spp
  discovery', i.e., the ability to derive a good spp value from MTUs
- The current value capped packet sizes to 8000 bytes CHDR packets, even
  when we wanted to use bigger ones

This patch changes the following:
- noc_block_base now has improved docs for MTU, and additional APIs
  (get_max_payload_size(), get_chdr_hdr_len()) which return the
  current payload size given MTU and CHDR width, and the CHDR header
  length.
- The internally used graph nodes for TX and RX streamers also get
  equipped with the same new two API calls.
- The radio, siggen, and replay block all where doing different
  calculations for their spp/ipp values. Now, they all use the max
  payload value to calculate spp/ipp. Unit tests where adapted
  accordingly. Usage of DEFAULT_SPP was removed.
- The replay block used a hardcoded 16 bytes for header lengths, which
  was replaced by get_chdr_hdr_len()
- The TX and RX streamers where discarding the MTU value and using the
  max payload size as the MTU, which then propagated throughout the
  graph. Now, both values are stored and can be used where appropriate.

5 files changed, 108 insertions, 18 deletions

diff --git a/host/lib/include/uhdlib/rfnoc/chdr_rx_data_xport.hpp b/host/lib/include/uhdlib/rfnoc/chdr_rx_data_xport.hpp
index cb0987446..08930f9f3 100644
--- a/host/lib/include/uhdlib/rfnoc/chdr_rx_data_xport.hpp
+++ b/host/lib/include/uhdlib/rfnoc/chdr_rx_data_xport.hpp
@@ -177,13 +177,41 @@ public:
      */
     ~chdr_rx_data_xport();
 
-    /*! Returns maximum number payload bytes
+    /*! Returns MTU for this transport in bytes
      *
-     * \return maximum payload bytes per packet
+     * MTU is the max size for CHDR packets, including headers. For most
+     * applications, get_max_payload_size() is probably the more useful method.
+     * Compare also noc_block_base::get_mtu().
+     *
+     * \return MTU in bytes
+     */
+    size_t get_mtu() const
+    {
+        return _mtu;
+    }
+
+    /*! Return the size of a CHDR packet header, in bytes.
+     *
+     * This helper function factors in the CHDR width for this transport.
+     * Compare also noc_block_base::get_chdr_hdr_len().
+     *
+     * \returns the length of a CHDR header in bytes
+     */
+    size_t get_chdr_hdr_len() const
+    {
+        return _hdr_len;
+    }
+
+    /*! Returns maximum number of payload bytes
+     *
+     * This is smaller than the MTU. Compare also
+     * noc_block_base::get_max_payload_size().
+     *
+     * \return maximum number of payload bytes
      */
     size_t get_max_payload_size() const
     {
-        return _max_payload_size;
+        return _mtu - _hdr_len;
     }
 
     /*!
@@ -389,8 +417,11 @@ private:
     // Flow control state
     rx_flow_ctrl_state _fc_state;
 
-    // Maximum data payload in bytes
-    size_t _max_payload_size = 0;
+    // MTU in bytes
+    size_t _mtu = 0;
+
+    // Size of CHDR headers
+    size_t _hdr_len = 0;
 
     // Sequence number for data packets
     uint16_t _data_seq_num = 0;
diff --git a/host/lib/include/uhdlib/rfnoc/chdr_tx_data_xport.hpp b/host/lib/include/uhdlib/rfnoc/chdr_tx_data_xport.hpp
index 4e8ebe24d..e0887db6c 100644
--- a/host/lib/include/uhdlib/rfnoc/chdr_tx_data_xport.hpp
+++ b/host/lib/include/uhdlib/rfnoc/chdr_tx_data_xport.hpp
@@ -175,13 +175,41 @@ public:
      */
     ~chdr_tx_data_xport();
 
+    /*! Returns MTU for this transport in bytes
+     *
+     * MTU is the max size for CHDR packets, including headers. For most
+     * applications, get_max_payload_size() is probably the more useful method.
+     * Compare also noc_block_base::get_mtu().
+     *
+     * \return MTU in bytes
+     */
+    size_t get_mtu() const
+    {
+        return _mtu;
+    }
+
+    /*! Return the size of a CHDR packet header, in bytes.
+     *
+     * This helper function factors in the CHDR width for this transport.
+     * Compare also noc_block_base::get_chdr_hdr_len().
+     *
+     * \returns the length of a CHDR header in bytes
+     */
+    size_t get_chdr_hdr_len() const
+    {
+        return _hdr_len;
+    }
+
     /*! Returns maximum number of payload bytes
      *
+     * This is smaller than the MTU. Compare also
+     * noc_block_base::get_max_payload_size().
+     *
      * \return maximum number of payload bytes
      */
     size_t get_max_payload_size() const
     {
-        return _max_payload_size;
+        return _mtu - _hdr_len;
     }
 
     /*!
@@ -373,8 +401,11 @@ private:
     // Flow control state
     tx_flow_ctrl_state _fc_state;
 
-    // Maximum data payload in bytes
-    size_t _max_payload_size = 0;
+    // MTU in bytes
+    size_t _mtu = 0;
+
+    // Size of CHDR headers
+    size_t _hdr_len = 0;
 
     // Sequence number for data packets
     uint16_t _data_seq_num = 0;
diff --git a/host/lib/include/uhdlib/rfnoc/radio_control_impl.hpp b/host/lib/include/uhdlib/rfnoc/radio_control_impl.hpp
index b60b0f2a2..e268dbc07 100644
--- a/host/lib/include/uhdlib/rfnoc/radio_control_impl.hpp
+++ b/host/lib/include/uhdlib/rfnoc/radio_control_impl.hpp
@@ -356,6 +356,15 @@ private:
         const std::vector<uint32_t>& data,
         boost::optional<uint64_t> timestamp);
 
+    //! Return the maximum samples per packet of size \p bytes
+    //
+    // Given a packet of size \p bytes, how many samples can we fit in there?
+    // This gives the answer, factoring in item size and samples per clock.
+    //
+    // \param bytes Number of bytes we can fill with samples (excluding bytes
+    //              required for CHDR headers!)
+    int get_max_spp(const size_t bytes);
+
     //! FPGA compat number
     const uint32_t _fpga_compat;
 
diff --git a/host/lib/include/uhdlib/transport/rx_streamer_impl.hpp b/host/lib/include/uhdlib/transport/rx_streamer_impl.hpp
index 79c196f3f..f776d9373 100644
--- a/host/lib/include/uhdlib/transport/rx_streamer_impl.hpp
+++ b/host/lib/include/uhdlib/transport/rx_streamer_impl.hpp
@@ -88,7 +88,6 @@ public:
 
         if (stream_args.args.has_key("spp")) {
             _spp = stream_args.args.cast<size_t>("spp", _spp);
-            _mtu = _spp * _convert_info.bytes_per_otw_item;
         }
     }
 
@@ -97,7 +96,8 @@ public:
     // them
     virtual void connect_channel(const size_t channel, typename transport_t::uptr xport)
     {
-        const size_t mtu = xport->get_max_payload_size();
+        const size_t mtu = xport->get_mtu();
+        _hdr_len = std::max(_hdr_len, xport->get_chdr_hdr_len());
         _zero_copy_streamer.connect_channel(channel, std::move(xport));
 
         if (mtu < _mtu) {
@@ -201,11 +201,15 @@ protected:
         return _mtu;
     }
 
-    //! Sets the MTU and calculates spp
+    //! Sets the MTU and checks spp. If spp would exceed the new MTU, it is
+    // reduced accordingly.
     void set_mtu(const size_t mtu)
     {
         _mtu = mtu;
-        _spp = _mtu / _convert_info.bytes_per_otw_item;
+        const size_t spp_from_mtu = (_mtu - _hdr_len) / _convert_info.bytes_per_otw_item;
+        if (spp_from_mtu < _spp) {
+            _spp = spp_from_mtu;
+        }
     }
 
     //! Configures sample rate for conversion of timestamp
@@ -401,7 +405,12 @@ private:
     // MTU, determined when xport is connected and modifiable by subclass
     size_t _mtu = std::numeric_limits<std::size_t>::max();
 
-    // Maximum number of samples per packet
+    // Size of CHDR header in bytes
+    size_t _hdr_len = 0;
+
+    // Maximum number of samples per packet. Note that this is not necessarily
+    // related to the MTU, it is a user-chosen value. However, it is always
+    // bounded by the MTU.
     size_t _spp = std::numeric_limits<std::size_t>::max();
 
     // Num samps remaining in buffer currently held by zero copy streamer
diff --git a/host/lib/include/uhdlib/transport/tx_streamer_impl.hpp b/host/lib/include/uhdlib/transport/tx_streamer_impl.hpp
index f814c8192..9dc3b0c35 100644
--- a/host/lib/include/uhdlib/transport/tx_streamer_impl.hpp
+++ b/host/lib/include/uhdlib/transport/tx_streamer_impl.hpp
@@ -11,6 +11,7 @@
 #include <uhd/stream.hpp>
 #include <uhd/types/metadata.hpp>
 #include <uhd/utils/tasks.hpp>
+#include <uhd/utils/log.hpp>
 #include <uhdlib/transport/tx_streamer_zero_copy.hpp>
 #include <limits>
 #include <vector>
@@ -111,13 +112,13 @@ public:
 
         if (stream_args.args.has_key("spp")) {
             _spp = stream_args.args.cast<size_t>("spp", _spp);
-            _mtu = _spp * _convert_info.bytes_per_otw_item;
         }
     }
 
     virtual void connect_channel(const size_t channel, typename transport_t::uptr xport)
     {
-        const size_t mtu = xport->get_max_payload_size();
+        const size_t mtu = xport->get_mtu();
+        _hdr_len = std::max(_hdr_len, xport->get_chdr_hdr_len());
         _zero_copy_streamer.connect_channel(channel, std::move(xport));
 
         if (mtu < _mtu) {
@@ -323,11 +324,15 @@ protected:
         return _mtu;
     }
 
-    //! Sets the MTU and calculates spp
+    //! Sets the MTU and checks spp. If spp would exceed the new MTU, it is
+    // reduced accordingly.
     void set_mtu(const size_t mtu)
     {
         _mtu = mtu;
-        _spp = _mtu / _convert_info.bytes_per_otw_item;
+        const size_t spp_from_mtu = (_mtu - _hdr_len) / _convert_info.bytes_per_otw_item;
+        if (spp_from_mtu < _spp) {
+            _spp = spp_from_mtu;
+        }
     }
 
     //! Configures scaling factor for conversion
@@ -444,7 +449,12 @@ private:
     // MTU, determined when xport is connected and modifiable by subclass
     size_t _mtu = std::numeric_limits<std::size_t>::max();
 
-    // Maximum number of samples per packet
+    // Size of CHDR header in bytes
+    size_t _hdr_len = 0;
+
+    // Maximum number of samples per packet. Note that this is not necessarily
+    // related to the MTU, it is a user-chosen value. However, it is always
+    // bounded by the MTU.
     size_t _spp = std::numeric_limits<std::size_t>::max();
 
     // Metadata cache for send calls with no data