utils: Expose CHDR Parsing API

This commit introduces a new public api in uhd::utils which allows serializing
and deserializing chdr packets.

As far as testing, this commit adds the chdr_parse_test test. It uses a
wireshark trace located in rfnoc_packets_*.cpp as well as hand coded
packets from hardcoded_packets.cpp to test the serialization and
deserialization process

Signed-off-by: Samuel O'Brien <sam.obrien@ni.com>

6 files changed, 371 insertions, 11 deletions

diff --git a/host/include/uhd/rfnoc/chdr_types.hpp b/host/include/uhd/rfnoc/chdr_types.hpp
index 2f6c27c84..006aac708 100644
--- a/host/include/uhd/rfnoc/chdr_types.hpp
+++ b/host/include/uhd/rfnoc/chdr_types.hpp
@@ -9,7 +9,9 @@
 #include <uhd/rfnoc/rfnoc_types.hpp>
 #include <uhd/types/endianness.hpp>
 #include <uhd/utils/byteswap.hpp>
+#include <boost/format.hpp>
 #include <boost/optional.hpp>
+#include <deque>
 #include <list>
 #include <memory>
 #include <vector>
@@ -173,6 +175,19 @@ public: // Functions
         return *this;
     }
 
+    //! Return a string representation of this object
+    inline const std::string to_string() const
+    {
+        // The static_casts are because vc and num_mdata are uint8_t -> unsigned char
+        // For some reason, despite the %u meaning unsigned int, boost still formats them
+        // as chars
+        return str(boost::format("chdr_header{vc:%u, eob:%b, eov:%b, pkt_type:%u, "
+                                 "num_mdata:%u, seq_num:%u, length:%u, dst_epid:%u}\n")
+                   % static_cast<uint16_t>(get_vc()) % get_eob() % get_eov()
+                   % get_pkt_type() % static_cast<uint16_t>(get_num_mdata())
+                   % get_seq_num() % get_length() % get_dst_epid());
+    }
+
 private:
     // The flattened representation of the header stored in host order
     uint64_t _flat_hdr = 0;
@@ -246,7 +261,7 @@ enum ctrl_opcode_t {
     OP_USER6       = 0xF,
 };
 
-class ctrl_payload
+class UHD_API ctrl_payload
 {
 public: // Members
     //! Destination port for transaction (10 bits)
@@ -314,6 +329,9 @@ public: // Functions
         deserialize(buff, num_elems, conv_byte_order);
     }
 
+    //! Get the serialized size of this payload in 64 bit words
+    size_t get_length() const;
+
     // Return whether or not we have a valid timestamp
     bool has_timestamp() const
     {
@@ -373,7 +391,7 @@ enum strs_status_t {
     STRS_RTERR   = 0x4, //! Unexpected destination (routing error)
 };
 
-class strs_payload
+class UHD_API strs_payload
 {
 public: // Members
     //! The source EPID for the stream (16 bits)
@@ -435,6 +453,9 @@ public: // Functions
         deserialize(buff, num_elems, conv_byte_order);
     }
 
+    //! Get the serialized size of this payload in 64 bit words
+    size_t get_length() const;
+
     //! Comparison operator (==)
     bool operator==(const strs_payload& rhs) const;
 
@@ -476,7 +497,7 @@ enum strc_op_code_t {
     STRC_RESYNC = 0x2, //! Re-synchronize flow control
 };
 
-class strc_payload
+class UHD_API strc_payload
 {
 public: // Members
     //! The source EPID for the stream (16 bits)
@@ -534,6 +555,9 @@ public: // Functions
         deserialize(buff, num_elems, conv_byte_order);
     }
 
+    //! Get the serialized size of this payload in 64 bit words
+    size_t get_length() const;
+
     //! Comparison operator (==)
     bool operator==(const strc_payload& rhs) const;
 
@@ -566,7 +590,7 @@ private:
 //! A class that represents a single management operation
 //  An operation consists of an operation code and some
 //  payload associated with that operation.
-class mgmt_op_t
+class UHD_API mgmt_op_t
 {
 public:
     // Operation code
@@ -684,6 +708,9 @@ public:
         return (_op_code == rhs._op_code) && (_op_payload == rhs._op_payload);
     }
 
+    //! Return a string representation of this object
+    const std::string to_string() const;
+
 private:
     op_code_t _op_code;
     payload_t _op_payload;
@@ -692,7 +719,7 @@ private:
 //! A class that represents a single management hop
 //  A hop is a collection for management transactions for
 //  a single node.
-class mgmt_hop_t
+class UHD_API mgmt_hop_t
 {
 public:
     mgmt_hop_t()                      = default;
@@ -737,6 +764,9 @@ public:
         return _ops == rhs._ops;
     }
 
+    //! Return a string representation of this object
+    const std::string to_string() const;
+
 private:
     std::vector<mgmt_op_t> _ops;
 };
@@ -744,7 +774,7 @@ private:
 //! A class that represents a complete multi-hop management transaction
 //  A transaction is a collection of hops, where each hop is a collection
 //  of management transactions.
-class mgmt_payload
+class UHD_API mgmt_payload
 {
 public:
     mgmt_payload()                        = default;
@@ -779,6 +809,14 @@ public:
         return _hops.at(i);
     }
 
+    //! Pop the first hop of the transaction and return it
+    inline mgmt_hop_t pop_hop()
+    {
+        auto hop = _hops.front();
+        _hops.pop_front();
+        return hop;
+    }
+
     inline size_t get_size_bytes() const
     {
         size_t num_lines = 1; /* header */
@@ -823,9 +861,15 @@ public:
         deserialize(buff, num_elems, conv_byte_order);
     }
 
+    //! Get the serialized size of this payload in 64 bit words
+    size_t get_length() const;
+
     //! Return a string representation of this object
     const std::string to_string() const;
 
+    //! Return a string representaiton of the hops contained by this object
+    const std::string hops_to_string() const;
+
     //! Return the source EPID for this transaction
     inline sep_id_t get_src_epid() const
     {
@@ -867,11 +911,39 @@ public:
     }
 
 private:
-    sep_id_t _src_epid = 0;
-    uint16_t _protover = 0;
-    chdr_w_t _chdr_w   = CHDR_W_64;
-    std::vector<mgmt_hop_t> _hops;
+    sep_id_t _src_epid   = 0;
+    uint16_t _protover   = 0;
+    chdr_w_t _chdr_w     = CHDR_W_64;
     size_t _padding_size = 0;
+    std::deque<mgmt_hop_t> _hops;
 };
 
+//! Conversion from payload_t to pkt_type
+template <typename payload_t>
+constexpr packet_type_t payload_to_packet_type();
+
+template <>
+constexpr packet_type_t payload_to_packet_type<ctrl_payload>()
+{
+    return PKT_TYPE_CTRL;
+}
+
+template <>
+constexpr packet_type_t payload_to_packet_type<mgmt_payload>()
+{
+    return PKT_TYPE_MGMT;
+}
+
+template <>
+constexpr packet_type_t payload_to_packet_type<strc_payload>()
+{
+    return PKT_TYPE_STRC;
+}
+
+template <>
+constexpr packet_type_t payload_to_packet_type<strs_payload>()
+{
+    return PKT_TYPE_STRS;
+}
+
 }}} // namespace uhd::rfnoc::chdr
diff --git a/host/include/uhd/rfnoc/rfnoc_types.hpp b/host/include/uhd/rfnoc/rfnoc_types.hpp
index beb79324f..606850a27 100644
--- a/host/include/uhd/rfnoc/rfnoc_types.hpp
+++ b/host/include/uhd/rfnoc/rfnoc_types.hpp
@@ -6,7 +6,8 @@
 
 #pragma once
 
-#include <memory>
+#include <cstddef>
+#include <cstdint>
 
 namespace uhd { namespace rfnoc {
 
diff --git a/host/include/uhd/utils/CMakeLists.txt b/host/include/uhd/utils/CMakeLists.txt
index 17d5b2380..08d488771 100644
--- a/host/include/uhd/utils/CMakeLists.txt
+++ b/host/include/uhd/utils/CMakeLists.txt
@@ -5,6 +5,8 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
 
+add_subdirectory(chdr)
+
 UHD_INSTALL(FILES
     algorithm.hpp
     assert_has.hpp
diff --git a/host/include/uhd/utils/chdr/CMakeLists.txt b/host/include/uhd/utils/chdr/CMakeLists.txt
new file mode 100644
index 000000000..b0bf43ce2
--- /dev/null
+++ b/host/include/uhd/utils/chdr/CMakeLists.txt
@@ -0,0 +1,12 @@
+#
+# Copyright 2020 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+UHD_INSTALL(FILES
+    chdr_packet.hpp
+    chdr_packet.ipp
+    DESTINATION ${INCLUDE_DIR}/uhd/utils/chdr
+    COMPONENT headers
+)
diff --git a/host/include/uhd/utils/chdr/chdr_packet.hpp b/host/include/uhd/utils/chdr/chdr_packet.hpp
new file mode 100644
index 000000000..758782d70
--- /dev/null
+++ b/host/include/uhd/utils/chdr/chdr_packet.hpp
@@ -0,0 +1,177 @@
+//
+// Copyright 2020 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+
+#pragma once
+
+#include <uhd/rfnoc/chdr_types.hpp>
+#include <boost/optional.hpp>
+#include <memory>
+#include <vector>
+
+namespace uhd { namespace utils { namespace chdr {
+
+/*! A Generic class that represents a CHDR Packet
+ *
+ * Whether the packet has a specific type of payload is not specified
+ */
+class UHD_API chdr_packet
+{
+public:
+    /*! Constructs a CHDR Packet from a header and a payload
+     *
+     * timestamp and metadata are optional and will be empty if omitted
+     */
+    template <typename payload_t>
+    chdr_packet(uhd::rfnoc::chdr_w_t chdr_w,
+        uhd::rfnoc::chdr::chdr_header header,
+        payload_t payload,
+        boost::optional<uint64_t> timestamp = boost::none,
+        std::vector<uint64_t> metadata      = {});
+
+    /*! Construct a CHDR Packet from a header and raw payload words
+     *
+     * timestamp and metadata are optional and will be empty if omitted
+     */
+    chdr_packet(uhd::rfnoc::chdr_w_t chdr_w,
+        uhd::rfnoc::chdr::chdr_header header,
+        std::vector<uint8_t> payload_data,
+        boost::optional<uint64_t> timestamp = boost::none,
+        std::vector<uint64_t> mdata         = {});
+
+    /*! Returns the contents of the CHDR header
+     *
+     * \return The CHDR header
+     */
+    uhd::rfnoc::chdr::chdr_header get_header() const;
+
+    /*! Change this object's header
+     *
+     * \param header The new CHDR header
+     */
+    void set_header(uhd::rfnoc::chdr::chdr_header header);
+
+    /*! Returns the timestamp in the packet as an optional value
+     *
+     * \return A boost::optional which if initialized has the timestamp
+     */
+    boost::optional<uint64_t> get_timestamp() const;
+
+    /*! Sets the timestamp in the packet
+     *
+     * \param timestamp the timestamp to set, or boost::none for no timestamp
+     */
+    void set_timestamp(boost::optional<uint64_t> timestamp);
+
+    /*! Returns a const reference to the metadata
+     *
+     * \return The metadata reference
+     */
+    const std::vector<uint64_t>& get_metadata() const;
+
+    /*! Moves the metadata out of this object and returns it in a new vector
+     *
+     * \return A vector containing the moved metadata
+     */
+    void set_metadata(std::vector<uint64_t> metadata);
+
+    /*! Serialize a CHDR Packet into a vector of bytes
+     *
+     * \param endianness the endianness of the returned vector (link endianness)
+     * \return a vector of bytes which represents the CHDR Packet in serialize form
+     */
+    std::vector<uint8_t> serialize_to_byte_vector(
+        endianness_t endianness = uhd::ENDIANNESS_LITTLE) const;
+
+    /*! Serialize a CHDR Packet into a buffer
+     *
+     * \param endianness the endianness of the output buffer (link endianness)
+     * \param first the start of the output buffer
+     * \param last the end of the output buffer
+     */
+    template <typename OutputIterator>
+    void serialize(OutputIterator first,
+        OutputIterator last,
+        endianness_t endianness = uhd::ENDIANNESS_LITTLE) const;
+
+    /*! Deserialize a CHDR Packet from a buffer of bytes
+     *
+     * \param chdr_w the CHDR_W of the incoming packet
+     * \param endianness the endianness of the input buffer (link endianness)
+     * \param first the start of the input buffer
+     * \param last the end of the input buffer
+     * \return a CHDR Packet with an unspecified payload type
+     */
+    template <typename InputIterator>
+    static chdr_packet deserialize(uhd::rfnoc::chdr_w_t chdr_w,
+        InputIterator first,
+        InputIterator last,
+        endianness_t endianness = uhd::ENDIANNESS_LITTLE);
+
+    /*! Get the total serialized length of the packet
+     *
+     * \return The length of the packet in bytes
+     */
+    size_t get_packet_len() const;
+
+    /*! Returns a const reference to the payload bytes
+     *
+     * \return The payload reference
+     */
+    const std::vector<uint8_t>& get_payload_bytes() const;
+
+    /*! Sets the current payload
+     *
+     * \param bytes the payload to store inside this object (It is moved from)
+     */
+    void set_payload_bytes(std::vector<uint8_t> bytes);
+
+    /*! Parses the data out of this objects payload field into a payload_t object
+     *
+     * \param endianness The link endianness of the CHDR Link
+     * \return The parsed payload_t object
+     */
+    template <typename payload_t>
+    payload_t get_payload(uhd::endianness_t endianness = uhd::ENDIANNESS_LITTLE) const;
+
+    /*! Serializes the payload object into bytes and stores it in this object's payload
+     * field
+     *
+     * \param payload the payload object to store
+     * \param endianness The link endianness of the CHDR Link
+     */
+    template <typename payload_t>
+    void set_payload(
+        payload_t payload, uhd::endianness_t endianness = uhd::ENDIANNESS_LITTLE);
+
+    //! Return a string representation of this object
+    const std::string to_string() const;
+
+    //! Return a string representation of this object and deserialize its payload
+    template <typename payload_t>
+    const std::string to_string_with_payload(
+        uhd::endianness_t endianness = uhd::ENDIANNESS_LITTLE) const;
+
+private:
+    void serialize_ptr(endianness_t endianness, void* start, void* end) const;
+    static chdr_packet deserialize_ptr(uhd::rfnoc::chdr_w_t chdr_w,
+        endianness_t endianness,
+        const void* start,
+        const void* end);
+    inline void set_header_lengths()
+    {
+        _header.set_num_mdata(_mdata.size() / (uhd::rfnoc::chdr_w_to_bits(_chdr_w) / 64));
+        _header.set_length(get_packet_len());
+    }
+    uhd::rfnoc::chdr_w_t _chdr_w;
+    uhd::rfnoc::chdr::chdr_header _header;
+    std::vector<uint8_t> _payload;
+    boost::optional<uint64_t> _timestamp;
+    std::vector<uint64_t> _mdata;
+};
+
+}}} // namespace uhd::utils::chdr
+
+#include <uhd/utils/chdr/chdr_packet.ipp>
diff --git a/host/include/uhd/utils/chdr/chdr_packet.ipp b/host/include/uhd/utils/chdr/chdr_packet.ipp
new file mode 100644
index 000000000..866c9f93d
--- /dev/null
+++ b/host/include/uhd/utils/chdr/chdr_packet.ipp
@@ -0,0 +1,96 @@
+//
+// Copyright 2020 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+
+#pragma once
+
+#include <uhd/exception.hpp>
+#include <uhd/types/endianness.hpp>
+
+namespace chdr_rfnoc = uhd::rfnoc::chdr;
+namespace chdr_util  = uhd::utils::chdr;
+
+template <typename payload_t>
+chdr_util::chdr_packet::chdr_packet(uhd::rfnoc::chdr_w_t chdr_w,
+    chdr_rfnoc::chdr_header header,
+    payload_t payload,
+    boost::optional<uint64_t> timestamp,
+    std::vector<uint64_t> metadata)
+    : chdr_packet(chdr_w, header, std::vector<uint8_t>(), timestamp, std::move(metadata))
+{
+    set_payload(payload);
+}
+
+template <typename OutputIterator>
+void chdr_util::chdr_packet::serialize(
+    OutputIterator first, OutputIterator last, uhd::endianness_t endianness) const
+{
+    void* start = static_cast<void*>(&*first);
+    void* end   = static_cast<void*>(&*last);
+    serialize_ptr(endianness, start, end);
+}
+
+template <typename InputIterator>
+chdr_util::chdr_packet chdr_util::chdr_packet::deserialize(uhd::rfnoc::chdr_w_t chdr_w,
+    InputIterator first,
+    InputIterator last,
+    uhd::endianness_t endianness)
+{
+    void* start = static_cast<void*>(&*first);
+    void* end   = static_cast<void*>(&*last);
+    return deserialize_ptr(chdr_w, endianness, start, end);
+}
+
+template <typename payload_t>
+payload_t chdr_util::chdr_packet::get_payload(uhd::endianness_t endianness) const
+{
+    payload_t payload;
+    // Only Data Packets are allowed to have end on a incomplete CHDR_W length.
+    // This method is never called on a data packet. (They don't have a payload_t)
+    UHD_ASSERT_THROW(this->_payload.size() % sizeof(uint64_t) == 0)
+    auto conv_byte_order = [endianness](uint64_t x) -> uint64_t {
+        return (endianness == uhd::ENDIANNESS_BIG) ? uhd::htonx<uint64_t>(x)
+                                                   : uhd::htowx<uint64_t>(x);
+    };
+    payload.deserialize(reinterpret_cast<const uint64_t*>(this->_payload.data()),
+        this->_payload.size(),
+        conv_byte_order);
+    return payload;
+}
+
+template <typename payload_t>
+void chdr_util::chdr_packet::set_payload(payload_t payload, uhd::endianness_t endianness)
+{
+    _header.set_pkt_type(chdr_rfnoc::payload_to_packet_type<payload_t>());
+    // Meaning a 64 bit word (The basic unit of data for payloads)
+    size_t payload_len_words = payload.get_length();
+    this->_payload.resize(payload_len_words * sizeof(uint64_t), 0);
+    auto conv_byte_order = [endianness](uint64_t x) -> uint64_t {
+        return (endianness == uhd::ENDIANNESS_BIG) ? uhd::htonx<uint64_t>(x)
+                                                   : uhd::htowx<uint64_t>(x);
+    };
+    payload.serialize(reinterpret_cast<uint64_t*>(this->_payload.data()),
+        this->_payload.size(),
+        conv_byte_order);
+    set_header_lengths();
+}
+
+template <typename payload_t>
+const std::string chdr_util::chdr_packet::to_string_with_payload(
+    uhd::endianness_t endianness) const
+{
+    payload_t payload = this->get_payload<payload_t>(endianness);
+    return to_string() + payload.to_string();
+}
+
+template <>
+const std::string
+chdr_util::chdr_packet::to_string_with_payload<chdr_rfnoc::mgmt_payload>(
+    uhd::endianness_t endianness) const
+{
+    chdr_rfnoc::mgmt_payload payload =
+        this->get_payload<chdr_rfnoc::mgmt_payload>(endianness);
+    return to_string() + payload.to_string() + payload.hops_to_string();
+}