diff options
Diffstat (limited to 'host/lib/transport')
| -rw-r--r-- | host/lib/transport/CMakeLists.txt | 24 | ||||
| -rw-r--r-- | host/lib/transport/convert_types_impl.hpp | 201 | ||||
| -rwxr-xr-x | host/lib/transport/gen_convert_types.py | 122 | ||||
| -rwxr-xr-x | host/lib/transport/gen_vrt_if_packet.py (renamed from host/lib/transport/gen_vrt.py) | 126 | ||||
| -rw-r--r-- | host/lib/transport/udp_zero_copy_asio.cpp | 9 | ||||
| -rw-r--r-- | host/lib/transport/vrt_packet_handler.hpp | 301 |
6 files changed, 489 insertions, 294 deletions
diff --git a/host/lib/transport/CMakeLists.txt b/host/lib/transport/CMakeLists.txt index 872865d6c..bde2b72b9 100644 --- a/host/lib/transport/CMakeLists.txt +++ b/host/lib/transport/CMakeLists.txt @@ -18,6 +18,16 @@ #This file will be included by cmake, use absolute paths! ######################################################################## +# Check for SIMD headers +######################################################################## +INCLUDE(CheckIncludeFileCXX) +CHECK_INCLUDE_FILE_CXX(emmintrin.h HAVE_EMMINTRIN_H) + +IF(HAVE_EMMINTRIN_H) + ADD_DEFINITIONS(-DHAVE_EMMINTRIN_H) +ENDIF(HAVE_EMMINTRIN_H) + +######################################################################## # Setup defines for interface address discovery ######################################################################## MESSAGE(STATUS "Configuring interface address discovery...") @@ -40,8 +50,8 @@ ENDIF(HAVE_IFADDRS_H) # Append to the list of sources for lib uhd ######################################################################## LIBUHD_PYTHON_GEN_SOURCE( - ${CMAKE_SOURCE_DIR}/lib/transport/gen_vrt.py - ${CMAKE_BINARY_DIR}/lib/transport/vrt.cpp + ${CMAKE_SOURCE_DIR}/lib/transport/gen_vrt_if_packet.py + ${CMAKE_BINARY_DIR}/lib/transport/vrt_if_packet.cpp ) LIBUHD_PYTHON_GEN_SOURCE( @@ -49,6 +59,16 @@ LIBUHD_PYTHON_GEN_SOURCE( ${CMAKE_BINARY_DIR}/lib/transport/convert_types.cpp ) +# append this directory to the include path so the generated convert types +# can include the implementation convert types file in the source directory +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/lib/transport) + +# make the generated convert types depend on the implementation header +SET_SOURCE_FILES_PROPERTIES( + ${CMAKE_BINARY_DIR}/lib/transport/convert_types.cpp PROPERTIES + OBJECT_DEPENDS ${CMAKE_SOURCE_DIR}/lib/transport/convert_types_impl.hpp +) + LIBUHD_APPEND_SOURCES( ${CMAKE_SOURCE_DIR}/lib/transport/if_addrs.cpp ${CMAKE_SOURCE_DIR}/lib/transport/udp_simple.cpp diff --git a/host/lib/transport/convert_types_impl.hpp b/host/lib/transport/convert_types_impl.hpp new file mode 100644 index 000000000..5958b08cb --- /dev/null +++ b/host/lib/transport/convert_types_impl.hpp @@ -0,0 +1,201 @@ +// +// Copyright 2010 Ettus Research LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. +// + +#ifndef INCLUDED_LIBUHD_TRANSPORT_CONVERT_TYPES_IMPL_HPP +#define INCLUDED_LIBUHD_TRANSPORT_CONVERT_TYPES_IMPL_HPP + +#include <uhd/config.hpp> +#include <uhd/utils/byteswap.hpp> +#include <boost/cstdint.hpp> +#include <cstring> +#include <complex> + +#ifdef HAVE_EMMINTRIN_H + #define USE_EMMINTRIN_H //use sse2 intrinsics +#endif + +/*********************************************************************** + * Typedefs + **********************************************************************/ +typedef std::complex<float> fc32_t; +typedef std::complex<boost::int16_t> sc16_t; +typedef boost::uint32_t item32_t; + +/*********************************************************************** + * Convert complex short buffer to items32 + **********************************************************************/ +static UHD_INLINE void sc16_to_item32_nswap( + const sc16_t *input, item32_t *output, size_t nsamps +){ + std::memcpy(output, input, nsamps*sizeof(item32_t)); +} + +static UHD_INLINE void sc16_to_item32_bswap( + const sc16_t *input, item32_t *output, size_t nsamps +){ + const item32_t *item32_input = (const item32_t *)input; + for (size_t i = 0; i < nsamps; i++){ + output[i] = uhd::byteswap(item32_input[i]); + } +} + +/*********************************************************************** + * Convert items32 buffer to complex short + **********************************************************************/ +static UHD_INLINE void item32_to_sc16_nswap( + const item32_t *input, sc16_t *output, size_t nsamps +){ + std::memcpy(output, input, nsamps*sizeof(item32_t)); +} + +static UHD_INLINE void item32_to_sc16_bswap( + const item32_t *input, sc16_t *output, size_t nsamps +){ + item32_t *item32_output = (item32_t *)output; + for (size_t i = 0; i < nsamps; i++){ + item32_output[i] = uhd::byteswap(input[i]); + } +} + +/*********************************************************************** + * Convert complex float buffer to items32 + **********************************************************************/ +static const float shorts_per_float = float(32767); + +static UHD_INLINE item32_t fc32_to_item32(fc32_t num){ + boost::uint16_t real = boost::int16_t(num.real()*shorts_per_float); + boost::uint16_t imag = boost::int16_t(num.imag()*shorts_per_float); + return (item32_t(real) << 16) | (item32_t(imag) << 0); +} + +static UHD_INLINE void fc32_to_item32_nswap( + const fc32_t *input, item32_t *output, size_t nsamps +){ + for (size_t i = 0; i < nsamps; i++){ + output[i] = fc32_to_item32(input[i]); + } +} + +#if defined(USE_EMMINTRIN_H) +#include <emmintrin.h> + +static UHD_INLINE void fc32_to_item32_bswap( + const fc32_t *input, item32_t *output, size_t nsamps +){ + __m128 scalar = _mm_set_ps1(shorts_per_float); + + //convert blocks of samples with intrinsics + size_t i = 0; for (; i < (nsamps & ~0x3); i+=4){ + //load from input + __m128 tmplo = _mm_loadu_ps(reinterpret_cast<const float *>(input+i+0)); + __m128 tmphi = _mm_loadu_ps(reinterpret_cast<const float *>(input+i+2)); + + //convert and scale + __m128i tmpilo = _mm_cvtps_epi32(_mm_mul_ps(tmplo, scalar)); + __m128i tmpihi = _mm_cvtps_epi32(_mm_mul_ps(tmphi, scalar)); + + //pack + byteswap -> byteswap 32 bit words + __m128i tmpi = _mm_packs_epi32(tmpilo, tmpihi); + tmpi = _mm_or_si128(_mm_srli_epi16(tmpi, 8), _mm_slli_epi16(tmpi, 8)); + + //store to output + _mm_storeu_si128(reinterpret_cast<__m128i *>(output+i), tmpi); + } + + //convert remainder + for (; i < nsamps; i++){ + output[i] = uhd::byteswap(fc32_to_item32(input[i])); + } +} + +#else +static UHD_INLINE void fc32_to_item32_bswap( + const fc32_t *input, item32_t *output, size_t nsamps +){ + for (size_t i = 0; i < nsamps; i++){ + output[i] = uhd::byteswap(fc32_to_item32(input[i])); + } +} + +#endif + +/*********************************************************************** + * Convert items32 buffer to complex float + **********************************************************************/ +static const float floats_per_short = float(1.0/shorts_per_float); + +static UHD_INLINE fc32_t item32_to_fc32(item32_t item){ + return fc32_t( + float(boost::int16_t(item >> 16)*floats_per_short), + float(boost::int16_t(item >> 0)*floats_per_short) + ); +} + +static UHD_INLINE void item32_to_fc32_nswap( + const item32_t *input, fc32_t *output, size_t nsamps +){ + for (size_t i = 0; i < nsamps; i++){ + output[i] = item32_to_fc32(input[i]); + } +} + +#if defined(USE_EMMINTRIN_H) +#include <emmintrin.h> + +static UHD_INLINE void item32_to_fc32_bswap( + const item32_t *input, fc32_t *output, size_t nsamps +){ + __m128 scalar = _mm_set_ps1(floats_per_short/(1 << 16)); + __m128i zeroi = _mm_setzero_si128(); + + //convert blocks of samples with intrinsics + size_t i = 0; for (; i < (nsamps & ~0x3); i+=4){ + //load from input + __m128i tmpi = _mm_loadu_si128(reinterpret_cast<const __m128i *>(input+i)); + + //byteswap + unpack -> byteswap 32 bit words + tmpi = _mm_or_si128(_mm_srli_epi16(tmpi, 8), _mm_slli_epi16(tmpi, 8)); + __m128i tmpilo = _mm_unpacklo_epi16(zeroi, tmpi); //value in upper 16 bits + __m128i tmpihi = _mm_unpackhi_epi16(zeroi, tmpi); + + //convert and scale + __m128 tmplo = _mm_mul_ps(_mm_cvtepi32_ps(tmpilo), scalar); + __m128 tmphi = _mm_mul_ps(_mm_cvtepi32_ps(tmpihi), scalar); + + //store to output + _mm_storeu_ps(reinterpret_cast<float *>(output+i+0), tmplo); + _mm_storeu_ps(reinterpret_cast<float *>(output+i+2), tmphi); + } + + //convert remainder + for (; i < nsamps; i++){ + output[i] = item32_to_fc32(uhd::byteswap(input[i])); + } +} + +#else +static UHD_INLINE void item32_to_fc32_bswap( + const item32_t *input, fc32_t *output, size_t nsamps +){ + for (size_t i = 0; i < nsamps; i++){ + output[i] = item32_to_fc32(uhd::byteswap(input[i])); + } +} + +#endif + +#endif /* INCLUDED_LIBUHD_TRANSPORT_CONVERT_TYPES_IMPL_HPP */ diff --git a/host/lib/transport/gen_convert_types.py b/host/lib/transport/gen_convert_types.py index af2bcc7cb..951b634d9 100755 --- a/host/lib/transport/gen_convert_types.py +++ b/host/lib/transport/gen_convert_types.py @@ -24,66 +24,15 @@ TMPL_TEXT = """ \#include <uhd/config.hpp> \#include <uhd/transport/convert_types.hpp> -\#include <uhd/utils/byteswap.hpp> \#include <boost/cstdint.hpp> \#include <boost/detail/endian.hpp> \#include <stdexcept> -\#include <complex> - -//define the endian macros to convert integers -\#ifdef BOOST_BIG_ENDIAN - \#define BE_MACRO(x) x - \#define LE_MACRO(x) uhd::byteswap(x) - static const bool is_big_endian = true; -\#else - \#define BE_MACRO(x) uhd::byteswap(x) - \#define LE_MACRO(x) x - static const bool is_big_endian = false; -\#endif +\#include "convert_types_impl.hpp" using namespace uhd; /*********************************************************************** - * Constants - **********************************************************************/ -typedef std::complex<float> fc32_t; -typedef std::complex<boost::int16_t> sc16_t; -typedef boost::uint32_t item32_t; - -static const float shorts_per_float = float(32767); -static const float floats_per_short = float(1.0/shorts_per_float); - -/*********************************************************************** - * Single-sample converters - **********************************************************************/ -static UHD_INLINE item32_t sc16_to_item32(sc16_t num){ - boost::uint16_t real = boost::int16_t(num.real()); - boost::uint16_t imag = boost::int16_t(num.imag()); - return (item32_t(real) << 16) | (item32_t(imag) << 0); -} - -static UHD_INLINE sc16_t item32_to_sc16(item32_t item){ - return sc16_t( - boost::uint16_t(item >> 16), - boost::uint16_t(item >> 0) - ); -} - -static UHD_INLINE item32_t fc32_to_item32(fc32_t num){ - boost::uint16_t real = boost::int16_t(num.real()*shorts_per_float); - boost::uint16_t imag = boost::int16_t(num.imag()*shorts_per_float); - return (item32_t(real) << 16) | (item32_t(imag) << 0); -} - -static UHD_INLINE fc32_t item32_to_fc32(item32_t item){ - return fc32_t( - float(boost::int16_t(item >> 16)*floats_per_short), - float(boost::int16_t(item >> 0)*floats_per_short) - ); -} - -/*********************************************************************** - * Sample-buffer converters + * Generate predicate for jump table **********************************************************************/ UHD_INLINE boost::uint8_t get_pred( const io_type_t &io_type, @@ -92,27 +41,34 @@ UHD_INLINE boost::uint8_t get_pred( boost::uint8_t pred = 0; switch(otw_type.byteorder){ - case otw_type_t::BO_BIG_ENDIAN: pred |= $ph.be_p; break; - case otw_type_t::BO_LITTLE_ENDIAN: pred |= $ph.le_p; break; - ##let the compiler determine the native byte order (we could use python sys.byteorder) - case otw_type_t::BO_NATIVE: pred |= (is_big_endian)? $ph.be_p : $ph.le_p; break; - default: throw std::runtime_error("unhandled byteorder type"); + \#ifdef BOOST_BIG_ENDIAN + case otw_type_t::BO_BIG_ENDIAN: pred |= $ph.nswap_p; break; + case otw_type_t::BO_LITTLE_ENDIAN: pred |= $ph.bswap_p; break; + \#else + case otw_type_t::BO_BIG_ENDIAN: pred |= $ph.bswap_p; break; + case otw_type_t::BO_LITTLE_ENDIAN: pred |= $ph.nswap_p; break; + \#endif + case otw_type_t::BO_NATIVE: pred |= $ph.nswap_p; break; + default: throw std::runtime_error("unhandled otw byteorder type"); } - switch(otw_type.width){ - case 16: pred |= $ph.w16_p; break; - default: throw std::runtime_error("unhandled bit width"); + switch(otw_type.get_sample_size()){ + case sizeof(boost::uint32_t): pred |= $ph.item32_p; break; + default: throw std::runtime_error("unhandled otw sample size"); } switch(io_type.tid){ - case io_type_t::COMPLEX_INT16: pred |= $ph.sc16_p; break; case io_type_t::COMPLEX_FLOAT32: pred |= $ph.fc32_p; break; + case io_type_t::COMPLEX_INT16: pred |= $ph.sc16_p; break; default: throw std::runtime_error("unhandled io type id"); } return pred; } +/*********************************************************************** + * Convert host type to device type + **********************************************************************/ void transport::convert_io_type_to_otw_type( const void *io_buff, const io_type_t &io_type, void *otw_buff, const otw_type_t &otw_type, @@ -123,16 +79,16 @@ void transport::convert_io_type_to_otw_type( case $pred: #set $out_type = $ph.get_dev_type($pred) #set $in_type = $ph.get_host_type($pred) - #set $converter = $in_type+"_to_"+$out_type - #set $xe_macro = $ph.get_xe_macro($pred) - for (size_t i = 0; i < num_samps; i++){ - (($(out_type)_t *)otw_buff)[i] = $(xe_macro)($(converter)(((const $(in_type)_t *)io_buff)[i])); - } + #set $converter = '_'.join([$in_type, 'to', $out_type, $ph.get_swap_type($pred)]) + $(converter)((const $(in_type)_t *)io_buff, ($(out_type)_t *)otw_buff, num_samps); break; #end for } } +/*********************************************************************** + * Convert device type to host type + **********************************************************************/ void transport::convert_otw_type_to_io_type( const void *otw_buff, const otw_type_t &otw_type, void *io_buff, const io_type_t &io_type, @@ -143,11 +99,8 @@ void transport::convert_otw_type_to_io_type( case $pred: #set $out_type = $ph.get_host_type($pred) #set $in_type = $ph.get_dev_type($pred) - #set $converter = $in_type+"_to_"+$out_type - #set $xe_macro = $ph.get_xe_macro($pred) - for (size_t i = 0; i < num_samps; i++){ - (($(out_type)_t *)io_buff)[i] = $(converter)($(xe_macro)(((const $(in_type)_t *)otw_buff)[i])); - } + #set $converter = '_'.join([$in_type, 'to', $out_type, $ph.get_swap_type($pred)]) + $(converter)((const $(in_type)_t *)otw_buff, ($(out_type)_t *)io_buff, num_samps); break; #end for } @@ -160,29 +113,32 @@ def parse_tmpl(_tmpl_text, **kwargs): return str(Template(_tmpl_text, kwargs)) class ph: - be_p = 0b00001 - le_p = 0b00000 - w16_p = 0b00000 - sc16_p = 0b00010 - fc32_p = 0b00000 + bswap_p = 0b00001 + nswap_p = 0b00000 + item32_p = 0b00000 + sc16_p = 0b00010 + fc32_p = 0b00000 nbits = 2 #see above @staticmethod - def get_xe_macro(pred): - if (pred & ph.be_p) == ph.be_p: return 'BE_MACRO' - if (pred & ph.le_p) == ph.le_p: return 'LE_MACRO' + def has(pred, flag): return (pred & flag) == flag + + @staticmethod + def get_swap_type(pred): + if ph.has(pred, ph.bswap_p): return 'bswap' + if ph.has(pred, ph.nswap_p): return 'nswap' raise NotImplementedError @staticmethod def get_dev_type(pred): - if (pred & ph.w16_p) == ph.w16_p: return 'item32' + if ph.has(pred, ph.item32_p): return 'item32' raise NotImplementedError @staticmethod def get_host_type(pred): - if (pred & ph.sc16_p) == ph.sc16_p: return 'sc16' - if (pred & ph.fc32_p) == ph.fc32_p: return 'fc32' + if ph.has(pred, ph.sc16_p): return 'sc16' + if ph.has(pred, ph.fc32_p): return 'fc32' raise NotImplementedError if __name__ == '__main__': diff --git a/host/lib/transport/gen_vrt.py b/host/lib/transport/gen_vrt_if_packet.py index 6cdd6645d..7910ff60d 100755 --- a/host/lib/transport/gen_vrt.py +++ b/host/lib/transport/gen_vrt_if_packet.py @@ -31,7 +31,7 @@ TMPL_TEXT = """ * This file was generated by $file on $time.strftime("%c") **********************************************************************/ -\#include <uhd/transport/vrt.hpp> +\#include <uhd/transport/vrt_if_packet.hpp> \#include <uhd/utils/byteswap.hpp> \#include <boost/detail/endian.hpp> \#include <stdexcept> @@ -61,20 +61,22 @@ using namespace uhd::transport; #set $tsf_p = 0b01000 #set $tlr_p = 0b10000 -void vrt::pack_$(suffix)( - const tx_metadata_t &metadata, //input - boost::uint32_t *header_buff, //output - size_t &num_header_words32, //output - size_t num_payload_words32, //input - size_t &num_packet_words32, //output - size_t packet_count, //input - double tick_rate //input +static UHD_INLINE void pack_uint64_$(suffix)(boost::uint64_t num, boost::uint32_t *mem){ + *(reinterpret_cast<boost::uint64_t *>(mem)) = $(XE_MACRO)(num); +} + +void vrt::if_hdr_pack_$(suffix)( + boost::uint32_t *packet_buff, + if_packet_info_t &if_packet_info ){ boost::uint32_t vrt_hdr_flags = 0; boost::uint8_t pred = 0; - if (metadata.has_stream_id) pred |= $hex($sid_p); - if (metadata.has_time_spec) pred |= $hex($tsi_p | $tsf_p); + if (if_packet_info.has_sid) pred |= $hex($sid_p); + if (if_packet_info.has_cid) pred |= $hex($cid_p); + if (if_packet_info.has_tsi) pred |= $hex($tsi_p); + if (if_packet_info.has_tsf) pred |= $hex($tsf_p); + if (if_packet_info.has_tlr) pred |= $hex($tlr_p); switch(pred){ #for $pred in range(2**5) @@ -83,81 +85,75 @@ void vrt::pack_$(suffix)( #set $flags = 0 ########## Stream ID ########## #if $pred & $sid_p - header_buff[$num_header_words] = $(XE_MACRO)(metadata.stream_id); + packet_buff[$num_header_words] = $(XE_MACRO)(if_packet_info.sid); #set $num_header_words += 1 #set $flags |= (0x1 << 28); #end if ########## Class ID ########## #if $pred & $cid_p - header_buff[$num_header_words] = 0; - #set $num_header_words += 1 - header_buff[$num_header_words] = 0; - #set $num_header_words += 1 + pack_uint64_$(suffix)(if_packet_info.cid, packet_buff+$num_header_words); + #set $num_header_words += 2 #set $flags |= (0x1 << 27); #end if ########## Integer Time ########## #if $pred & $tsi_p - header_buff[$num_header_words] = $(XE_MACRO)(metadata.time_spec.secs); + packet_buff[$num_header_words] = $(XE_MACRO)(if_packet_info.tsi); #set $num_header_words += 1 #set $flags |= (0x3 << 22); #end if ########## Fractional Time ########## #if $pred & $tsf_p - header_buff[$num_header_words] = 0; - #set $num_header_words += 1 - header_buff[$num_header_words] = $(XE_MACRO)(metadata.time_spec.get_ticks(tick_rate)); - #set $num_header_words += 1 + pack_uint64_$(suffix)(if_packet_info.tsf, packet_buff+$num_header_words); + #set $num_header_words += 2 #set $flags |= (0x1 << 20); #end if ########## Trailer ########## #if $pred & $tlr_p + //packet_buff[$num_header_words+if_packet_info.num_payload_words32] = $(XE_MACRO)(if_packet_info.tlr); #set $flags |= (0x1 << 26); #set $num_trailer_words = 1; #else + //packet_buff[$num_header_words+if_packet_info.num_payload_words32] = 0; #set $num_trailer_words = 0; #end if ########## Variables ########## - num_header_words32 = $num_header_words; - num_packet_words32 = $($num_header_words + $num_trailer_words) + num_payload_words32; + if_packet_info.num_header_words32 = $num_header_words; + if_packet_info.num_packet_words32 = $($num_header_words + $num_trailer_words) + if_packet_info.num_payload_words32; vrt_hdr_flags = $hex($flags); break; #end for } //set the burst flags - if (metadata.start_of_burst) vrt_hdr_flags |= $hex(0x1 << 25); - if (metadata.end_of_burst) vrt_hdr_flags |= $hex(0x1 << 24); + if (if_packet_info.sob) vrt_hdr_flags |= $hex(0x1 << 25); + if (if_packet_info.eob) vrt_hdr_flags |= $hex(0x1 << 24); //fill in complete header word - header_buff[0] = $(XE_MACRO)(boost::uint32_t(0 + packet_buff[0] = $(XE_MACRO)(boost::uint32_t(0 | vrt_hdr_flags - | ((packet_count & 0xf) << 16) - | (num_packet_words32 & 0xffff) + | ((if_packet_info.packet_count & 0xf) << 16) + | (if_packet_info.num_packet_words32 & 0xffff) )); } -void vrt::unpack_$(suffix)( - rx_metadata_t &metadata, //output - const boost::uint32_t *header_buff, //input - size_t &num_header_words32, //output - size_t &num_payload_words32, //output - size_t num_packet_words32, //input - size_t &packet_count, //output - double tick_rate //input -){ - //clear the metadata - metadata = rx_metadata_t(); +static UHD_INLINE void unpack_uint64_$(suffix)(boost::uint64_t &num, const boost::uint32_t *mem){ + num = $(XE_MACRO)(*reinterpret_cast<const boost::uint64_t *>(mem)); +} +void vrt::if_hdr_unpack_$(suffix)( + const boost::uint32_t *packet_buff, + if_packet_info_t &if_packet_info +){ //extract vrt header - boost::uint32_t vrt_hdr_word = $(XE_MACRO)(header_buff[0]); + boost::uint32_t vrt_hdr_word = $(XE_MACRO)(packet_buff[0]); size_t packet_words32 = vrt_hdr_word & 0xffff; - packet_count = (vrt_hdr_word >> 16) & 0xf; + if_packet_info.packet_count = (vrt_hdr_word >> 16) & 0xf; //failure cases - if (packet_words32 == 0 or num_packet_words32 < packet_words32) + if (if_packet_info.num_packet_words32 < packet_words32) throw std::runtime_error("bad vrt header or packet fragment"); if (vrt_hdr_word & (0x7 << 29)) - throw std::runtime_error("unsupported vrt packet type"); + throw std::runtime_error("bad vrt header or unsupported packet type"); boost::uint8_t pred = 0; if(vrt_hdr_word & $hex(0x1 << 28)) pred |= $hex($sid_p); @@ -169,45 +165,55 @@ void vrt::unpack_$(suffix)( switch(pred){ #for $pred in range(2**5) case $pred: - #set $set_has_time_spec = False + #set $has_time_spec = False #set $num_header_words = 1 ########## Stream ID ########## #if $pred & $sid_p - metadata.has_stream_id = true; - metadata.stream_id = $(XE_MACRO)(header_buff[$num_header_words]); + if_packet_info.has_sid = true; + if_packet_info.sid = $(XE_MACRO)(packet_buff[$num_header_words]); #set $num_header_words += 1 + #else + if_packet_info.has_sid = false; #end if ########## Class ID ########## #if $pred & $cid_p - #set $num_header_words += 1 - #set $num_header_words += 1 + if_packet_info.has_cid = true; + unpack_uint64_$(suffix)(if_packet_info.cid, packet_buff+$num_header_words); + #set $num_header_words += 2 + #else + if_packet_info.has_cid = false; #end if ########## Integer Time ########## #if $pred & $tsi_p - metadata.has_time_spec = true; - #set $set_has_time_spec = True - metadata.time_spec.secs = $(XE_MACRO)(header_buff[$num_header_words]); + if_packet_info.has_tsi = true; + if_packet_info.tsi = $(XE_MACRO)(packet_buff[$num_header_words]); #set $num_header_words += 1 + #else + if_packet_info.has_tsi = false; #end if ########## Fractional Time ########## #if $pred & $tsf_p - #if not $set_has_time_spec - metadata.has_time_spec = true; - #set $set_has_time_spec = True - #end if - #set $num_header_words += 1 - metadata.time_spec.set_ticks($(XE_MACRO)(header_buff[$num_header_words]), tick_rate); - #set $num_header_words += 1 + if_packet_info.has_tsf = true; + unpack_uint64_$(suffix)(if_packet_info.tsf, packet_buff+$num_header_words); + #set $num_header_words += 2 + #else + if_packet_info.has_tsf = false; #end if ########## Trailer ########## #if $pred & $tlr_p + if_packet_info.has_tlr = true; + if_packet_info.tlr = $(XE_MACRO)(packet_buff[$num_header_words+packet_words32]); #set $num_trailer_words = 1; #else + if_packet_info.has_tlr = false; #set $num_trailer_words = 0; #end if ########## Variables ########## - num_header_words32 = $num_header_words; - num_payload_words32 = packet_words32 - $($num_header_words + $num_trailer_words); + //another failure case + if (packet_words32 < $($num_header_words + $num_trailer_words)) + throw std::runtime_error("bad vrt header or invalid packet length"); + if_packet_info.num_header_words32 = $num_header_words; + if_packet_info.num_payload_words32 = packet_words32 - $($num_header_words + $num_trailer_words); break; #end for } diff --git a/host/lib/transport/udp_zero_copy_asio.cpp b/host/lib/transport/udp_zero_copy_asio.cpp index c3c02707e..7f9292d24 100644 --- a/host/lib/transport/udp_zero_copy_asio.cpp +++ b/host/lib/transport/udp_zero_copy_asio.cpp @@ -27,7 +27,8 @@ using namespace uhd::transport; /*********************************************************************** * Constants **********************************************************************/ -static const size_t MIN_SOCK_BUFF_SIZE = size_t(100e3); +//enough buffering for half a second of samples at full rate on usrp2 +static const size_t MIN_SOCK_BUFF_SIZE = size_t(sizeof(boost::uint32_t) * 25e6 * 0.5); static const size_t MAX_DGRAM_SIZE = 1500; //assume max size on send and recv static const double RECV_TIMEOUT = 0.1; //100 ms @@ -159,6 +160,12 @@ template<typename Opt> static void resize_buff_helper( //otherwise, ensure that the buffer is at least the minimum size else if (udp_trans->get_buff_size<Opt>() < MIN_SOCK_BUFF_SIZE){ resize_buff_helper<Opt>(udp_trans, MIN_SOCK_BUFF_SIZE, name); + if (udp_trans->get_buff_size<Opt>() < MIN_SOCK_BUFF_SIZE){ + std::cerr << boost::format( + "Warning: the %s buffer size is smaller than the recommended size of %d bytes.\n" + " See the USRP2 application notes on buffer resizing." + ) % name % MIN_SOCK_BUFF_SIZE << std::endl; + } } } diff --git a/host/lib/transport/vrt_packet_handler.hpp b/host/lib/transport/vrt_packet_handler.hpp index 8dfc7b3d0..68edeb1e1 100644 --- a/host/lib/transport/vrt_packet_handler.hpp +++ b/host/lib/transport/vrt_packet_handler.hpp @@ -20,48 +20,51 @@ #include <uhd/config.hpp> #include <uhd/device.hpp> +#include <uhd/utils/assert.hpp> #include <uhd/types/io_type.hpp> #include <uhd/types/otw_type.hpp> #include <uhd/types/metadata.hpp> -#include <uhd/transport/vrt.hpp> +#include <uhd/transport/vrt_if_packet.hpp> #include <uhd/transport/convert_types.hpp> #include <uhd/transport/zero_copy.hpp> -#include <boost/asio/buffer.hpp> #include <boost/function.hpp> #include <stdexcept> #include <iostream> +#include <vector> namespace vrt_packet_handler{ /*********************************************************************** * vrt packet handler for recv **********************************************************************/ + typedef std::vector<uhd::transport::managed_recv_buffer::sptr> managed_recv_buffs_t; + typedef boost::function<bool(managed_recv_buffs_t &)> get_recv_buffs_t; + struct recv_state{ + //width of the receiver in channels + size_t width; + //init the expected seq number - size_t next_packet_seq; + std::vector<size_t> next_packet_seq; //state variables to handle fragments - uhd::transport::managed_recv_buffer::sptr managed_buff; - boost::asio::const_buffer copy_buff; + managed_recv_buffs_t managed_buffs; + std::vector<const boost::uint8_t *> copy_buffs; + size_t size_of_copy_buffs; size_t fragment_offset_in_samps; - recv_state(void){ - //first expected seq is zero - next_packet_seq = 0; - - //initially empty copy buffer - copy_buff = boost::asio::buffer("", 0); + recv_state(size_t width): + width(width), + next_packet_seq(width, 0), + managed_buffs(width), + copy_buffs(width, NULL), + size_of_copy_buffs(0), + fragment_offset_in_samps(0) + { + /* NOP */ } }; - typedef boost::function<uhd::transport::managed_recv_buffer::sptr(void)> get_recv_buff_t; - - typedef boost::function<void(uhd::transport::managed_recv_buffer::sptr)> recv_cb_t; - - static UHD_INLINE void recv_cb_nop(uhd::transport::managed_recv_buffer::sptr){ - /* NOP */ - } - /******************************************************************* * Unpack a received vrt header and set the copy buffer. * - helper function for vrt_packet_handler::_recv1 @@ -74,33 +77,39 @@ namespace vrt_packet_handler{ vrt_unpacker_type vrt_unpacker, size_t vrt_header_offset_words32 ){ - size_t num_packet_words32 = state.managed_buff->size()/sizeof(boost::uint32_t); + size_t num_packet_words32 = state.managed_buffs[0]->size()/sizeof(boost::uint32_t); if (num_packet_words32 <= vrt_header_offset_words32){ - state.copy_buff = boost::asio::buffer("", 0); + state.size_of_copy_buffs = 0; return; //must exit here after setting the buffer } - const boost::uint32_t *vrt_hdr = state.managed_buff->cast<const boost::uint32_t *>() + vrt_header_offset_words32; - size_t num_header_words32_out, num_payload_words32_out, packet_count_out; - vrt_unpacker( - metadata, //output - vrt_hdr, //input - num_header_words32_out, //output - num_payload_words32_out, //output - num_packet_words32, //input - packet_count_out, //output - tick_rate - ); - //handle the packet count / sequence number - if (packet_count_out != state.next_packet_seq){ - std::cerr << "S" << (packet_count_out - state.next_packet_seq)%16; + //vrt unpack each managed buffer + uhd::transport::vrt::if_packet_info_t if_packet_info; + for (size_t i = 0; i < state.width; i++){ + const boost::uint32_t *vrt_hdr = state.managed_buffs[i]->cast<const boost::uint32_t *>() + vrt_header_offset_words32; + if_packet_info.num_packet_words32 = num_packet_words32 - vrt_header_offset_words32; + vrt_unpacker(vrt_hdr, if_packet_info); + + //handle the packet count / sequence number + if (if_packet_info.packet_count != state.next_packet_seq[i]){ + std::cerr << "S" << (if_packet_info.packet_count - state.next_packet_seq[i])%16; + } + state.next_packet_seq[i] = (if_packet_info.packet_count+1)%16; + + //setup the buffer to point to the data + state.copy_buffs[i] = reinterpret_cast<const boost::uint8_t *>(vrt_hdr + if_packet_info.num_header_words32); + + //store the minimum payload length into the copy buffer length + size_t num_payload_bytes = if_packet_info.num_payload_words32*sizeof(boost::uint32_t); + if (i == 0 or state.size_of_copy_buffs > num_payload_bytes){ + state.size_of_copy_buffs = num_payload_bytes; + } } - state.next_packet_seq = (packet_count_out+1)%16; - //setup the buffer to point to the data - state.copy_buff = boost::asio::buffer( - vrt_hdr + num_header_words32_out, - num_payload_words32_out*sizeof(boost::uint32_t) + //store the last vrt info into the metadata + metadata.has_time_spec = if_packet_info.has_tsi and if_packet_info.has_tsf; + metadata.time_spec = uhd::time_spec_t( + time_t(if_packet_info.tsi), size_t(if_packet_info.tsf), tick_rate ); } @@ -111,24 +120,24 @@ namespace vrt_packet_handler{ template<typename vrt_unpacker_type> static UHD_INLINE size_t _recv1( recv_state &state, - void *recv_mem, + const std::vector<void *> &buffs, + size_t offset_bytes, size_t total_samps, uhd::rx_metadata_t &metadata, const uhd::io_type_t &io_type, const uhd::otw_type_t &otw_type, double tick_rate, vrt_unpacker_type vrt_unpacker, - const get_recv_buff_t &get_recv_buff, + const get_recv_buffs_t &get_recv_buffs, //use these two params to handle a layer above vrt - size_t vrt_header_offset_words32, - const recv_cb_t &recv_cb + size_t vrt_header_offset_words32 ){ + metadata.has_time_spec = false; //false unless set in the helper + //perform a receive if no rx data is waiting to be copied - if (boost::asio::buffer_size(state.copy_buff) == 0){ + if (state.size_of_copy_buffs == 0){ state.fragment_offset_in_samps = 0; - state.managed_buff = get_recv_buff(); - if (state.managed_buff.get() == NULL) return 0; - recv_cb(state.managed_buff); //callback before vrt unpack + if (not get_recv_buffs(state.managed_buffs)) return 0; try{ _recv1_helper( state, metadata, tick_rate, vrt_unpacker, vrt_header_offset_words32 @@ -141,28 +150,30 @@ namespace vrt_packet_handler{ //extract the number of samples available to copy size_t bytes_per_item = otw_type.get_sample_size(); - size_t bytes_available = boost::asio::buffer_size(state.copy_buff); - size_t num_samps = std::min(total_samps, bytes_available/bytes_per_item); + size_t bytes_available = state.size_of_copy_buffs; + size_t nsamps_to_copy = std::min(total_samps, bytes_available/bytes_per_item); + size_t bytes_to_copy = nsamps_to_copy*bytes_per_item; + + for (size_t i = 0; i < state.width; i++){ + //copy-convert the samples from the recv buffer + uhd::transport::convert_otw_type_to_io_type( + state.copy_buffs[i], otw_type, + reinterpret_cast<boost::uint8_t *>(buffs[i]) + offset_bytes, + io_type, nsamps_to_copy + ); + + //update the rx copy buffer to reflect the bytes copied + state.copy_buffs[i] += bytes_to_copy; + } + //update the copy buffer's availability + state.size_of_copy_buffs -= bytes_to_copy; //setup the fragment flags and offset - metadata.more_fragments = total_samps < num_samps; + metadata.more_fragments = state.size_of_copy_buffs != 0; metadata.fragment_offset = state.fragment_offset_in_samps; - state.fragment_offset_in_samps += num_samps; //set for next call - - //copy-convert the samples from the recv buffer - uhd::transport::convert_otw_type_to_io_type( - boost::asio::buffer_cast<const void*>(state.copy_buff), otw_type, - recv_mem, io_type, num_samps - ); - - //update the rx copy buffer to reflect the bytes copied - size_t bytes_copied = num_samps*bytes_per_item; - state.copy_buff = boost::asio::buffer( - boost::asio::buffer_cast<const boost::uint8_t*>(state.copy_buff) + bytes_copied, - bytes_available - bytes_copied - ); + state.fragment_offset_in_samps += nsamps_to_copy; //set for next call - return num_samps; + return nsamps_to_copy; } /******************************************************************* @@ -171,21 +182,18 @@ namespace vrt_packet_handler{ template<typename vrt_unpacker_type> static UHD_INLINE size_t recv( recv_state &state, - const boost::asio::mutable_buffer &buff, + const std::vector<void *> &buffs, + const size_t total_num_samps, uhd::rx_metadata_t &metadata, uhd::device::recv_mode_t recv_mode, const uhd::io_type_t &io_type, const uhd::otw_type_t &otw_type, double tick_rate, vrt_unpacker_type vrt_unpacker, - const get_recv_buff_t &get_recv_buff, + const get_recv_buffs_t &get_recv_buffs, //use these two params to handle a layer above vrt - size_t vrt_header_offset_words32 = 0, - const recv_cb_t& recv_cb = &recv_cb_nop + size_t vrt_header_offset_words32 = 0 ){ - metadata = uhd::rx_metadata_t(); //init the metadata - const size_t total_num_samps = boost::asio::buffer_size(buff)/io_type.size; - switch(recv_mode){ //////////////////////////////////////////////////////////////// @@ -193,15 +201,14 @@ namespace vrt_packet_handler{ //////////////////////////////////////////////////////////////// return _recv1( state, - boost::asio::buffer_cast<void *>(buff), + buffs, 0, total_num_samps, metadata, io_type, otw_type, tick_rate, vrt_unpacker, - get_recv_buff, - vrt_header_offset_words32, - recv_cb + get_recv_buffs, + vrt_header_offset_words32 ); } @@ -213,15 +220,14 @@ namespace vrt_packet_handler{ while(accum_num_samps < total_num_samps){ size_t num_samps = _recv1( state, - boost::asio::buffer_cast<boost::uint8_t *>(buff) + (accum_num_samps*io_type.size), + buffs, accum_num_samps*io_type.size, total_num_samps - accum_num_samps, (accum_num_samps == 0)? metadata : tmp_md, //only the first metadata gets kept io_type, otw_type, tick_rate, vrt_unpacker, - get_recv_buff, - vrt_header_offset_words32, - recv_cb + get_recv_buffs, + vrt_header_offset_words32 ); if (num_samps == 0) break; //had a recv timeout or error, break loop accum_num_samps += num_samps; @@ -236,23 +242,18 @@ namespace vrt_packet_handler{ /*********************************************************************** * vrt packet handler for send **********************************************************************/ + typedef std::vector<uhd::transport::managed_send_buffer::sptr> managed_send_buffs_t; + typedef boost::function<bool(managed_send_buffs_t &)> get_send_buffs_t; + struct send_state{ //init the expected seq number size_t next_packet_seq; - send_state(void){ - next_packet_seq = 0; + send_state(void) : next_packet_seq(0){ + /* NOP */ } }; - typedef boost::function<uhd::transport::managed_send_buffer::sptr(void)> get_send_buff_t; - - typedef boost::function<void(uhd::transport::managed_send_buffer::sptr)> send_cb_t; - - static UHD_INLINE void send_cb_nop(uhd::transport::managed_send_buffer::sptr){ - /* NOP */ - } - /******************************************************************* * Pack a vrt header, copy-convert the data, and send it. * - helper function for vrt_packet_handler::send @@ -260,46 +261,42 @@ namespace vrt_packet_handler{ template<typename vrt_packer_type> static UHD_INLINE void _send1( send_state &state, - const void *send_mem, + const std::vector<const void *> &buffs, + size_t offset_bytes, size_t num_samps, - const uhd::tx_metadata_t &metadata, + uhd::transport::vrt::if_packet_info_t &if_packet_info, const uhd::io_type_t &io_type, const uhd::otw_type_t &otw_type, - double tick_rate, vrt_packer_type vrt_packer, - const get_send_buff_t &get_send_buff, - size_t vrt_header_offset_words32, - const send_cb_t& send_cb + const get_send_buffs_t &get_send_buffs, + size_t vrt_header_offset_words32 ){ - //get a new managed send buffer - uhd::transport::managed_send_buffer::sptr send_buff = get_send_buff(); - boost::uint32_t *tx_mem = send_buff->cast<boost::uint32_t *>() + vrt_header_offset_words32; - - size_t num_header_words32, num_packet_words32; - size_t packet_count = state.next_packet_seq++; - - //pack metadata into a vrt header - vrt_packer( - metadata, //input - tx_mem, //output - num_header_words32, //output - num_samps, //input - num_packet_words32, //output - packet_count, //input - tick_rate - ); - - //copy-convert the samples into the send buffer - uhd::transport::convert_io_type_to_otw_type( - send_mem, io_type, - tx_mem + num_header_words32, otw_type, - num_samps - ); - - send_cb(send_buff); //callback after memory filled + //load the rest of the if_packet_info in here + if_packet_info.num_payload_words32 = (num_samps*otw_type.get_sample_size())/sizeof(boost::uint32_t); + if_packet_info.packet_count = state.next_packet_seq++; + + //get send buffers for each channel + managed_send_buffs_t send_buffs(buffs.size()); + UHD_ASSERT_THROW(get_send_buffs(send_buffs)); + + for (size_t i = 0; i < buffs.size(); i++){ + //calculate pointers with offsets to io and otw memory + const boost::uint8_t *io_mem = reinterpret_cast<const boost::uint8_t *>(buffs[i]) + offset_bytes; + boost::uint32_t *otw_mem = send_buffs[i]->cast<boost::uint32_t *>() + vrt_header_offset_words32; + + //pack metadata into a vrt header + vrt_packer(otw_mem, if_packet_info); + + //copy-convert the samples into the send buffer + uhd::transport::convert_io_type_to_otw_type( + io_mem, io_type, + otw_mem + if_packet_info.num_header_words32, otw_type, + num_samps + ); - //commit the samples to the zero-copy interface - send_buff->commit(num_packet_words32*sizeof(boost::uint32_t)); + //commit the samples to the zero-copy interface + send_buffs[i]->commit(if_packet_info.num_packet_words32*sizeof(boost::uint32_t)); + } } /******************************************************************* @@ -308,20 +305,27 @@ namespace vrt_packet_handler{ template<typename vrt_packer_type> static UHD_INLINE size_t send( send_state &state, - const boost::asio::const_buffer &buff, + const std::vector<const void *> &buffs, + const size_t total_num_samps, const uhd::tx_metadata_t &metadata, uhd::device::send_mode_t send_mode, const uhd::io_type_t &io_type, const uhd::otw_type_t &otw_type, double tick_rate, vrt_packer_type vrt_packer, - const get_send_buff_t &get_send_buff, + const get_send_buffs_t &get_send_buffs, size_t max_samples_per_packet, //use these two params to handle a layer above vrt - size_t vrt_header_offset_words32 = 0, - const send_cb_t& send_cb = &send_cb_nop + size_t vrt_header_offset_words32 = 0 ){ - const size_t total_num_samps = boost::asio::buffer_size(buff)/io_type.size; + //translate the metadata to vrt if packet info + uhd::transport::vrt::if_packet_info_t if_packet_info; + if_packet_info.has_sid = false; + if_packet_info.has_cid = false; + if_packet_info.has_tlr = false; + if_packet_info.tsi = boost::uint32_t(metadata.time_spec.get_full_secs()); + if_packet_info.tsf = boost::uint64_t(metadata.time_spec.get_tick_count(tick_rate)); + if (total_num_samps <= max_samples_per_packet) send_mode = uhd::device::SEND_MODE_ONE_PACKET; switch(send_mode){ @@ -329,17 +333,22 @@ namespace vrt_packet_handler{ case uhd::device::SEND_MODE_ONE_PACKET:{ //////////////////////////////////////////////////////////////// size_t num_samps = std::min(total_num_samps, max_samples_per_packet); + + //fill in parts of the packet info overwrote in full buff mode + if_packet_info.has_tsi = metadata.has_time_spec; + if_packet_info.has_tsf = metadata.has_time_spec; + if_packet_info.sob = metadata.start_of_burst; + if_packet_info.eob = metadata.end_of_burst; + _send1( state, - boost::asio::buffer_cast<const void *>(buff), + buffs, 0, num_samps, - metadata, + if_packet_info, io_type, otw_type, - tick_rate, vrt_packer, - get_send_buff, - vrt_header_offset_words32, - send_cb + get_send_buffs, + vrt_header_offset_words32 ); return num_samps; } @@ -352,29 +361,25 @@ namespace vrt_packet_handler{ static const size_t first_fragment_index = 0; const size_t final_fragment_index = num_fragments-1; - //make a rw copy of the metadata to re-flag below - uhd::tx_metadata_t md(metadata); - //loop through the following fragment indexes for (size_t n = first_fragment_index; n <= final_fragment_index; n++){ //calculate new flags for the fragments - md.has_time_spec = metadata.has_time_spec and (n == first_fragment_index); - md.start_of_burst = metadata.start_of_burst and (n == first_fragment_index); - md.end_of_burst = metadata.end_of_burst and (n == final_fragment_index); + if_packet_info.has_tsi = metadata.has_time_spec and (n == first_fragment_index); + if_packet_info.has_tsf = metadata.has_time_spec and (n == first_fragment_index); + if_packet_info.sob = metadata.start_of_burst and (n == first_fragment_index); + if_packet_info.eob = metadata.end_of_burst and (n == final_fragment_index); //send the fragment with the helper function _send1( state, - boost::asio::buffer_cast<const boost::uint8_t *>(buff) + (n*max_samples_per_packet*io_type.size), + buffs, n*max_samples_per_packet*io_type.size, (n == final_fragment_index)?(total_num_samps%max_samples_per_packet):max_samples_per_packet, - md, + if_packet_info, io_type, otw_type, - tick_rate, vrt_packer, - get_send_buff, - vrt_header_offset_words32, - send_cb + get_send_buffs, + vrt_header_offset_words32 ); } return total_num_samps; |