From da297ccf82836dc3f1e1c9a4daef8e85a5b9a4d5 Mon Sep 17 00:00:00 2001 From: michael-west Date: Thu, 13 Jan 2022 11:07:56 -0800 Subject: RFNoC: Cache and re-use host endpoints Constatntly incrementing endpoints was causing the entries in the routing table on the device to be exhausted, eventually resulting in a timeout error on control packets. Since a connection between the host and a stream endpoint on a device in a given direction is unique, the host endpoints can be cached and re-used. This change does that. Signed-off-by: michael-west --- host/lib/rfnoc/link_stream_manager.cpp | 37 +++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/host/lib/rfnoc/link_stream_manager.cpp b/host/lib/rfnoc/link_stream_manager.cpp index d73eff319..6557e316a 100644 --- a/host/lib/rfnoc/link_stream_manager.cpp +++ b/host/lib/rfnoc/link_stream_manager.cpp @@ -10,6 +10,7 @@ #include #include #include +#include using namespace uhd; using namespace uhd::rfnoc; @@ -226,16 +227,22 @@ public: _ensure_ep_is_reachable(dst_addr); // Generate a new destination (device) EPID instance - sep_id_t dst_epid = + const sep_id_t dst_epid = _epid_alloc->allocate_epid(dst_addr, *_mgmt_portal, *_ctrl_xport); if (!_mgmt_portal->get_endpoint_info(dst_epid).has_data) { throw uhd::rfnoc_error("Downstream endpoint does not support data traffic"); } - // Create a new destination (host) endpoint and EPID - sep_addr_t sw_epid_addr(_my_device_id, SEP_INST_DATA_BASE + (_data_ep_inst++)); - sep_id_t src_epid = _epid_alloc->allocate_epid(sw_epid_addr); + // Create a new source (host) endpoint and EPID + { + std::lock_guard ep_lock(_data_ep_lock); + if (_data_src_ep_map.count(dst_addr) == 0) { + _data_src_ep_map[dst_addr] = SEP_INST_DATA_BASE + _data_ep_inst++; + } + } + const sep_addr_t sw_epid_addr(_my_device_id, _data_src_ep_map[dst_addr]); + const sep_id_t src_epid = _epid_alloc->allocate_epid(sw_epid_addr); _allocated_epids.insert(src_epid); return _mb_iface.make_tx_data_transport(*_mgmt_portal, @@ -256,7 +263,7 @@ public: _ensure_ep_is_reachable(src_addr); // Generate a new source (device) EPID instance - sep_id_t src_epid = + const sep_id_t src_epid = _epid_alloc->allocate_epid(src_addr, *_mgmt_portal, *_ctrl_xport); if (!_mgmt_portal->get_endpoint_info(src_epid).has_data) { @@ -264,8 +271,14 @@ public: } // Create a new destination (host) endpoint and EPID - sep_addr_t sw_epid_addr(_my_device_id, SEP_INST_DATA_BASE + (_data_ep_inst++)); - sep_id_t dst_epid = _epid_alloc->allocate_epid(sw_epid_addr); + { + std::lock_guard ep_lock(_data_ep_lock); + if (_data_dst_ep_map.count(src_addr) == 0) { + _data_dst_ep_map[src_addr] = SEP_INST_DATA_BASE + _data_ep_inst++; + } + } + const sep_addr_t sw_epid_addr(_my_device_id, _data_dst_ep_map[src_addr]); + const sep_id_t dst_epid = _epid_alloc->allocate_epid(sw_epid_addr); _allocated_epids.insert(dst_epid); return _mb_iface.make_rx_data_transport(*_mgmt_portal, @@ -317,8 +330,18 @@ private: chdr_ctrl_endpoint::uptr _ctrl_ep; // A map of all client zero instances indexed by the destination std::map _client_zero_map; + + // Data endpoint mutex + std::mutex _data_ep_lock; // Data endpoint instance sep_inst_t _data_ep_inst; + + // Maps to cache local data endpoints for re-use. Assumes each connection + // between the host and a stream endpoint on a given device is unique for + // each direction. Re-using enpdoints is needed because the routing table + // in the FPGA is limited in how many entries can be made. + std::map _data_src_ep_map; + std::map _data_dst_ep_map; }; link_stream_manager::uptr link_stream_manager::make( -- cgit v1.2.3