diff options
| -rw-r--r-- | host/include/uhd/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | host/include/uhd/cal/CMakeLists.txt | 12 | ||||
| -rw-r--r-- | host/include/uhd/cal/database.hpp | 139 | ||||
| -rw-r--r-- | host/lib/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | host/lib/cal/CMakeLists.txt | 13 | ||||
| -rw-r--r-- | host/lib/cal/cal_python.hpp | 61 | ||||
| -rw-r--r-- | host/lib/cal/database.cpp | 205 | ||||
| -rw-r--r-- | host/lib/rc/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | host/lib/rc/cal/test.cal | 1 | ||||
| -rw-r--r-- | host/python/uhd/usrp/cal/libtypes.py | 19 | ||||
| -rw-r--r-- | host/tests/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | host/tests/cal_database_test.cpp | 88 |
12 files changed, 542 insertions, 0 deletions
diff --git a/host/include/uhd/CMakeLists.txt b/host/include/uhd/CMakeLists.txt index 429d4fe63..f6bdd2a8f 100644 --- a/host/include/uhd/CMakeLists.txt +++ b/host/include/uhd/CMakeLists.txt @@ -6,6 +6,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # +add_subdirectory(cal) add_subdirectory(rfnoc) add_subdirectory(transport) add_subdirectory(types) diff --git a/host/include/uhd/cal/CMakeLists.txt b/host/include/uhd/cal/CMakeLists.txt new file mode 100644 index 000000000..6c8355fec --- /dev/null +++ b/host/include/uhd/cal/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# Copyright 2020 Ettus Research, a National Instruments Brand +# +# SPDX-License-Identifier: GPL-3.0-or-later +# + +UHD_INSTALL(FILES + database.hpp + DESTINATION ${INCLUDE_DIR}/uhd/cal + COMPONENT headers +) + diff --git a/host/include/uhd/cal/database.hpp b/host/include/uhd/cal/database.hpp new file mode 100644 index 000000000..ca607de06 --- /dev/null +++ b/host/include/uhd/cal/database.hpp @@ -0,0 +1,139 @@ +// +// Copyright 2020 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_LIBUHD_CAL_DATABASE_HPP +#define INCLUDED_LIBUHD_CAL_DATABASE_HPP + +#include <uhd/config.hpp> +#include <stddef.h> +#include <string> +#include <vector> + +namespace uhd { namespace usrp { namespace cal { + +//! Identify the source of calibration data, i.e., where was it stored +// +// This enum lists the sources in reverse order of priority, i.e., user-provided +// data has the highest priority, and hard-coded data from the resource compiler +// has the lowest priority. +enum class source { + NONE, //!< No calibration data available + ANY, //!< Undefined source + RC, //!< Internal Resource Compiler (i.e., hard-coded within UHD) + FLASH, //!< Stored on device flash memory, e.g. EEPROM + FILESYSTEM, //!< Stored on the local filesystem + USER //!< Provided by the user +}; + +/*! Calibration Data Storage/Retrieval Class + * + * UHD can store calibration data on disk or compiled within UHD. This class + * provides access to both locations. + * + * \section cal_db_blob Format of binary data + * + * This class can read and write binary data, but it does not verify the data + * or expect any kind of format. It simply manages BLOBs (binary large objects). + * It is up to the consumers and producers of this data to agree on a format. + * Typically, since this class stores calibration data, it will be consuming + * data that was produced by uhd::usrp::cal::container::serialize(). + * + * \section cal_db_serial Serial number and key + * + * Calibration data is indexed by two keys: An arbitrary key that describes the + * type of calibration data (e.g., "rx_iq") and a serial number. The serial + * number has to uniquely identify the device for which the calibration data was + * obtained. This can either be the serial number of the daughterboard (if the + * calibration data only relates to the daughterboard), the motherboard (for + * example, if there is no such thing as a daughterboard, or the data only + * relates to the motherboard), it can be combination of both daughterboard and + * motherboard serial (if the calibration data is only valid for a combination), + * or it can be a combination of a device serial number and a channel index + * (if a device with single serial has different channels that have separate + * characteristics). + * + * It is up to the individual device drivers which value they use for the serial + * numbers and keys. + * + * Note that the serial number is irrelevant when the data is pulled out of the + * resource compiler. By definition, it is not permitted to store data in the + * resource compiler that is specific to a certain serial number, only data that + * applies to an entire family of devices is permitted. + */ +class UHD_API database +{ +public: + + //! Return a calibration data set as a serialized string + // + // Note: the \p source_type parameter can be used to specify where to read + // cal data from. However, this class only has + // access to RC and FILESYSTEM type cal data. ANY + // will pick FILESYSTEM data if both are available, + // and RC data if only RC data is available. + // \param key The calibration type key (e.g., "rx_iq") + // \param serial The serial number of the device this data is for. See also + // \ref cal_db_serial + // \param source_type Where to read the calibration data from. See comments + // above. For anything other than RC, FILESYSTEM, or ANY, + // this will always throw a uhd::key_error because this + // class does not have access to user data or EEPROM data. + // + // \throws uhd::key_error if no calibration data is found matching the source + // type. + static std::vector<uint8_t> read_cal_data(const std::string& key, + const std::string& serial, + const source source_type = source::ANY); + + //! Check if calibration data exists for a given source type + // + // This can be called before calling read_cal_data() to avoid having to + // catch an exception. If \p source_type is FILESYSTEM, then it will only + // return true if a file is found with the appropriate cal data. The same + // is true for RC. If \p is ANY, then having either RC or FILESYSTEM data + // will yield true. + // + // \param key The calibration type key (e.g., "rx_iq") + // \param serial The serial number of the device this data is for. See also + // \ref cal_db_serial + // \param source_type Where to read the calibration data from. For anything + // other than RC, FILESYSTEM, or ANY, this will always + // return false because this class does not have access + // to user data or EEPROM data. + // \return true if calibration data is available that matches this key/serial + // pair. + static bool has_cal_data(const std::string& key, + const std::string& serial, + const source source_type = source::ANY); + + //! Store calibration data to the local filesystem database + // + // This implies a source type of FILESYSTEM. Note that writing the data does + // not apply it to a currently running UHD session. Devices will typically + // load calibration data at initialization time, and thus this call will + // take effect only for future UHD sessions. + // + // If calibration data for this key/serial pair already exists in the + // database, the original data will be backed up by renaming the original + // file from `filename.cal` to `filename.cal.$TIMESTAMP`. Alternatively, a + // custom extension can be chosen instead of `$TIMESTAMP`. + // + // \param key The calibration type key (e.g., "rx_iq") + // \param serial The serial number of the device this data is for. See also + // \ref cal_db_serial + // \param cal_data The calibration data to be written + // \param backup_ext A custom extension for backing up calibration data. If + // left empty, a POSIX timestamp is used. + static void write_cal_data(const std::string& key, + const std::string& serial, + const std::vector<uint8_t>& cal_data, + const std::string& backup_ext = ""); +}; + + +}}} // namespace uhd::usrp::cal + +#endif /* INCLUDED_LIBUHD_CAL_DATABASE_HPP */ diff --git a/host/lib/CMakeLists.txt b/host/lib/CMakeLists.txt index 4d747c9c7..5a3377a55 100644 --- a/host/lib/CMakeLists.txt +++ b/host/lib/CMakeLists.txt @@ -82,6 +82,7 @@ LIBUHD_REGISTER_COMPONENT("DPDK" ENABLE_DPDK ON "ENABLE_MPMD;DPDK_FOUND" OFF OFF # Include subdirectories (different than add) ######################################################################## INCLUDE_SUBDIRECTORY(include) +INCLUDE_SUBDIRECTORY(cal) INCLUDE_SUBDIRECTORY(ic_reg_maps) INCLUDE_SUBDIRECTORY(types) INCLUDE_SUBDIRECTORY(convert) diff --git a/host/lib/cal/CMakeLists.txt b/host/lib/cal/CMakeLists.txt new file mode 100644 index 000000000..5c2c8a617 --- /dev/null +++ b/host/lib/cal/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright 2020 Ettus Research, a National Instruments Brand +# +# SPDX-License-Identifier: GPL-3.0-or-later +# + +######################################################################## +# This file included, use CMake directory variables +######################################################################## +LIBUHD_APPEND_SOURCES( + ${CMAKE_CURRENT_SOURCE_DIR}/database.cpp +) + diff --git a/host/lib/cal/cal_python.hpp b/host/lib/cal/cal_python.hpp new file mode 100644 index 000000000..0fe87046f --- /dev/null +++ b/host/lib/cal/cal_python.hpp @@ -0,0 +1,61 @@ +// +// Copyright 2020 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#ifndef INCLUDED_UHD_CAL_PYTHON_HPP +#define INCLUDED_UHD_CAL_PYTHON_HPP + +#include <uhd/cal/database.hpp> + +std::vector<uint8_t> pybytes_to_vector(const py::bytes& data) +{ + const std::string data_str = std::string(data); + return std::vector<uint8_t>(data_str.cbegin(), data_str.cend()); +} + +py::bytes vector_to_pybytes(const std::vector<uint8_t>& data) +{ + return py::bytes(std::string(data.cbegin(), data.cend())); +} + +void export_cal(py::module& m) +{ + using namespace uhd::usrp::cal; + + // Cal Database + using database = uhd::usrp::cal::database; + using source = uhd::usrp::cal::source; + + py::enum_<source>(m, "source") + .value("ANY", source::ANY) + .value("RC", source::RC) + .value("FILESYSTEM", source::FILESYSTEM) + .value("FLASH", source::FLASH) + .value("USER", source::USER) + .value("NONE", source::NONE); + + py::class_<database>(m, "database") + .def_static("read_cal_data", + [](const std::string& key, + const std::string& serial, + const source source_type) { + return vector_to_pybytes( + database::read_cal_data(key, serial, source_type)); + }, + py::arg("key"), + py::arg("serial"), + py::arg("source_type") = source::ANY) + .def_static("has_cal_data", + &database::has_cal_data, + py::arg("key"), + py::arg("serial"), + py::arg("source_type") = source::ANY) + .def_static("write_cal_data", + [](const std::string& key, const std::string& serial, const py::bytes data) { + database::write_cal_data(key, serial, pybytes_to_vector(data)); + }); +} + +#endif /* INCLUDED_UHD_CAL_PYTHON_HPP */ diff --git a/host/lib/cal/database.cpp b/host/lib/cal/database.cpp new file mode 100644 index 000000000..16fcd4b71 --- /dev/null +++ b/host/lib/cal/database.cpp @@ -0,0 +1,205 @@ +// +// Copyright 2020 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include <uhd/cal/database.hpp> +#include <uhd/exception.hpp> +#include <uhd/utils/log.hpp> +#include <uhd/utils/paths.hpp> +#include <cmrc/cmrc.hpp> +#include <boost/filesystem.hpp> +#include <ctime> +#include <fstream> + +CMRC_DECLARE(rc); + +using namespace uhd::usrp::cal; +namespace rc = cmrc::rc; +namespace fs = boost::filesystem; + +namespace { +constexpr char LOG_ID[] = "CAL::DATABASE"; +constexpr char CAL_EXT[] = ".cal"; +//! This value is just for sanity checking. We pick a value (in bytes) that we +// are guaranteed to never exceed. Its only purpose is to avoid loading files +// that can't possibly be valid cal data based on the filesize. This can avoid +// someone bringing down a UHD session by trying to import a huge file, because +// we first load it entirely into heap space, and then deserialize it from there. +constexpr size_t CALDATA_MAX_SIZE = 10 * 1024 * 1024; // 10 MiB + + +//! Map a cal resource key into a source::RC path name +std::string get_cal_path_rc(const std::string& key) +{ + return std::string("cal/") + key + CAL_EXT; +} + +//! Return true if a cal data resource with given key exists +bool has_cal_data_rc(const std::string& key) +{ + auto fs = rc::get_filesystem(); + return fs.is_file(get_cal_path_rc(key)); +} + +//! Return a byte array for a given cal resource +std::vector<uint8_t> get_cal_data_rc(const std::string& key) +{ + try { + auto fs = rc::get_filesystem(); + auto file = fs.open(get_cal_path_rc(key)); + return std::vector<uint8_t>(file.cbegin(), file.cend()); + } catch (const std::system_error&) { + throw uhd::key_error(std::string("Unable to open resource with key: ") + key); + } +} + +void check_or_create_dir(fs::path dir) +{ + if (fs::exists(dir)) { + if (fs::is_directory(dir)) { + return; + } + UHD_LOG_ERROR(LOG_ID, "Path exists, but is not a directory: " << dir); + throw uhd::runtime_error("Path exists, but is not a directory!"); + } + + if (!fs::create_directory(dir)) { + UHD_LOG_ERROR(LOG_ID, "Cannot create cal data directory: " << dir); + throw uhd::runtime_error("Cannot create cal data directory!"); + } + UHD_LOG_DEBUG(LOG_ID, "Created directory: " << dir); +} + +//! Make sure the calibration storage directory exists. +// +// The path returned by uhd::get_cal_data_path() might not exist (e.g., when run +// for the first time). This directory must be created before we try writing to +// it, or we won't be able to open the file. +// +// C++ doesn't have a mkdir -p equivalent, so we check the parent directory and +// the directory itself, in that order. Most of the time, the cal data path is +// in $XDG_DATA_HOME/uhd/cal_data. We assume that $XDG_DATA_HOME exists, and +// then first check $XDG_DATA_HOME/uhd, then $XDG_DATA_HOME/uhd/cal_data. +// +// This will not work if the user sets $UHD_CAL_DATA_PATH to an arbitrary path +// that requires multiple levels of directories to be created, but they will get +// a clear error message in that case. +void assert_cal_dir_exists() +{ + const auto cal_path = fs::path(uhd::get_cal_data_path()); + if (!cal_path.parent_path().empty()) { + check_or_create_dir(cal_path.parent_path()); + } + check_or_create_dir(cal_path); +} + + +//! Map a cal resource key into a filesystem path name (relative to get_cal_data_path()) +std::string get_cal_path_fs(const std::string& key, const std::string& serial) +{ + return key + "_" + serial + CAL_EXT; +} + +//! Return true if a cal data resource with given key exists +bool has_cal_data_fs(const std::string& key, const std::string& serial) +{ + auto const cal_file_path = + fs::path(uhd::get_cal_data_path()) / get_cal_path_fs(key, serial); + UHD_LOG_TRACE(LOG_ID, "Checking for file at " << cal_file_path.string()); + // We might want to check readability also + return fs::exists(cal_file_path) && fs::is_regular_file(cal_file_path); +} + +//! Return a byte array for a given filesystem resource +std::vector<uint8_t> get_cal_data_fs(const std::string& key, const std::string& serial) +{ + if (!has_cal_data_fs(key, serial)) { + throw uhd::key_error( + std::string("Cannot find cal file for key=") + key + ", serial=" + serial); + } + const auto cal_file_path = + fs::path(uhd::get_cal_data_path()) / get_cal_path_fs(key, serial); + // We read the filesize first to do a sanity check (is this file small + // enough to reasonably be cal data?) and also to pre-allocate heap space in + // which we'll load the full data for future deserialization. + const size_t filesize = fs::file_size(cal_file_path); + if (filesize > CALDATA_MAX_SIZE) { + throw uhd::key_error( + std::string("The following cal data file exceeds maximum size limitations: ") + + cal_file_path.string()); + } + std::vector<uint8_t> result(filesize, 0); + std::ifstream file(cal_file_path.string(), std::ios::binary); + UHD_LOG_TRACE(LOG_ID, "Reading " << filesize << " bytes from " << cal_file_path); + file.read(reinterpret_cast<char*>(&result[0]), filesize); + return result; +} + +} // namespace + +std::vector<uint8_t> database::read_cal_data( + const std::string& key, const std::string& serial, const source source_type) +{ + if (source_type == source::FILESYSTEM || source_type == source::ANY) { + if (has_cal_data_fs(key, serial)) { + return get_cal_data_fs(key, serial); + } + } + + if (source_type == source::RC || source_type == source::ANY) { + if (has_cal_data_rc(key)) { + return get_cal_data_rc(key); + } + } + + const std::string err_msg = + std::string("Calibration Data not found for: key=") + key + ", serial=" + serial; + UHD_LOG_ERROR(LOG_ID, err_msg); + throw uhd::key_error(err_msg); +} + +bool database::has_cal_data( + const std::string& key, const std::string& serial, const source source_type) +{ + if (source_type == source::FILESYSTEM || source_type == source::ANY) { + if (has_cal_data_fs(key, serial)) { + return true; + } + } + + if (source_type == source::RC || source_type == source::ANY) { + if (has_cal_data_rc(key)) { + return true; + } + } + + return false; +} + +void database::write_cal_data(const std::string& key, + const std::string& serial, + const std::vector<uint8_t>& cal_data, + const std::string& backup_ext) +{ + assert_cal_dir_exists(); + + const auto cal_file_path = + (fs::path(uhd::get_cal_data_path()) / get_cal_path_fs(key, serial)).string(); + + if (fs::exists(cal_file_path)) { + const auto ext = backup_ext.empty() ? std::to_string(time(NULL)) : backup_ext; + const auto cal_file_path_backup = fs::path(uhd::get_cal_data_path()) + / (get_cal_path_fs(key, serial) + "." + ext); + UHD_LOG_WARNING(LOG_ID, + "Calibration data already exists for key: `" + << key << "' serial: `" << serial + << "'. Backing up to: " << cal_file_path_backup); + fs::rename(fs::path(cal_file_path), cal_file_path_backup); + } + + std::ofstream file(cal_file_path, std::ios::binary); + UHD_LOG_DEBUG(LOG_ID, "Writing to " << cal_file_path); + file.write(reinterpret_cast<const char*>(cal_data.data()), cal_data.size()); +} diff --git a/host/lib/rc/CMakeLists.txt b/host/lib/rc/CMakeLists.txt index 1595e7e85..d2a01bffb 100644 --- a/host/lib/rc/CMakeLists.txt +++ b/host/lib/rc/CMakeLists.txt @@ -8,4 +8,5 @@ include(CMakeRC) cmrc_add_resource_library(uhd-resources ALIAS uhd_rc NAMESPACE rc + cal/test.cal ) diff --git a/host/lib/rc/cal/test.cal b/host/lib/rc/cal/test.cal new file mode 100644 index 000000000..861c2bb2c --- /dev/null +++ b/host/lib/rc/cal/test.cal @@ -0,0 +1 @@ +rc::cal::test_data
\ No newline at end of file diff --git a/host/python/uhd/usrp/cal/libtypes.py b/host/python/uhd/usrp/cal/libtypes.py new file mode 100644 index 000000000..754028c24 --- /dev/null +++ b/host/python/uhd/usrp/cal/libtypes.py @@ -0,0 +1,19 @@ +# +# Copyright 2020 Ettus Research, a National Instruments Brand +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +""" +Import cal types into Python +""" + +from ... import libpyuhd as lib + +# Disable PyLint because we want to make it look like the following classes are +# defined in Python, but they're just renames of lib types. They therefore +# follow name conventions for Python classes, not for global constants. +# pylint: disable=invalid-name +# database is a class, but we treat it like a namespace, i.e., a submodule +database = lib.cal.database +Source = lib.cal.source +# pylint: enable=invalid-name diff --git a/host/tests/CMakeLists.txt b/host/tests/CMakeLists.txt index bf47f6638..f987aa194 100644 --- a/host/tests/CMakeLists.txt +++ b/host/tests/CMakeLists.txt @@ -25,6 +25,7 @@ set(test_sources buffer_test.cpp byteswap_test.cpp cast_test.cpp + cal_database_test.cpp chdr_test.cpp constrained_device_args_test.cpp convert_test.cpp diff --git a/host/tests/cal_database_test.cpp b/host/tests/cal_database_test.cpp new file mode 100644 index 000000000..cd189138b --- /dev/null +++ b/host/tests/cal_database_test.cpp @@ -0,0 +1,88 @@ +// +// Copyright 2020 Ettus Research, a National Instruments Brand +// +// SPDX-License-Identifier: GPL-3.0-or-later +// + +#include <uhd/cal/database.hpp> +#include <uhd/utils/paths.hpp> +#include <stdlib.h> // putenv or _putenv +#include <boost/filesystem.hpp> +#include <boost/test/unit_test.hpp> +#include <iostream> + +using namespace uhd::usrp::cal; +namespace fs = boost::filesystem; + +BOOST_AUTO_TEST_CASE(test_rc) +{ + BOOST_CHECK(!database::has_cal_data("does_not_exist", "1234", source::RC)); + BOOST_CHECK(database::has_cal_data("test", "1234", source::RC)); + BOOST_CHECK(database::has_cal_data("test", "1234")); + BOOST_CHECK(!database::has_cal_data("test", "1234", source::FILESYSTEM)); + + const auto test_data = database::read_cal_data("test", "", source::RC); + const std::string test_str(test_data.cbegin(), test_data.cend()); + // The expected string is also in the test.cal file. We could, in this test, + // open that file and dynamically generate the expected data, but let's not. + // First, that adds complexity here, and second, both test.cal and this test + // are hashed with the same git commit, and thus we also test the integrity + // of test.cal. + BOOST_CHECK_EQUAL(test_str, "rc::cal::test_data"); +} + +BOOST_AUTO_TEST_CASE(test_fs) +{ + BOOST_CHECK(!database::has_cal_data("does_not_exist", "1234", source::FILESYSTEM)); + + const auto tmp_dir = uhd::get_tmp_path(); + const auto tmp_cal_path = fs::path(tmp_dir) / "CAL_TEST"; + boost::system::error_code ec; + fs::create_directory(tmp_cal_path, ec); + if (ec) { + std::cout << "WARNING: Could not create temp cal path. Skipping test." + << std::endl; + return; + } + std::cout << "Using temporary cal path: " << tmp_cal_path << std::endl; + + // Now we do a non-portable hack to override the cal path during runtime: +#ifdef UHD_PLATFORM_WIN32 + const std::string putenv_str = + std::string("UHD_CAL_DATA_PATH=") + tmp_cal_path.string(); + _putenv(putenv_str.c_str()); +#else + setenv("UHD_CAL_DATA_PATH", tmp_cal_path.string().c_str(), /* overwrite */ 1); +#endif + + // Because of the hack, we won't fail if it didn't work, but instead, print + // a warning and exit this test. Running the following lines requires the + // hack to succeed. + if (uhd::get_cal_data_path() != tmp_cal_path) { + std::cout << "WARNING: Unable to update UHD_CAL_DATA_PATH! get_cal_data_path(): " + << uhd::get_cal_data_path() << std::endl; + return; + } + + std::vector<uint8_t> mock_data{1, 2, 3, 4, 5}; + database::write_cal_data("mock_data", "1234", mock_data); + auto mock_data_rb = database::read_cal_data("mock_data", "1234"); + BOOST_CHECK_EQUAL_COLLECTIONS( + mock_data.begin(), mock_data.end(), mock_data_rb.begin(), mock_data_rb.end()); + + BOOST_CHECK(!database::has_cal_data("mock_data", "abcd")); + std::vector<uint8_t> mock_data2{2, 3, 4, 5, 6}; + database::write_cal_data("mock_data", "abcd", mock_data); + // Write it twice to force a backup + database::write_cal_data("mock_data", "abcd", mock_data2, "BACKUP"); + mock_data_rb = database::read_cal_data("mock_data", "abcd"); + BOOST_CHECK_EQUAL_COLLECTIONS( + mock_data2.begin(), mock_data2.end(), mock_data_rb.begin(), mock_data_rb.end()); + BOOST_CHECK(database::has_cal_data("mock_data", "abcd")); + BOOST_CHECK(fs::exists(tmp_cal_path / "mock_data_abcd.cal.BACKUP")); + + fs::remove_all(tmp_cal_path, ec); + if (ec) { + std::cout << "WARNING: Could not remove temp cal path." << std::endl; + } +} |