diff options
| -rw-r--r-- | Makefile.am | 17 | ||||
| -rw-r--r-- | configure.ac | 4 | ||||
| -rw-r--r-- | doc/TIMESTAMPS.rst | 1 | ||||
| -rw-r--r-- | doc/example.mux | 19 | ||||
| -rwxr-xr-x | doc/zmq_remote.py | 22 | ||||
| -rw-r--r-- | lib/ClockTAI.cpp | 422 | ||||
| -rw-r--r-- | lib/ClockTAI.h | 61 | ||||
| -rw-r--r-- | lib/Json.cpp | 122 | ||||
| -rw-r--r-- | lib/Json.h | 63 | ||||
| -rw-r--r-- | lib/RemoteControl.cpp | 57 | ||||
| -rw-r--r-- | lib/RemoteControl.h | 12 | ||||
| -rw-r--r-- | lib/charset/README | 2 | ||||
| -rw-r--r-- | lib/charset/utf8.h | 80 | ||||
| -rw-r--r-- | lib/charset/utf8/checked.h | 176 | ||||
| -rw-r--r-- | lib/charset/utf8/core.h | 275 | ||||
| -rw-r--r-- | lib/charset/utf8/cpp11.h | 70 | ||||
| -rw-r--r-- | lib/charset/utf8/cpp17.h | 96 | ||||
| -rw-r--r-- | lib/charset/utf8/cpp20.h | 124 | ||||
| -rw-r--r-- | lib/charset/utf8/unchecked.h | 163 | ||||
| -rw-r--r-- | src/DabMultiplexer.cpp | 10 | ||||
| -rw-r--r-- | src/DabMultiplexer.h | 4 | ||||
| -rw-r--r-- | src/ManagementServer.cpp | 6 | ||||
| -rw-r--r-- | src/MuxElements.cpp | 62 | ||||
| -rw-r--r-- | src/MuxElements.h | 16 | ||||
| -rw-r--r-- | src/dabOutput/dabOutputZMQ.cpp | 1 | ||||
| -rw-r--r-- | src/input/Edi.cpp | 21 | ||||
| -rw-r--r-- | src/input/Edi.h | 3 | ||||
| -rw-r--r-- | src/input/Zmq.cpp | 12 | ||||
| -rw-r--r-- | src/input/Zmq.h | 1 |
29 files changed, 1496 insertions, 426 deletions
diff --git a/Makefile.am b/Makefile.am index 48dcc5e..d512a16 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ # Copyright (C) 2008, 2009 Her Majesty the Queen in Right of Canada # (Communications Research Center Canada) # -# Copyright (C) 2018 Matthias P. Braendli +# Copyright (C) 2024 Matthias P. Braendli # http://opendigitalradio.org # This file is part of ODR-DabMux. @@ -49,10 +49,13 @@ lib_charset_sources = lib/charset/charset.cpp \ lib/charset/utf8/checked.h \ lib/charset/utf8/core.h \ lib/charset/utf8/unchecked.h \ + lib/charset/utf8/cpp11.h \ + lib/charset/utf8/cpp17.h \ + lib/charset/utf8/cpp20.h \ lib/charset/utf8.h odr_dabmux_CFLAGS =-Wall $(INCLUDE) $(PTHREAD_CFLAGS) $(GITVERSION_FLAGS) -odr_dabmux_CXXFLAGS =-Wall -std=c++11 $(PTHREAD_CXXFLAGS) $(INCLUDE) $(GITVERSION_FLAGS) $(BOOST_CPPFLAGS) $(ZMQ_CPPFLAGS) +odr_dabmux_CXXFLAGS =-Wall $(PTHREAD_CXXFLAGS) $(INCLUDE) $(GITVERSION_FLAGS) $(BOOST_CPPFLAGS) $(ZMQ_CPPFLAGS) odr_dabmux_LDADD =$(ZMQ_LIBS) $(BOOST_LDFLAGS) \ $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) $(BOOST_SYSTEM_LIB) @@ -150,6 +153,8 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ lib/Log.cpp \ lib/RemoteControl.cpp \ lib/RemoteControl.h \ + lib/Json.h \ + lib/Json.cpp \ lib/edi/STIDecoder.cpp \ lib/edi/STIDecoder.hpp \ lib/edi/STIWriter.cpp \ @@ -191,13 +196,15 @@ odr_zmq2farsync_SOURCES = src/zmq2farsync/zmq2farsync.cpp \ lib/Log.cpp \ lib/RemoteControl.cpp \ lib/RemoteControl.h \ + lib/Json.h \ + lib/Json.cpp \ lib/Socket.h \ lib/Socket.cpp \ lib/zmq.hpp odr_zmq2farsync_LDADD = $(ZMQ_LIBS) odr_zmq2farsync_CFLAGS = -Wall $(ZMQ_CPPFLAGS) $(PTHREAD_CFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) -odr_zmq2farsync_CXXFLAGS = -Wall -std=c++11 $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) $(ZMQ_CPPFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) +odr_zmq2farsync_CXXFLAGS = -Wall $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) $(ZMQ_CPPFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ src/zmq2edi/EDISender.h \ @@ -221,6 +228,8 @@ odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ lib/Log.cpp \ lib/RemoteControl.cpp \ lib/RemoteControl.h \ + lib/Json.h \ + lib/Json.cpp \ lib/crc.h \ lib/crc.c \ lib/ReedSolomon.h \ @@ -233,7 +242,7 @@ odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ odr_zmq2edi_LDADD = $(ZMQ_LIBS) $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) $(BOOST_SYSTEM_LIB) odr_zmq2edi_CFLAGS = -Wall $(ZMQ_CPPFLAGS) $(PTHREAD_CFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) -odr_zmq2edi_CXXFLAGS = -Wall -std=c++11 $(PTHREAD_LIBS) $(ZMQ_CPPFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) +odr_zmq2edi_CXXFLAGS = -Wall $(PTHREAD_LIBS) $(ZMQ_CPPFLAGS) $(GITVERSION_FLAGS) $(INCLUDE) man_MANS = man/odr-dabmux.1 \ man/odr-zmq2edi.1 diff --git a/configure.ac b/configure.ac index cc8f9b6..875c56f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ # Copyright (C) 2008, 2009 Her Majesty the Queen in Right of Canada # (Communications Research Center Canada) # -# Copyright (C) 2023 Matthias P. Braendli, http://opendigitalradio.org +# Copyright (C) 2024 Matthias P. Braendli, http://opendigitalradio.org # This file is part of ODR-DabMux. # @@ -34,7 +34,7 @@ AC_PROG_CC AM_PROG_CC_C_O AC_PROG_INSTALL -AX_CXX_COMPILE_STDCXX(11,noext,mandatory) +AX_CXX_COMPILE_STDCXX(17,noext,mandatory) # Checks for libraries. AX_PTHREAD([], AC_MSG_ERROR([requires pthread])) diff --git a/doc/TIMESTAMPS.rst b/doc/TIMESTAMPS.rst index d9c3563..7f48b58 100644 --- a/doc/TIMESTAMPS.rst +++ b/doc/TIMESTAMPS.rst @@ -13,7 +13,6 @@ The following table tries to summarise the differences. | Later than v2.3.1 | t_frame = t_tx = t_mux + tist_offset | negative, meaning delay before t_tx | Something larger than mod processing time | +-----------------------------+----------------------------------------------+-------------------------------------+-----------------------------------------------+ -For historical reasons, ODR-DabMod decodes absolute timestamp from MNSC, not from “EDI seconds”. The edilib tool decodes both EDI timestamp and MNSC, and can be used to verify both are identical. Issues in ODR-DabMux v2.3.1 diff --git a/doc/example.mux b/doc/example.mux index 9a5686b..383a2ab 100644 --- a/doc/example.mux +++ b/doc/example.mux @@ -36,18 +36,21 @@ general { ; This also enables time encoding using the MNSC. ; ; When TIST is enabled, and either EDI or a ZMQ output with metadata is used, - ; ODR-DabMux will download leap-second information from the IETF website, - ; and cache it locally in /var/tmp. It will refresh the data by itself - ; before it expires. + ; ODR-DabMux will download the leap-second information bulletin, + ; and cache it locally in /var/tmp. + ; It will refresh the bulletin by itself before it expires. If that fails, + ; ODR-DabMux will continue running with the current TAI-UTC clock offset. ; ; If it cannot load this information, ODR-DabMux cannot start up! ; ; If your system doesn't have access to internet, you have to take care ; to create the file before ODR-DabMux startup. Get it from - ; http://www.ietf.org/timezones/data/leap-seconds.list + ; https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list ; and save it to ; /var/tmp/odr-dabmux-leap-seconds.cache - ; Refresh the file before expiry otherwise ODR-DabMux will abort! + ; + ; ODR-DabMux will start up even with an expired bulletin in cache, but will + ; output a warning. ; ; Use the RC interface 'get clocktai expiry' command to check how long ; your file is still valid. @@ -60,7 +63,11 @@ general { ; The URLs used to fetch the TAI bulletin can be overridden if needed. ; URLs are given as a pipe-separated list, and the default value is: - ;tai_clock_bulletins "https://www.ietf.org/timezones/data/leap-seconds.list|https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list" + ;tai_clock_bulletins "https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list" + ; + ; Through RC, use `set clocktai url <LIST_OF_URLS>` to change the URLs used to download the bulletin + ; during runtime. + ; You may also override the bulletin using `set clocktai tai_utc_offset 37` to set a TAI-UTC offset manually. ; ; You may also use a file:// URL if you take care of updating the file ; yourself and store it locally. diff --git a/doc/zmq_remote.py b/doc/zmq_remote.py index 56465d3..7581575 100755 --- a/doc/zmq_remote.py +++ b/doc/zmq_remote.py @@ -16,7 +16,7 @@ poller = zmq.Poller() poller.register(sock, zmq.POLLIN) if len(sys.argv) < 2: - print("Usage: program url cmd [args...]") + print("Usage: program url cmd [args...]", file=sys.stderr) sys.exit(1) sock.connect(sys.argv[1]) @@ -25,7 +25,7 @@ message_parts = sys.argv[2:] # first do a ping test -print("ping") +print("ping", file=sys.stderr) sock.send(b"ping") socks = dict(poller.poll(1000)) @@ -33,9 +33,9 @@ if socks: if socks.get(sock) == zmq.POLLIN: data = sock.recv_multipart() - print("Received: {}".format(len(data))) + print("Received: {}".format(len(data)), file=sys.stderr) for i,part in enumerate(data): - print(" {}".format(part)) + print(" {}".format(part), file=sys.stderr) for i, part in enumerate(message_parts): if i == len(message_parts) - 1: @@ -43,18 +43,22 @@ if socks: else: f = zmq.SNDMORE - print("Send {}({}): '{}'".format(i, f, part)) + print("Send {}({}): '{}'".format(i, f, part), file=sys.stderr) sock.send(part.encode(), flags=f) data = sock.recv_multipart() - print("Received: {}".format(len(data))) - for i,part in enumerate(data): - print(" RX {}: {}".format(i, part.decode().replace('\n',' '))) + print("Received: {}".format(len(data)), file=sys.stderr) + for i, part in enumerate(data): + if message_parts[0] == 'showjson': + # This allows you to pipe the JSON into another tool + print(part.decode()) + else: + print(" RX {}: {}".format(i, part.decode().replace('\n',' ')), file=sys.stderr) else: - print("ZMQ error: timeout") + print("ZMQ error: timeout", file=sys.stderr) context.destroy(linger=5) # This is free and unencumbered software released into the public domain. diff --git a/lib/ClockTAI.cpp b/lib/ClockTAI.cpp index a244aba..d8e37ea 100644 --- a/lib/ClockTAI.cpp +++ b/lib/ClockTAI.cpp @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -33,6 +33,7 @@ * ClockTAI.cpp Log.cpp RemoteControl.cpp -lboost_system -o taitest && ./taitest */ +#include <iterator> #ifdef HAVE_CONFIG_H # include "config.h" #endif @@ -61,22 +62,15 @@ using namespace std; -#ifdef DOWNLOADED_IN_THE_PAST_TEST -static bool wait_longer = true; -#endif - constexpr int refresh_retry_interval_hours = 1; // Offset between NTP time and POSIX time: // timestamp_unix = timestamp_ntp - NTP_UNIX_OFFSET constexpr int64_t NTP_UNIX_OFFSET = 2208988800L; -constexpr int64_t MONTH = 3600 * 24 * 30; - -// leap seconds insertion bulletin is available from the IETF and in the TZ -// distribution -static array<const char*, 2> default_tai_urls = { - "https://www.ietf.org/timezones/data/leap-seconds.list", +// leap seconds insertion bulletin was previously available from the IETF and in the TZ +// distribution, but in late 2023 IETF stopped serving the file. +static array<const char*, 1> default_tai_urls = { "https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list", }; @@ -84,6 +78,32 @@ static array<const char*, 2> default_tai_urls = { // /var/tmp "must not be deleted when the system is booted." static const char *tai_cache_location = "/var/tmp/odr-leap-seconds.cache"; +static string join_string_with_pipe(const vector<string>& vec) +{ + stringstream ss; + for (auto it = vec.cbegin(); it != vec.cend(); ++it) { + ss << *it; + if (it + 1 != vec.cend()) { + ss << "|"; + } + } + return ss.str(); +} + +static vector<string> split_pipe_separated_string(const string& s) +{ + stringstream ss; + ss << s; + + string elem; + vector<string> components; + while (getline(ss, elem, '|')) { + components.push_back(elem); + } + return components; +} + + // read TAI offset from a valid bulletin in IETF format static int parse_ietf_bulletin(const std::string& bulletin) { @@ -146,20 +166,31 @@ static int parse_ietf_bulletin(const std::string& bulletin) throw runtime_error("No data in TAI bulletin"); } + // With the current evolution of the offset, we're probably going + // to reach 500 long after DAB gets replaced by another standard. + // Or maybe leap seconds get abolished first... + if (tai_utc_offset < 0 or tai_utc_offset > 500) { + throw runtime_error("Unreasonable TAI-UTC offset calculated"); + } + return tai_utc_offset; } +int64_t BulletinState::expires_in() const { + time_t now = time(nullptr); + return expires_at - now; +} -struct bulletin_state { - bool valid = false; - int64_t expiry = 0; - int offset = 0; +bool BulletinState::usable() const { + return valid and expires_in() > 0; +} - bool usable() const { return valid and expiry > 0; } - bool expires_soon() const { return usable() and expiry < 1 * MONTH; } -}; +bool BulletinState::expires_soon() const { + constexpr int64_t MONTH = 3600 * 24 * 30; + return usable() and expires_in() < 1 * MONTH; +} -static bulletin_state parse_bulletin(const string& bulletin) +BulletinState Bulletin::state() const { // The bulletin contains one line that specifies an expiration date // in NTP time. If that point in time is in the future, we consider @@ -168,11 +199,22 @@ static bulletin_state parse_bulletin(const string& bulletin) // The entry looks like this: //#@ 3707596800 - bulletin_state ret; + BulletinState ret; - std::regex regex_expiration(R"(#@\s+([0-9]+))"); + if (std::holds_alternative<Bulletin::OverrideData>(bulletin_or_override)) { + const auto& od = std::get<Bulletin::OverrideData>(bulletin_or_override); + ret.offset = od.offset; + ret.expires_at = od.expires_at; + ret.valid = true; +#ifdef TAI_TEST + etiLog.level(debug) << "state() from Override!"; +#endif + return ret; + } - time_t now = time(nullptr); + const auto& bulletin = std::get<string>(bulletin_or_override); + + std::regex regex_expiration(R"(#@\s+([0-9]+))"); stringstream ss(bulletin); @@ -190,20 +232,29 @@ static bulletin_state parse_bulletin(const string& bulletin) const int64_t expiry_unix = std::stoll(expiry_data_str) - NTP_UNIX_OFFSET; #ifdef TAI_TEST - etiLog.level(info) << "Bulletin expires in " << expiry_unix - now; + { + // Do not use `now` for anything else but debugging, otherwise it + // breaks the cache. + time_t now = time(nullptr); + etiLog.level(debug) << "Bulletin " << + get_source() << " expires in " << expiry_unix - now; + } #endif - ret.expiry = expiry_unix - now; + ret.expires_at = expiry_unix; ret.offset = parse_ietf_bulletin(bulletin); ret.valid = true; } catch (const invalid_argument& e) { - etiLog.level(warn) << "Could not parse bulletin: " << e.what(); + etiLog.level(warn) << "Could not parse bulletin from " << + get_source() << ": " << e.what(); } catch (const out_of_range&) { - etiLog.level(warn) << "Parse bulletin: conversion is out of range"; + etiLog.level(warn) << "Parse bulletin from " << + get_source() << ": conversion is out of range"; } catch (const runtime_error& e) { - etiLog.level(warn) << "Parse bulletin: " << e.what(); + etiLog.level(warn) << "Parse bulletin from " << + get_source() << ": " << e.what(); } break; } @@ -224,9 +275,9 @@ static size_t fill_bulletin(char *ptr, size_t size, size_t nmemb, void *ctx) return len; } -static string download_tai_utc_bulletin(const char* url) +Bulletin Bulletin::download_from_url(const char* url) { - stringstream bulletin; + stringstream bulletin_data; #ifdef HAVE_CURL CURL *curl; @@ -238,7 +289,7 @@ static string download_tai_utc_bulletin(const char* url) /* Tell libcurl to follow redirection */ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_bulletin); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &bulletin); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &bulletin_data); res = curl_easy_perform(curl); /* always cleanup ! */ @@ -249,19 +300,39 @@ static string download_tai_utc_bulletin(const char* url) string(curl_easy_strerror(res))); } } - return bulletin.str(); + Bulletin bulletin; + bulletin.source = url; + bulletin.bulletin_or_override = bulletin_data.str(); + return bulletin; #else throw runtime_error("Cannot download TAI Clock information without cURL"); #endif // HAVE_CURL } -static string load_bulletin_from_file(const char* cache_filename) +Bulletin Bulletin::create_with_fixed_offset(int offset) { + Bulletin bulletin; + bulletin.source = "manual override"; + + OverrideData od; + od.offset = offset; + time_t now = time(nullptr); + // 10 years is probably equivalent to infinity in this case... + od.expires_at = now + 10L * 365 * 24 * 3600; + bulletin.bulletin_or_override = od; + return bulletin; +} + +Bulletin Bulletin::load_from_file(const char* cache_filename) +{ + Bulletin bulletin; + bulletin.source = cache_filename; + int fd = open(cache_filename, O_RDWR); // lockf requires O_RDWR if (fd == -1) { etiLog.level(error) << "TAI-UTC bulletin open cache for reading: " << strerror(errno); - return ""; + return bulletin; } lseek(fd, 0, SEEK_SET); @@ -280,7 +351,7 @@ static string load_bulletin_from_file(const char* cache_filename) close(fd); etiLog.level(error) << "TAI-UTC bulletin read cache: " << strerror(errno); - return ""; + return bulletin; } copy(buf.data(), buf.data() + ret, back_inserter(new_bulletin_data)); @@ -288,7 +359,7 @@ static string load_bulletin_from_file(const char* cache_filename) close(fd); - return string{new_bulletin_data.data(), new_bulletin_data.size()}; + bulletin.bulletin_or_override = string{new_bulletin_data.data(), new_bulletin_data.size()}; } else { etiLog.level(error) << @@ -296,13 +367,25 @@ static string load_bulletin_from_file(const char* cache_filename) strerror(errno); close(fd); } - return ""; + return bulletin; +} + +void Bulletin::clear_expiry_if_overridden() +{ + if (std::holds_alternative<Bulletin::OverrideData>(bulletin_or_override)) { + auto& od = std::get<Bulletin::OverrideData>(bulletin_or_override); + time_t now = time(nullptr); + od.expires_at = now; + } } ClockTAI::ClockTAI(const std::vector<std::string>& bulletin_urls) : RemoteControllable("clocktai") { + RC_ADD_PARAMETER(tai_utc_offset, "TAI-UTC offset"); RC_ADD_PARAMETER(expiry, "Number of seconds until TAI Bulletin expires"); + RC_ADD_PARAMETER(expires_at, "UNIX timestamp when TAI Bulletin expires"); + RC_ADD_PARAMETER(url, "URLs used to fetch the bulletin, separated by pipes"); if (bulletin_urls.empty()) { etiLog.level(debug) << "Initialising default TAI Bulletin URLs"; @@ -315,138 +398,119 @@ ClockTAI::ClockTAI(const std::vector<std::string>& bulletin_urls) : m_bulletin_urls = bulletin_urls; } - for (const auto& url : m_bulletin_urls) { - etiLog.level(info) << "TAI Bulletin URL: '" << url << "'"; - } + etiLog.level(debug) << "ClockTAI uses bulletin URL: '" << join_string_with_pipe(m_bulletin_urls) << "'"; } -int ClockTAI::get_valid_offset() +BulletinState ClockTAI::get_valid_offset() { - int offset = 0; - bool offset_valid = false; - bool refresh_m_bulletin = false; - std::unique_lock<std::mutex> lock(m_data_mutex); - const auto state = parse_bulletin(m_bulletin); - if (state.usable()) { + const auto state = m_bulletin.state(); #if TAI_TEST - etiLog.level(info) << "Bulletin already valid"; + etiLog.level(info) << "TAI get_valid_offset STEP 1 "; + etiLog.level(info) << " " << m_bulletin.get_source() << " " << + state.valid << " " << state.usable() << " " << state.expires_in(); #endif - offset = state.offset; - offset_valid = true; - - refresh_m_bulletin = state.expires_soon(); - } - else { - refresh_m_bulletin = true; + if (state.usable()) { + return state; } - if (refresh_m_bulletin) { - const auto cache_bulletin = load_bulletin_from_file(tai_cache_location); #if TAI_TEST - etiLog.level(info) << "Loaded cache bulletin with " << - std::count_if(cache_bulletin.cbegin(), cache_bulletin.cend(), - [](const char c){ return c == '\n'; }) << " lines"; + etiLog.level(info) << "TAI get_valid_offset STEP 2"; #endif - const auto cache_state = parse_bulletin(cache_bulletin); + const auto cache_bulletin = Bulletin::load_from_file(tai_cache_location); + const auto cache_state = cache_bulletin.state(); + if (cache_state.usable()) { + m_bulletin = cache_bulletin; +#if TAI_TEST + etiLog.level(info) << "TAI get_valid_offset STEP 2 take cache"; +#endif + return cache_state; + } - if (cache_state.usable() and not cache_state.expires_soon()) { - m_bulletin = cache_bulletin; - offset = cache_state.offset; - offset_valid = true; #if TAI_TEST - etiLog.level(info) << "Bulletin from cache valid with offset=" << offset; + etiLog.level(info) << "TAI get_valid_offset STEP 3"; #endif - } - else { - for (const auto& url : m_bulletin_urls) { - try { + + vector<Bulletin> bulletins({m_bulletin, cache_bulletin}); + + for (const auto& url : m_bulletin_urls) { + try { #if TAI_TEST - etiLog.level(info) << "Load bulletin from " << url; + etiLog.level(info) << "Load bulletin from " << url; #endif - const auto new_bulletin = download_tai_utc_bulletin(url.c_str()); - const auto new_state = parse_bulletin(new_bulletin); - if (new_state.usable()) { - m_bulletin = new_bulletin; - offset = new_state.offset; - offset_valid = true; - - etiLog.level(debug) << "Loaded valid TAI Bulletin from " << - url << " giving offset=" << offset; - } - else { - etiLog.level(debug) << "Skipping invalid TAI bulletin from " - << url; - } - } - catch (const runtime_error& e) { - etiLog.level(warn) << - "TAI-UTC offset could not be retrieved from " << - url << " : " << e.what(); - } + const auto new_bulletin = Bulletin::download_from_url(url.c_str()); + bulletins.push_back(new_bulletin); - if (offset_valid) { - update_cache(tai_cache_location); - break; - } + const auto new_state = new_bulletin.state(); + if (new_state.usable()) { + m_bulletin = new_bulletin; + new_bulletin.store_to_cache(tai_cache_location); + + etiLog.level(debug) << "Loaded valid TAI Bulletin from " << + url << " giving offset=" << new_state.offset; + return new_state; + } + else { + etiLog.level(debug) << "Skipping invalid TAI bulletin from " + << url; } } + catch (const runtime_error& e) { + etiLog.level(warn) << + "TAI-UTC offset could not be retrieved from " << + url << " : " << e.what(); + } } - if (offset_valid) { - // With the current evolution of the offset, we're probably going - // to reach 500 long after DAB gets replaced by another standard. - if (offset < 0 or offset > 500) { - stringstream ss; - ss << "TAI offset " << offset << " out of range"; - throw range_error(ss.str()); - } +#if TAI_TEST + etiLog.level(info) << "TAI get_valid_offset STEP 4"; +#endif - return offset; - } - else { - // Try again later - throw download_failed(); + // Maybe we have a valid but expired bulletin available. + // Place bulletins with largest expiry first + std::sort(bulletins.begin(), bulletins.end(), + [](const Bulletin& a, const Bulletin& b) { + return a.state().expires_at > b.state().expires_at; }); + + for (const auto& bulletin : bulletins) { + const auto& state = bulletin.state(); + if (state.valid) { + etiLog.level(warn) << "Taking TAI-UTC offset from expired bulletin from " << + bulletin.get_source() << " : " << state.offset << "s expired " << + state.expires_in() << "s ago"; + m_bulletin = bulletin; + return state; + } } + + throw download_failed(); } int ClockTAI::get_offset() { using namespace std::chrono; - const auto time_now = system_clock::now(); + const auto time_now = steady_clock::now(); std::unique_lock<std::mutex> lock(m_data_mutex); - if (not m_offset_valid) { -#ifdef DOWNLOADED_IN_THE_PAST_TEST - // Assume we've downloaded it in the past: - - m_offset = 37; // Valid in early 2017 - m_offset_valid = true; - - // Simulate requiring a new download - m_bulletin_refresh_time = time_now - hours(24 * 40); -#else + if (not m_state.has_value()) { // First time we run we must block until we know // the offset lock.unlock(); try { - m_offset = get_valid_offset(); + m_state = get_valid_offset(); } catch (const download_failed&) { throw runtime_error("Unable to download TAI bulletin"); } lock.lock(); - m_offset_valid = true; - m_bulletin_refresh_time = time_now; -#endif - etiLog.level(info) << - "Initialised TAI-UTC offset to " << m_offset << "s."; + m_state_last_updated = time_now; + etiLog.level(info) << "Initialised TAI-UTC offset to " << m_state->offset << "s."; } - if (m_bulletin_refresh_time + hours(1) < time_now) { + if (m_state_last_updated + hours(1) < time_now) { // Once per hour, parse the bulletin again, and // if necessary trigger a download. // Leap seconds are announced several months in advance @@ -457,23 +521,23 @@ int ClockTAI::get_offset() switch (state) { case future_status::ready: try { - m_offset = m_offset_future.get(); - m_offset_valid = true; - m_bulletin_refresh_time = time_now; + m_state = m_offset_future.get(); + m_state_last_updated = time_now; etiLog.level(info) << - "Updated TAI-UTC offset to " << m_offset << "s."; + "Updated TAI-UTC offset to " << m_state->offset << "s."; } catch (const download_failed&) { etiLog.level(warn) << "TAI-UTC download failed, will retry in " << refresh_retry_interval_hours << " hour(s)"; - m_bulletin_refresh_time += hours(refresh_retry_interval_hours); - } -#ifdef DOWNLOADED_IN_THE_PAST_TEST - wait_longer = false; +#if TAI_TEST + m_state_last_updated += seconds(11); +#else + m_state_last_updated += hours(refresh_retry_interval_hours); #endif + } break; case future_status::deferred: @@ -493,7 +557,10 @@ int ClockTAI::get_offset() } } - return m_offset; + if (m_state) { + return m_state->offset; + } + throw std::logic_error("ClockTAI: No valid m_state at end of get_offset()"); } #if SUPPORT_SETTING_CLOCK_TAI @@ -514,8 +581,13 @@ int ClockTAI::update_local_tai_clock(int offset) } #endif -void ClockTAI::update_cache(const char* cache_filename) +void Bulletin::store_to_cache(const char* cache_filename) const { + if (not std::holds_alternative<string>(bulletin_or_override)) { + etiLog.level(error) << "ClockTAI: Cannot store an artificial bulletin to cache!"; + } + const auto& bulletin = std::get<string>(bulletin_or_override); + int fd = open(cache_filename, O_RDWR | O_CREAT, 00664); if (fd == -1) { etiLog.level(error) << @@ -529,8 +601,8 @@ void ClockTAI::update_cache(const char* cache_filename) ssize_t ret = lockf(fd, F_LOCK, 0); if (ret == 0) { // exclusive lock acquired - const char *data = m_bulletin.data(); - size_t remaining = m_bulletin.size(); + const char *data = bulletin.data(); + size_t remaining = bulletin.size(); while (remaining > 0) { ret = write(fd, data, remaining); @@ -557,13 +629,35 @@ void ClockTAI::update_cache(const char* cache_filename) } } - void ClockTAI::set_parameter(const string& parameter, const string& value) { - if (parameter == "expiry") { + if (parameter == "expiry" or parameter == "expires_at") { throw ParameterError("Parameter '" + parameter + "' is read-only in controllable " + get_rc_name()); } + else if (parameter == "tai_utc_offset") { + const auto offset = std::stoi(value); + auto b = Bulletin::create_with_fixed_offset(offset); + + etiLog.level(warn) << "ClockTAI: manually overriding UTC-TAI offset to " << offset; + + std::unique_lock<std::mutex> lock(m_data_mutex); + m_bulletin = b; + m_state = b.state(); + m_state_last_updated = chrono::steady_clock::now(); + } + else if (parameter == "url") { + { + std::unique_lock<std::mutex> lock(m_data_mutex); + m_bulletin_urls = split_pipe_separated_string(value); + m_state_last_updated = chrono::steady_clock::time_point::min(); + + // Setting URL expires the bulletin, if it was manually overridden, + // so that the selection logic doesn't prefer it + m_bulletin.clear_expiry_if_overridden(); + } + etiLog.level(info) << "ClockTAI: triggering a reload from URLs..."; + } else { throw ParameterError("Parameter '" + parameter + "' is not exported by controllable " + get_rc_name()); @@ -574,13 +668,22 @@ const string ClockTAI::get_parameter(const string& parameter) const { if (parameter == "expiry") { std::unique_lock<std::mutex> lock(m_data_mutex); - const int64_t expiry = parse_bulletin(m_bulletin).expiry; - if (expiry > 0) { - return to_string(expiry); - } - else { - return "Bulletin expired or invalid!"; + return to_string(m_bulletin.state().expires_in()); + } + else if (parameter == "expires_at") { + std::unique_lock<std::mutex> lock(m_data_mutex); + return to_string(m_bulletin.state().expires_at); + } + else if (parameter == "tai_utc_offset") { + std::unique_lock<std::mutex> lock(m_data_mutex); + if (m_state) { + return to_string(m_state->offset); } + throw ParameterError("Parameter '" + parameter + + "' has no current value" + get_rc_name()); + } + else if (parameter == "url") { + return join_string_with_pipe(m_bulletin_urls);; } else { throw ParameterError("Parameter '" + parameter + @@ -588,6 +691,35 @@ const string ClockTAI::get_parameter(const string& parameter) const } } +const json::map_t ClockTAI::get_all_values() const +{ + json::map_t stat; + std::unique_lock<std::mutex> lock(m_data_mutex); + + const auto& state = m_bulletin.state(); + +#if TAI_TEST + etiLog.level(debug) << "CALC FROM m_bulletin: " << state.valid << " " << + state.offset << " " << state.expires_at << " -> " << state.expires_in(); + etiLog.level(debug) << "CACHED IN m_state: " << m_state->valid << " " << + m_state->offset << " " << m_state->expires_at << " -> " << m_state->expires_in(); +#endif + + stat["tai_utc_offset"].v = state.offset; + + stat["expiry"].v = state.expires_in(); // Might be negative when expired or 0 when invalid + if (state.valid) { + stat["expires_at"].v = state.expires_at; + } + else { + stat["expires_at"].v = nullopt; + } + + stat["url"].v = join_string_with_pipe(m_bulletin_urls); + + return stat; +} + #if 0 // Example testing code void debug_tai_clk() diff --git a/lib/ClockTAI.h b/lib/ClockTAI.h index 743cf68..8cd00e5 100644 --- a/lib/ClockTAI.h +++ b/lib/ClockTAI.h @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -42,12 +42,49 @@ #include <mutex> #include <string> #include <vector> +#include <optional> +#include <variant> #include "RemoteControl.h" // EDI needs to know UTC-TAI, but doesn't need the CLOCK_TAI to be set. // We can keep this code, maybe for future use #define SUPPORT_SETTING_CLOCK_TAI 0 +struct BulletinState { + bool valid = false; + int64_t expires_at = 0; + int offset = 0; + + int64_t expires_in() const; + bool usable() const; + bool expires_soon() const; +}; + +class Bulletin { + public: + static Bulletin download_from_url(const char *url); + static Bulletin create_with_fixed_offset(int offset); + static Bulletin load_from_file(const char *cache_filename); + + void clear_expiry_if_overridden(); + + void store_to_cache(const char* cache_filename) const; + + std::string get_source() const { return source; } + BulletinState state() const; + private: + // URL or file path from which the bulletin has been/will be loaded + std::string source; + + struct OverrideData { + int offset = 0; + int expires_at = 0; + }; + // string: A cache of the bulletin, or empty string if not loaded + // int: A manually overridden offset + std::variant<std::string, OverrideData> bulletin_or_override; +}; + /* Loads, parses and represents TAI-UTC offset information from the IETF bulletin */ class ClockTAI : public RemoteControllable { public: @@ -71,33 +108,31 @@ class ClockTAI : public RemoteControllable { // download it, and calculate the TAI-UTC offset. // Returns the offset or throws download_failed or a range_error // if the offset is out of bounds. - int get_valid_offset(void); + BulletinState get_valid_offset(void); // Download of new bulletin is done asynchronously - std::future<int> m_offset_future; + std::future<BulletinState> m_offset_future; // Protect all data members, as RC functions are in another thread mutable std::mutex m_data_mutex; - // The currently used TAI-UTC offset, extracted from m_bulletin and cached here - // to avoid having to parse the bulletin all the time - int m_offset = 0; - int m_offset_valid = false; - std::vector<std::string> m_bulletin_urls; - std::string m_bulletin; - std::chrono::system_clock::time_point m_bulletin_refresh_time; - - // Update the cache file with the current m_bulletin - void update_cache(const char* cache_filename); + Bulletin m_bulletin; + // The currently used TAI-UTC offset, extracted the bulletin and cached + // here to avoid having to parse the bulletin all the time + std::optional<BulletinState> m_state; + std::chrono::steady_clock::time_point m_state_last_updated; + public: /* Remote control */ virtual void set_parameter(const std::string& parameter, const std::string& value); /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + + virtual const json::map_t get_all_values() const; }; diff --git a/lib/Json.cpp b/lib/Json.cpp new file mode 100644 index 0000000..4dc2f25 --- /dev/null +++ b/lib/Json.cpp @@ -0,0 +1,122 @@ +/* + Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2023 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ +#include <string> +#include <iostream> +#include <sstream> +#include <iomanip> +#include <string> +#include <algorithm> + +#include "Json.h" + +namespace json { + static std::string escape_json(const std::string &s) { + std::ostringstream o; + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': o << "\\\""; break; + case '\\': o << "\\\\"; break; + case '\b': o << "\\b"; break; + case '\f': o << "\\f"; break; + case '\n': o << "\\n"; break; + case '\r': o << "\\r"; break; + case '\t': o << "\\t"; break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" + << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(*c); + } else { + o << *c; + } + } + } + return o.str(); + } + + std::string map_to_json(const map_t& values) { + std::ostringstream ss; + ss << "{ "; + size_t ix = 0; + for (const auto& element : values) { + if (ix > 0) { + ss << ","; + } + + ss << "\"" << escape_json(element.first) << "\": "; + ss << value_to_json(element.second); + + ix++; + } + ss << " }"; + + return ss.str(); + } + + std::string value_to_json(const value_t& value) + { + std::ostringstream ss; + + if (std::holds_alternative<std::string>(value.v)) { + ss << "\"" << escape_json(std::get<std::string>(value.v)) << "\""; + } + else if (std::holds_alternative<double>(value.v)) { + ss << std::fixed << std::get<double>(value.v); + } + else if (std::holds_alternative<ssize_t>(value.v)) { + ss << std::get<ssize_t>(value.v); + } + else if (std::holds_alternative<size_t>(value.v)) { + ss << std::get<size_t>(value.v); + } + else if (std::holds_alternative<bool>(value.v)) { + ss << (std::get<bool>(value.v) ? "true" : "false"); + } + else if (std::holds_alternative<std::nullopt_t>(value.v)) { + ss << "null"; + } + else if (std::holds_alternative<std::vector<json::value_t> >(value.v)) { + const auto& vec = std::get<std::vector<json::value_t> >(value.v); + ss << "[ "; + size_t list_ix = 0; + for (const auto& list_element : vec) { + if (list_ix > 0) { + ss << ","; + } + ss << value_to_json(list_element); + list_ix++; + } + ss << "]"; + } + else if (std::holds_alternative<std::shared_ptr<json::map_t> >(value.v)) { + const map_t& v = *std::get<std::shared_ptr<json::map_t> >(value.v); + ss << map_to_json(v); + } + else { + throw std::logic_error("variant alternative not handled"); + } + + return ss.str(); + } +} diff --git a/lib/Json.h b/lib/Json.h new file mode 100644 index 0000000..65aa668 --- /dev/null +++ b/lib/Json.h @@ -0,0 +1,63 @@ +/* + Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2023 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + This module adds remote-control capability to some of the dabmux/dabmod modules. + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +#pragma once + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <vector> +#include <memory> +#include <optional> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <variant> + +namespace json { + + // STL containers are not required to support incomplete types, + // hence the shared_ptr + + struct value_t { + std::variant< + std::shared_ptr<std::unordered_map<std::string, value_t>>, + std::vector<value_t>, + std::string, + double, + size_t, + ssize_t, + bool, + std::nullopt_t> v; + }; + + using map_t = std::unordered_map<std::string, value_t>; + + std::string map_to_json(const map_t& values); + std::string value_to_json(const value_t& value); +} diff --git a/lib/RemoteControl.cpp b/lib/RemoteControl.cpp index 30dcb60..dca3373 100644 --- a/lib/RemoteControl.cpp +++ b/lib/RemoteControl.cpp @@ -25,6 +25,8 @@ #include <list> #include <string> #include <iostream> +#include <sstream> +#include <iomanip> #include <string> #include <algorithm> @@ -102,6 +104,18 @@ std::list< std::vector<std::string> > RemoteControllers::get_param_list_values(c return allparams; } + + +std::string RemoteControllers::get_showjson() { + json::map_t root; + for (auto &controllable : rcs.controllables) { + root[controllable->get_rc_name()].v = + std::make_shared<json::map_t>(controllable->get_all_values()); + } + + return json::map_to_json(root); +} + std::string RemoteControllers::get_param(const std::string& name, const std::string& param) { RemoteControllable* controllable = get_controllable_(name); return controllable->get_parameter(param); @@ -123,7 +137,7 @@ RemoteControllable* RemoteControllers::get_controllable_(const std::string& name [&](RemoteControllable* r) { return r->get_rc_name() == name; }); if (rc == controllables.end()) { - throw ParameterError("Module name unknown"); + throw ParameterError(string{"Module name '"} + name + "' unknown"); } else { return *rc; @@ -427,10 +441,15 @@ void RemoteControllerZmq::recv_all(zmq::socket_t& pSocket, std::vector<std::stri bool more = true; do { zmq::message_t msg; - pSocket.recv(msg); - std::string incoming((char*)msg.data(), msg.size()); - message.push_back(incoming); - more = msg.more(); + const auto zresult = pSocket.recv(msg); + if (zresult) { + std::string incoming((char*)msg.data(), msg.size()); + message.push_back(incoming); + more = msg.more(); + } + else { + more = false; + } } while (more); } @@ -457,6 +476,7 @@ void RemoteControllerZmq::send_fail_reply(zmq::socket_t &pSocket, const std::str void RemoteControllerZmq::process() { m_fault = false; + m_active = true; // create zmq reply socket for receiving ctrl parameters try { @@ -514,8 +534,21 @@ void RemoteControllerZmq::process() repSocket.send(zmsg, (--cohort_size > 0) ? zmq::send_flags::sndmore : zmq::send_flags::none); } } + else if (msg.size() == 1 && command == "showjson") { + try { + std::string json = rcs.get_showjson(); + + zmq::message_t zmsg(json.size()); + memcpy(zmsg.data(), json.data(), json.size()); + + repSocket.send(zmsg, zmq::send_flags::none); + } + catch (const ParameterError &err) { + send_fail_reply(repSocket, err.what()); + } + } else if (msg.size() == 2 && command == "show") { - std::string module((char*) msg[1].data(), msg[1].size()); + const std::string module((char*) msg[1].data(), msg[1].size()); try { list< vector<string> > r = rcs.get_param_list_values(module); size_t r_size = r.size(); @@ -533,8 +566,8 @@ void RemoteControllerZmq::process() } } else if (msg.size() == 3 && command == "get") { - std::string module((char*) msg[1].data(), msg[1].size()); - std::string parameter((char*) msg[2].data(), msg[2].size()); + const std::string module((char*) msg[1].data(), msg[1].size()); + const std::string parameter((char*) msg[2].data(), msg[2].size()); try { std::string value = rcs.get_param(module, parameter); @@ -547,9 +580,9 @@ void RemoteControllerZmq::process() } } else if (msg.size() == 4 && command == "set") { - std::string module((char*) msg[1].data(), msg[1].size()); - std::string parameter((char*) msg[2].data(), msg[2].size()); - std::string value((char*) msg[3].data(), msg[3].size()); + const std::string module((char*) msg[1].data(), msg[1].size()); + const std::string parameter((char*) msg[2].data(), msg[2].size()); + const std::string value((char*) msg[3].data(), msg[3].size()); try { rcs.set_param(module, parameter, value); @@ -561,7 +594,7 @@ void RemoteControllerZmq::process() } else { send_fail_reply(repSocket, - "Unsupported command. commands: list, show, get, set"); + "Unsupported command. commands: list, show, get, set, showjson"); } } } diff --git a/lib/RemoteControl.h b/lib/RemoteControl.h index 2358b3a..26f30d9 100644 --- a/lib/RemoteControl.h +++ b/lib/RemoteControl.h @@ -3,7 +3,7 @@ Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2023 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -36,6 +36,8 @@ #endif #include <list> +#include <unordered_map> +#include <variant> #include <map> #include <memory> #include <string> @@ -46,6 +48,7 @@ #include "Log.h" #include "Socket.h" +#include "Json.h" #define RC_ADD_PARAMETER(p, desc) { \ std::vector<std::string> p; \ @@ -113,13 +116,13 @@ class RemoteControllable { } /* Base function to set parameters. */ - virtual void set_parameter( - const std::string& parameter, - const std::string& value) = 0; + virtual void set_parameter(const std::string& parameter, const std::string& value) = 0; /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const = 0; + virtual const json::map_t get_all_values() const = 0; + protected: std::string m_rc_name; std::list< std::vector<std::string> > m_parameters; @@ -135,6 +138,7 @@ class RemoteControllers { void check_faults(); std::list< std::vector<std::string> > get_param_list_values(const std::string& name); std::string get_param(const std::string& name, const std::string& param); + std::string get_showjson(); void set_param( const std::string& name, diff --git a/lib/charset/README b/lib/charset/README index b20f76b..3217b87 100644 --- a/lib/charset/README +++ b/lib/charset/README @@ -1,2 +1,2 @@ This UTF-8 to EBU charset (defined in ETSI TS 101 756v1.8.1) was copied from -ODR-PadEnc. +ODR-PadEnc, with utf8 library v4.0.5 from https://github.com/nemtrif/utfcpp/tree/master/source/utf8 diff --git a/lib/charset/utf8.h b/lib/charset/utf8.h index 4e44514..b513530 100644 --- a/lib/charset/utf8.h +++ b/lib/charset/utf8.h @@ -1,34 +1,46 @@ -// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "utf8/checked.h"
-#include "utf8/unchecked.h"
-
-#endif // header guard
+// Copyright 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +/* +To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro +and set it to one of the values used by the __cplusplus predefined macro. + +For instance, + #define UTF_CPP_CPLUSPLUS 199711L +will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard. +Some library features will be disabled. + +If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus. +*/ + +#include "utf8/checked.h" +#include "utf8/unchecked.h" + +#endif // header guard diff --git a/lib/charset/utf8/checked.h b/lib/charset/utf8/checked.h index 1331155..98949f8 100644 --- a/lib/charset/utf8/checked.h +++ b/lib/charset/utf8/checked.h @@ -1,4 +1,4 @@ -// Copyright 2006 Nemanja Trifunovic +// Copyright 2006-2016 Nemanja Trifunovic /* Permission is hereby granted, free of charge, to any person or organization @@ -39,64 +39,62 @@ namespace utf8 // Exceptions that may be thrown from the library functions. class invalid_code_point : public exception { - uint32_t cp; + utfchar32_t cp; public: - invalid_code_point(uint32_t cp) : cp(cp) {} - virtual const char* what() const throw() { return "Invalid code point"; } - uint32_t code_point() const {return cp;} + invalid_code_point(utfchar32_t codepoint) : cp(codepoint) {} + virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; } + utfchar32_t code_point() const {return cp;} }; class invalid_utf8 : public exception { - uint8_t u8; + utfchar8_t u8; public: - invalid_utf8 (uint8_t u) : u8(u) {} - virtual const char* what() const throw() { return "Invalid UTF-8"; } - uint8_t utf8_octet() const {return u8;} + invalid_utf8 (utfchar8_t u) : u8(u) {} + invalid_utf8 (char c) : u8(static_cast<utfchar8_t>(c)) {} + virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; } + utfchar8_t utf8_octet() const {return u8;} }; class invalid_utf16 : public exception { - uint16_t u16; + utfchar16_t u16; public: - invalid_utf16 (uint16_t u) : u16(u) {} - virtual const char* what() const throw() { return "Invalid UTF-16"; } - uint16_t utf16_word() const {return u16;} + invalid_utf16 (utfchar16_t u) : u16(u) {} + virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; } + utfchar16_t utf16_word() const {return u16;} }; class not_enough_room : public exception { public: - virtual const char* what() const throw() { return "Not enough space"; } + virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Not enough space"; } }; /// The library API - functions intended to be called by the users template <typename octet_iterator> - octet_iterator append(uint32_t cp, octet_iterator result) + octet_iterator append(utfchar32_t cp, octet_iterator result) { if (!utf8::internal::is_code_point_valid(cp)) throw invalid_code_point(cp); - if (cp < 0x80) // one octet - *(result++) = static_cast<uint8_t>(cp); - else if (cp < 0x800) { // two octets - *(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) { // three octets - *(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0); - *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); - } - else { // four octets - *(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0); - *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80); - *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); - } - return result; + return internal::append(cp, result); + } + + inline void append(utfchar32_t cp, std::string& s) + { + append(cp, std::back_inserter(s)); + } + + template <typename word_iterator> + word_iterator append16(utfchar32_t cp, word_iterator result) + { + if (!utf8::internal::is_code_point_valid(cp)) + throw invalid_code_point(cp); + + return internal::append16(cp, result); } template <typename octet_iterator, typename output_iterator> - output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement) { while (start != end) { octet_iterator sequence_start = start; @@ -107,7 +105,9 @@ namespace utf8 *out++ = *it; break; case internal::NOT_ENOUGH_ROOM: - throw not_enough_room(); + out = utf8::append (replacement, out); + start = end; + break; case internal::INVALID_LEAD: out = utf8::append (replacement, out); ++start; @@ -129,14 +129,28 @@ namespace utf8 template <typename octet_iterator, typename output_iterator> inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) { - static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); + static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd); return utf8::replace_invalid(start, end, out, replacement_marker); } + inline std::string replace_invalid(const std::string& s, utfchar32_t replacement) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); + return result; + } + + inline std::string replace_invalid(const std::string& s) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + template <typename octet_iterator> - uint32_t next(octet_iterator& it, octet_iterator end) + utfchar32_t next(octet_iterator& it, octet_iterator end) { - uint32_t cp = 0; + utfchar32_t cp = 0; internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); switch (err_code) { case internal::UTF8_OK : @@ -146,21 +160,31 @@ namespace utf8 case internal::INVALID_LEAD : case internal::INCOMPLETE_SEQUENCE : case internal::OVERLONG_SEQUENCE : - throw invalid_utf8(*it); + throw invalid_utf8(static_cast<utfchar8_t>(*it)); case internal::INVALID_CODE_POINT : throw invalid_code_point(cp); } return cp; } + template <typename word_iterator> + utfchar32_t next16(word_iterator& it, word_iterator end) + { + utfchar32_t cp = 0; + internal::utf_error err_code = utf8::internal::validate_next16(it, end, cp); + if (err_code == internal::NOT_ENOUGH_ROOM) + throw not_enough_room(); + return cp; + } + template <typename octet_iterator> - uint32_t peek_next(octet_iterator it, octet_iterator end) + utfchar32_t peek_next(octet_iterator it, octet_iterator end) { return utf8::next(it, end); } template <typename octet_iterator> - uint32_t prior(octet_iterator& it, octet_iterator start) + utfchar32_t prior(octet_iterator& it, octet_iterator start) { // can't do much if it == start if (it == start) @@ -174,23 +198,19 @@ namespace utf8 return utf8::peek_next(it, end); } - /// Deprecated in versions that include "prior" - template <typename octet_iterator> - uint32_t previous(octet_iterator& it, octet_iterator pass_start) - { - octet_iterator end = it; - while (utf8::internal::is_trail(*(--it))) - if (it == pass_start) - throw invalid_utf8(*it); // error - no lead byte in the sequence - octet_iterator temp = it; - return utf8::next(temp, end); - } - template <typename octet_iterator, typename distance_type> void advance (octet_iterator& it, distance_type n, octet_iterator end) { - for (distance_type i = 0; i < n; ++i) - utf8::next(it, end); + const distance_type zero(0); + if (n < zero) { + // backward + for (distance_type i = n; i < zero; ++i) + utf8::prior(it, end); + } else { + // forward + for (distance_type i = zero; i < n; ++i) + utf8::next(it, end); + } } template <typename octet_iterator> @@ -207,23 +227,23 @@ namespace utf8 octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) { while (start != end) { - uint32_t cp = utf8::internal::mask16(*start++); + utfchar32_t cp = utf8::internal::mask16(*start++); // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { if (start != end) { - uint32_t trail_surrogate = utf8::internal::mask16(*start++); + const utfchar32_t trail_surrogate = utf8::internal::mask16(*start++); if (utf8::internal::is_trail_surrogate(trail_surrogate)) cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; else - throw invalid_utf16(static_cast<uint16_t>(trail_surrogate)); + throw invalid_utf16(static_cast<utfchar16_t>(trail_surrogate)); } else - throw invalid_utf16(static_cast<uint16_t>(cp)); + throw invalid_utf16(static_cast<utfchar16_t>(cp)); } // Lone trail surrogate else if (utf8::internal::is_trail_surrogate(cp)) - throw invalid_utf16(static_cast<uint16_t>(cp)); + throw invalid_utf16(static_cast<utfchar16_t>(cp)); result = utf8::append(cp, result); } @@ -233,14 +253,14 @@ namespace utf8 template <typename u16bit_iterator, typename octet_iterator> u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) { - while (start != end) { - uint32_t cp = utf8::next(start, end); + while (start < end) { + const utfchar32_t cp = utf8::next(start, end); if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + *result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); } else - *result++ = static_cast<uint16_t>(cp); + *result++ = static_cast<utfchar16_t>(cp); } return result; } @@ -257,7 +277,7 @@ namespace utf8 template <typename octet_iterator, typename u32bit_iterator> u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) { - while (start != end) + while (start < end) (*result++) = utf8::next(start, end); return result; @@ -265,23 +285,28 @@ namespace utf8 // The iterator class template <typename octet_iterator> - class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> { + class iterator { octet_iterator it; octet_iterator range_start; octet_iterator range_end; public: + typedef utfchar32_t value_type; + typedef utfchar32_t* pointer; + typedef utfchar32_t& reference; + typedef std::ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; iterator () {} explicit iterator (const octet_iterator& octet_it, - const octet_iterator& range_start, - const octet_iterator& range_end) : - it(octet_it), range_start(range_start), range_end(range_end) + const octet_iterator& rangestart, + const octet_iterator& rangeend) : + it(octet_it), range_start(rangestart), range_end(rangeend) { if (it < range_start || it > range_end) throw std::out_of_range("Invalid utf-8 iterator position"); } // the default "big three" are OK octet_iterator base () const { return it; } - uint32_t operator * () const + utfchar32_t operator * () const { octet_iterator temp = it; return utf8::next(temp, range_end); @@ -322,6 +347,13 @@ namespace utf8 } // namespace utf8 -#endif //header guard +#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later +#include "cpp20.h" +#elif UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later +#include "cpp17.h" +#elif UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later +#include "cpp11.h" +#endif // C++ 11 or later +#endif //header guard diff --git a/lib/charset/utf8/core.h b/lib/charset/utf8/core.h index 693d388..4494c53 100644 --- a/lib/charset/utf8/core.h +++ b/lib/charset/utf8/core.h @@ -29,15 +29,42 @@ DEALINGS IN THE SOFTWARE. #define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 #include <iterator> +#include <cstring> +#include <string> + +// Determine the C++ standard version. +// If the user defines UTF_CPP_CPLUSPLUS, use that. +// Otherwise, trust the unreliable predefined macro __cplusplus + +#if !defined UTF_CPP_CPLUSPLUS + #define UTF_CPP_CPLUSPLUS __cplusplus +#endif + +#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later + #define UTF_CPP_OVERRIDE override + #define UTF_CPP_NOEXCEPT noexcept +#else // C++ 98/03 + #define UTF_CPP_OVERRIDE + #define UTF_CPP_NOEXCEPT throw() +#endif // C++ 11 or later + namespace utf8 { - // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers - // You may need to change them to match your system. - // These typedefs have the same names as ones from cstdint, or boost/cstdint - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; +// The typedefs for 8-bit, 16-bit and 32-bit code units +#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later + #if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later + typedef char8_t utfchar8_t; + #else // C++ 11/14/17 + typedef unsigned char utfchar8_t; + #endif + typedef char16_t utfchar16_t; + typedef char32_t utfchar32_t; +#else // C++ 98/03 + typedef unsigned char utfchar8_t; + typedef unsigned short utfchar16_t; + typedef unsigned int utfchar32_t; +#endif // C++ 11 or later // Helper code - not intended to be directly called by the library users. May be changed at any time namespace internal @@ -45,61 +72,62 @@ namespace internal // Unicode constants // Leading (high) surrogates: 0xd800 - 0xdbff // Trailing (low) surrogates: 0xdc00 - 0xdfff - const uint16_t LEAD_SURROGATE_MIN = 0xd800u; - const uint16_t LEAD_SURROGATE_MAX = 0xdbffu; - const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u; - const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu; - const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); - const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + const utfchar16_t LEAD_SURROGATE_MIN = 0xd800u; + const utfchar16_t LEAD_SURROGATE_MAX = 0xdbffu; + const utfchar16_t TRAIL_SURROGATE_MIN = 0xdc00u; + const utfchar16_t TRAIL_SURROGATE_MAX = 0xdfffu; + const utfchar16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10) + const utfchar32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN // Maximum valid value for a Unicode code point - const uint32_t CODE_POINT_MAX = 0x0010ffffu; + const utfchar32_t CODE_POINT_MAX = 0x0010ffffu; template<typename octet_type> - inline uint8_t mask8(octet_type oc) + inline utfchar8_t mask8(octet_type oc) { - return static_cast<uint8_t>(0xff & oc); + return static_cast<utfchar8_t>(0xff & oc); } template<typename u16_type> - inline uint16_t mask16(u16_type oc) + inline utfchar16_t mask16(u16_type oc) { - return static_cast<uint16_t>(0xffff & oc); + return static_cast<utfchar16_t>(0xffff & oc); } + template<typename octet_type> inline bool is_trail(octet_type oc) { return ((utf8::internal::mask8(oc) >> 6) == 0x2); } - template <typename u16> - inline bool is_lead_surrogate(u16 cp) + inline bool is_lead_surrogate(utfchar32_t cp) { return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); } - template <typename u16> - inline bool is_trail_surrogate(u16 cp) + inline bool is_trail_surrogate(utfchar32_t cp) { return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); } - template <typename u16> - inline bool is_surrogate(u16 cp) + inline bool is_surrogate(utfchar32_t cp) { return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); } - template <typename u32> - inline bool is_code_point_valid(u32 cp) + inline bool is_code_point_valid(utfchar32_t cp) { return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); } + inline bool is_in_bmp(utfchar32_t cp) + { + return cp < utfchar32_t(0x10000); + } + template <typename octet_iterator> - inline typename std::iterator_traits<octet_iterator>::difference_type - sequence_length(octet_iterator lead_it) + int sequence_length(octet_iterator lead_it) { - uint8_t lead = utf8::internal::mask8(*lead_it); + const utfchar8_t lead = utf8::internal::mask8(*lead_it); if (lead < 0x80) return 1; else if ((lead >> 5) == 0x6) @@ -112,8 +140,7 @@ namespace internal return 0; } - template <typename octet_difference_type> - inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) + inline bool is_overlong_sequence(utfchar32_t cp, int length) { if (cp < 0x80) { if (length != 1) @@ -127,7 +154,6 @@ namespace internal if (length != 3) return true; } - return false; } @@ -135,22 +161,22 @@ namespace internal /// Helper for get_sequence_x template <typename octet_iterator> - utf_error increase_safely(octet_iterator& it, octet_iterator end) + utf_error increase_safely(octet_iterator& it, const octet_iterator end) { if (++it == end) return NOT_ENOUGH_ROOM; if (!utf8::internal::is_trail(*it)) return INCOMPLETE_SEQUENCE; - + return UTF8_OK; } - #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} + #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} /// get_sequence_x functions decode utf-8 sequences of the length x template <typename octet_iterator> - utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_1(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -161,11 +187,11 @@ namespace internal } template <typename octet_iterator> - utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_2(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; - + code_point = utf8::internal::mask8(*it); UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) @@ -176,7 +202,7 @@ namespace internal } template <typename octet_iterator> - utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_3(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -195,7 +221,7 @@ namespace internal } template <typename octet_iterator> - utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error get_sequence_4(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) { if (it == end) return NOT_ENOUGH_ROOM; @@ -220,21 +246,23 @@ namespace internal #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR template <typename octet_iterator> - utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point) + utf_error validate_next(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) { + if (it == end) + return NOT_ENOUGH_ROOM; + // Save the original value of it so we can go back in case of failure // Of course, it does not make much sense with i.e. stream iterators octet_iterator original_it = it; - uint32_t cp = 0; + utfchar32_t cp = 0; // Determine the sequence length based on the lead octet - typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type; - const octet_difference_type length = utf8::internal::sequence_length(it); + const int length = utf8::internal::sequence_length(it); // Get trail octets and calculate the code point utf_error err = UTF8_OK; switch (length) { - case 0: + case 0: return INVALID_LEAD; case 1: err = utf8::internal::get_sequence_1(it, end, cp); @@ -273,16 +301,133 @@ namespace internal template <typename octet_iterator> inline utf_error validate_next(octet_iterator& it, octet_iterator end) { - uint32_t ignored; + utfchar32_t ignored; return utf8::internal::validate_next(it, end, ignored); } + template <typename word_iterator> + utf_error validate_next16(word_iterator& it, word_iterator end, utfchar32_t& code_point) + { + if (it == end) + return NOT_ENOUGH_ROOM; + // Save the original value of it so we can go back in case of failure + // Of course, it does not make much sense with i.e. stream iterators + word_iterator original_it = it; + + utf_error err = UTF8_OK; + + const utfchar16_t first_word = *it++; + if (!is_surrogate(first_word)) { + code_point = first_word; + return UTF8_OK; + } + else { + if (it == end) + err = NOT_ENOUGH_ROOM; + else if (is_lead_surrogate(first_word)) { + const utfchar16_t second_word = *it++; + if (is_trail_surrogate(second_word)) { + code_point = (first_word << 10) + second_word + SURROGATE_OFFSET; + return UTF8_OK; + } else + err = INCOMPLETE_SEQUENCE; + + } else { + err = INVALID_LEAD; + } + } + // error branch + it = original_it; + return err; + } + + // Internal implementation of both checked and unchecked append() function + // This function will be invoked by the overloads below, as they will know + // the octet_type. + template <typename octet_iterator, typename octet_type> + octet_iterator append(utfchar32_t cp, octet_iterator result) { + if (cp < 0x80) // one octet + *(result++) = static_cast<octet_type>(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast<octet_type>((cp >> 6) | 0xc0); + *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast<octet_type>((cp >> 12) | 0xe0); + *(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast<octet_type>((cp >> 18) | 0xf0); + *(result++) = static_cast<octet_type>(((cp >> 12) & 0x3f)| 0x80); + *(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80); + *(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80); + } + return result; + } + + // One of the following overloads will be invoked from the API calls + + // A simple (but dangerous) case: the caller appends byte(s) to a char array + inline char* append(utfchar32_t cp, char* result) { + return append<char*, char>(cp, result); + } + + // Hopefully, most common case: the caller uses back_inserter + // i.e. append(cp, std::back_inserter(str)); + template<typename container_type> + std::back_insert_iterator<container_type> append + (utfchar32_t cp, std::back_insert_iterator<container_type> result) { + return append<std::back_insert_iterator<container_type>, + typename container_type::value_type>(cp, result); + } + + // The caller uses some other kind of output operator - not covered above + // Note that in this case we are not able to determine octet_type + // so we assume it's utfchar8_t; that can cause a conversion warning if we are wrong. + template <typename octet_iterator> + octet_iterator append(utfchar32_t cp, octet_iterator result) { + return append<octet_iterator, utfchar8_t>(cp, result); + } + + // Internal implementation of both checked and unchecked append16() function + // This function will be invoked by the overloads below, as they will know + // the word_type. + template <typename word_iterator, typename word_type> + word_iterator append16(utfchar32_t cp, word_iterator result) { + if (is_in_bmp(cp)) + *(result++) = static_cast<word_type>(cp); + else { + // Code points from the supplementary planes are encoded via surrogate pairs + *(result++) = static_cast<word_type>(LEAD_OFFSET + (cp >> 10)); + *(result++) = static_cast<word_type>(TRAIL_SURROGATE_MIN + (cp & 0x3FF)); + } + return result; + } + + // Hopefully, most common case: the caller uses back_inserter + // i.e. append16(cp, std::back_inserter(str)); + template<typename container_type> + std::back_insert_iterator<container_type> append16 + (utfchar32_t cp, std::back_insert_iterator<container_type> result) { + return append16<std::back_insert_iterator<container_type>, + typename container_type::value_type>(cp, result); + } + + // The caller uses some other kind of output operator - not covered above + // Note that in this case we are not able to determine word_type + // so we assume it's utfchar16_t; that can cause a conversion warning if we are wrong. + template <typename word_iterator> + word_iterator append16(utfchar32_t cp, word_iterator result) { + return append16<word_iterator, utfchar16_t>(cp, result); + } + } // namespace internal /// The library API - functions intended to be called by the users // Byte order mark - const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + const utfchar8_t bom[] = {0xef, 0xbb, 0xbf}; template <typename octet_iterator> octet_iterator find_invalid(octet_iterator start, octet_iterator end) @@ -296,12 +441,36 @@ namespace internal return result; } + inline const char* find_invalid(const char* str) + { + const char* end = str + std::strlen(str); + return find_invalid(str, end); + } + + inline std::size_t find_invalid(const std::string& s) + { + std::string::const_iterator invalid = find_invalid(s.begin(), s.end()); + return (invalid == s.end()) ? std::string::npos : static_cast<std::size_t>(invalid - s.begin()); + } + template <typename octet_iterator> inline bool is_valid(octet_iterator start, octet_iterator end) { return (utf8::find_invalid(start, end) == end); } + inline bool is_valid(const char* str) + { + return (*(utf8::find_invalid(str)) == '\0'); + } + + inline bool is_valid(const std::string& s) + { + return is_valid(s.begin(), s.end()); + } + + + template <typename octet_iterator> inline bool starts_with_bom (octet_iterator it, octet_iterator end) { @@ -311,17 +480,11 @@ namespace internal ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) ); } - - //Deprecated in release 2.3 - template <typename octet_iterator> - inline bool is_bom (octet_iterator it) + + inline bool starts_with_bom(const std::string& s) { - return ( - (utf8::internal::mask8(*it++)) == bom[0] && - (utf8::internal::mask8(*it++)) == bom[1] && - (utf8::internal::mask8(*it)) == bom[2] - ); - } + return starts_with_bom(s.begin(), s.end()); + } } // namespace utf8 #endif // header guard diff --git a/lib/charset/utf8/cpp11.h b/lib/charset/utf8/cpp11.h new file mode 100644 index 0000000..691633c --- /dev/null +++ b/lib/charset/utf8/cpp11.h @@ -0,0 +1,70 @@ +// Copyright 2018 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 +#define UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 + +#include "checked.h" + +namespace utf8 +{ + inline void append16(utfchar32_t cp, std::u16string& s) + { + append16(cp, std::back_inserter(s)); + } + + inline std::string utf16to8(const std::u16string& s) + { + std::string result; + utf16to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u16string utf8to16(const std::string& s) + { + std::u16string result; + utf8to16(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::string utf32to8(const std::u32string& s) + { + std::string result; + utf32to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u32string utf8to32(const std::string& s) + { + std::u32string result; + utf8to32(s.begin(), s.end(), std::back_inserter(result)); + return result; + } +} // namespace utf8 + +#endif // header guard + diff --git a/lib/charset/utf8/cpp17.h b/lib/charset/utf8/cpp17.h new file mode 100644 index 0000000..6e2fcc2 --- /dev/null +++ b/lib/charset/utf8/cpp17.h @@ -0,0 +1,96 @@ +// Copyright 2018 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9 +#define UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9 + +#include "cpp11.h" + +namespace utf8 +{ + inline std::string utf16to8(std::u16string_view s) + { + std::string result; + utf16to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u16string utf8to16(std::string_view s) + { + std::u16string result; + utf8to16(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::string utf32to8(std::u32string_view s) + { + std::string result; + utf32to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u32string utf8to32(std::string_view s) + { + std::u32string result; + utf8to32(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::size_t find_invalid(std::string_view s) + { + std::string_view::const_iterator invalid = find_invalid(s.begin(), s.end()); + return (invalid == s.end()) ? std::string_view::npos : static_cast<std::size_t>(invalid - s.begin()); + } + + inline bool is_valid(std::string_view s) + { + return is_valid(s.begin(), s.end()); + } + + inline std::string replace_invalid(std::string_view s, char32_t replacement) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); + return result; + } + + inline std::string replace_invalid(std::string_view s) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline bool starts_with_bom(std::string_view s) + { + return starts_with_bom(s.begin(), s.end()); + } + +} // namespace utf8 + +#endif // header guard + diff --git a/lib/charset/utf8/cpp20.h b/lib/charset/utf8/cpp20.h new file mode 100644 index 0000000..07b61d0 --- /dev/null +++ b/lib/charset/utf8/cpp20.h @@ -0,0 +1,124 @@ +// Copyright 2022 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9 +#define UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9 + +#include "cpp17.h" + +namespace utf8 +{ + inline std::u8string utf16tou8(const std::u16string& s) + { + std::u8string result; + utf16to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u8string utf16tou8(std::u16string_view s) + { + std::u8string result; + utf16to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u16string utf8to16(const std::u8string& s) + { + std::u16string result; + utf8to16(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u16string utf8to16(const std::u8string_view& s) + { + std::u16string result; + utf8to16(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u8string utf32tou8(const std::u32string& s) + { + std::u8string result; + utf32to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u8string utf32tou8(const std::u32string_view& s) + { + std::u8string result; + utf32to8(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u32string utf8to32(const std::u8string& s) + { + std::u32string result; + utf8to32(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::u32string utf8to32(const std::u8string_view& s) + { + std::u32string result; + utf8to32(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline std::size_t find_invalid(const std::u8string& s) + { + std::u8string::const_iterator invalid = find_invalid(s.begin(), s.end()); + return (invalid == s.end()) ? std::string_view::npos : static_cast<std::size_t>(invalid - s.begin()); + } + + inline bool is_valid(const std::u8string& s) + { + return is_valid(s.begin(), s.end()); + } + + inline std::u8string replace_invalid(const std::u8string& s, char32_t replacement) + { + std::u8string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); + return result; + } + + inline std::u8string replace_invalid(const std::u8string& s) + { + std::u8string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result)); + return result; + } + + inline bool starts_with_bom(const std::u8string& s) + { + return starts_with_bom(s.begin(), s.end()); + } + +} // namespace utf8 + +#endif // header guard + diff --git a/lib/charset/utf8/unchecked.h b/lib/charset/utf8/unchecked.h index cb24271..65d4948 100644 --- a/lib/charset/utf8/unchecked.h +++ b/lib/charset/utf8/unchecked.h @@ -32,37 +32,79 @@ DEALINGS IN THE SOFTWARE. namespace utf8 { - namespace unchecked + namespace unchecked { template <typename octet_iterator> - octet_iterator append(uint32_t cp, octet_iterator result) + octet_iterator append(utfchar32_t cp, octet_iterator result) { - if (cp < 0x80) // one octet - *(result++) = static_cast<uint8_t>(cp); - else if (cp < 0x800) { // two octets - *(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) { // three octets - *(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0); - *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); - } - else { // four octets - *(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0); - *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80); - *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80); + return internal::append(cp, result); + } + + template <typename word_iterator> + word_iterator append16(utfchar32_t cp, word_iterator result) + { + return internal::append16(cp, result); + } + + template <typename octet_iterator, typename output_iterator> + output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement) + { + while (start != end) { + octet_iterator sequence_start = start; + internal::utf_error err_code = utf8::internal::validate_next(start, end); + switch (err_code) { + case internal::UTF8_OK : + for (octet_iterator it = sequence_start; it != start; ++it) + *out++ = *it; + break; + case internal::NOT_ENOUGH_ROOM: + out = utf8::unchecked::append(replacement, out); + start = end; + break; + case internal::INVALID_LEAD: + out = utf8::unchecked::append(replacement, out); + ++start; + break; + case internal::INCOMPLETE_SEQUENCE: + case internal::OVERLONG_SEQUENCE: + case internal::INVALID_CODE_POINT: + out = utf8::unchecked::append(replacement, out); + ++start; + // just one replacement mark for the sequence + while (start != end && utf8::internal::is_trail(*start)) + ++start; + break; + } } + return out; + } + + template <typename octet_iterator, typename output_iterator> + inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) + { + static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd); + return utf8::unchecked::replace_invalid(start, end, out, replacement_marker); + } + + inline std::string replace_invalid(const std::string& s, utfchar32_t replacement) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); + return result; + } + + inline std::string replace_invalid(const std::string& s) + { + std::string result; + replace_invalid(s.begin(), s.end(), std::back_inserter(result)); return result; } template <typename octet_iterator> - uint32_t next(octet_iterator& it) + utfchar32_t next(octet_iterator& it) { - uint32_t cp = utf8::internal::mask8(*it); - typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it); - switch (length) { + utfchar32_t cp = utf8::internal::mask8(*it); + switch (utf8::internal::sequence_length(it)) { case 1: break; case 2: @@ -85,40 +127,50 @@ namespace utf8 break; } ++it; - return cp; + return cp; } template <typename octet_iterator> - uint32_t peek_next(octet_iterator it) + utfchar32_t peek_next(octet_iterator it) { - return utf8::unchecked::next(it); + return utf8::unchecked::next(it); } - template <typename octet_iterator> - uint32_t prior(octet_iterator& it) + template <typename word_iterator> + utfchar32_t next16(word_iterator& it) { - while (utf8::internal::is_trail(*(--it))) ; - octet_iterator temp = it; - return utf8::unchecked::next(temp); + utfchar32_t cp = utf8::internal::mask16(*it++); + if (utf8::internal::is_lead_surrogate(cp)) + return (cp << 10) + *it++ + utf8::internal::SURROGATE_OFFSET; + return cp; } - // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous) template <typename octet_iterator> - inline uint32_t previous(octet_iterator& it) + utfchar32_t prior(octet_iterator& it) { - return utf8::unchecked::prior(it); + while (utf8::internal::is_trail(*(--it))) ; + octet_iterator temp = it; + return utf8::unchecked::next(temp); } template <typename octet_iterator, typename distance_type> - void advance (octet_iterator& it, distance_type n) + void advance(octet_iterator& it, distance_type n) { - for (distance_type i = 0; i < n; ++i) - utf8::unchecked::next(it); + const distance_type zero(0); + if (n < zero) { + // backward + for (distance_type i = n; i < zero; ++i) + utf8::unchecked::prior(it); + } else { + // forward + for (distance_type i = zero; i < n; ++i) + utf8::unchecked::next(it); + } } template <typename octet_iterator> typename std::iterator_traits<octet_iterator>::difference_type - distance (octet_iterator first, octet_iterator last) + distance(octet_iterator first, octet_iterator last) { typename std::iterator_traits<octet_iterator>::difference_type dist; for (dist = 0; first < last; ++dist) @@ -127,37 +179,39 @@ namespace utf8 } template <typename u16bit_iterator, typename octet_iterator> - octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) - { + octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { while (start != end) { - uint32_t cp = utf8::internal::mask16(*start++); - // Take care of surrogate pairs first + utfchar32_t cp = utf8::internal::mask16(*start++); + // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { - uint32_t trail_surrogate = utf8::internal::mask16(*start++); + if (start == end) + return result; + utfchar32_t trail_surrogate = utf8::internal::mask16(*start++); cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; } result = utf8::unchecked::append(cp, result); } - return result; + return result; } template <typename u16bit_iterator, typename octet_iterator> - u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result) { while (start < end) { - uint32_t cp = utf8::unchecked::next(start); + utfchar32_t cp = utf8::unchecked::next(start); if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); + *result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET); + *result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); } else - *result++ = static_cast<uint16_t>(cp); + *result++ = static_cast<utfchar16_t>(cp); } return result; } template <typename octet_iterator, typename u32bit_iterator> - octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result) { while (start != end) result = utf8::unchecked::append(*(start++), result); @@ -166,7 +220,7 @@ namespace utf8 } template <typename octet_iterator, typename u32bit_iterator> - u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result) { while (start < end) (*result++) = utf8::unchecked::next(start); @@ -176,14 +230,19 @@ namespace utf8 // The iterator class template <typename octet_iterator> - class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> { + class iterator { octet_iterator it; public: + typedef utfchar32_t value_type; + typedef utfchar32_t* pointer; + typedef utfchar32_t& reference; + typedef std::ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; iterator () {} explicit iterator (const octet_iterator& octet_it): it(octet_it) {} // the default "big three" are OK octet_iterator base () const { return it; } - uint32_t operator * () const + utfchar32_t operator * () const { octet_iterator temp = it; return utf8::unchecked::next(temp); diff --git a/src/DabMultiplexer.cpp b/src/DabMultiplexer.cpp index fb5ec45..908caf9 100644 --- a/src/DabMultiplexer.cpp +++ b/src/DabMultiplexer.cpp @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2020 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li */ /* @@ -820,3 +820,11 @@ const std::string DabMultiplexer::get_parameter(const std::string& parameter) co } +const json::map_t DabMultiplexer::get_all_values() const +{ + json::map_t map; + map["frames"].v = m_currentFrame; + map["tist_offset"].v = m_tist_offset; + return map; +} + diff --git a/src/DabMultiplexer.h b/src/DabMultiplexer.h index 3b521ff..90e5767 100644 --- a/src/DabMultiplexer.h +++ b/src/DabMultiplexer.h @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li */ /* @@ -76,6 +76,8 @@ class DabMultiplexer : public RemoteControllable { /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + virtual const json::map_t get_all_values() const; + private: void prepare_subchannels(void); void prepare_services_components(void); diff --git a/src/ManagementServer.cpp b/src/ManagementServer.cpp index 599d744..568e80e 100644 --- a/src/ManagementServer.cpp +++ b/src/ManagementServer.cpp @@ -280,8 +280,10 @@ void ManagementServer::serverThread() if (pollItems[0].revents & ZMQ_POLLIN) { zmq::message_t zmq_message; - m_zmq_sock.recv(zmq_message); - handle_message(zmq_message); + const auto r = m_zmq_sock.recv(zmq_message); + if (r.has_value()) { + handle_message(zmq_message); + } } } } diff --git a/src/MuxElements.cpp b/src/MuxElements.cpp index 71ff270..d17b283 100644 --- a/src/MuxElements.cpp +++ b/src/MuxElements.cpp @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2020 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -86,7 +86,7 @@ bool AnnouncementCluster::is_active() if (*m_deferred_start_time <= now) { m_active = true; - m_deferred_start_time = boost::none; + m_deferred_start_time.reset(); } } @@ -96,7 +96,7 @@ bool AnnouncementCluster::is_active() if (*m_deferred_stop_time <= now) { m_active = false; - m_deferred_stop_time = boost::none; + m_deferred_stop_time.reset(); } } @@ -180,6 +180,34 @@ const string AnnouncementCluster::get_parameter(const string& parameter) const return ss.str(); } +const json::map_t AnnouncementCluster::get_all_values() const +{ + json::map_t map; + + lock_guard<mutex> lock(m_active_mutex); + map["active"].v = m_active; + + using namespace std::chrono; + + if (m_deferred_start_time) { + const auto diff = *m_deferred_start_time - steady_clock::now(); + map["start_in"].v = duration_cast<milliseconds>(diff).count(); + } + else { + map["start_in"].v = nullopt; + } + + if (m_deferred_stop_time) { + const auto diff = *m_deferred_stop_time - steady_clock::now(); + map["stop_in"].v = duration_cast<milliseconds>(diff).count(); + } + else { + map["stop_in"].v = nullopt; + } + + return map; +} + int DabLabel::setLabel(const std::string& label) { @@ -512,6 +540,16 @@ const string DabComponent::get_parameter(const string& parameter) const } +const json::map_t DabComponent::get_all_values() const +{ + json::map_t map; + // It's cleaner to have it separate in JSON, but we + // need the comma separated variant for setting + map["label"].v = label.long_label(); + map["shortlabel"].v = label.short_label(); + return map; +} + subchannel_type_t DabService::getType( const std::shared_ptr<dabEnsemble> ensemble) const { @@ -638,6 +676,16 @@ const string DabService::get_parameter(const string& parameter) const return ss.str(); } +const json::map_t DabService::get_all_values() const +{ + json::map_t map; + map["label"].v = label.long_label(); + map["shortlabel"].v = label.short_label(); + map["pty"].v = (int)pty_settings.pty; + map["ptysd"].v = (pty_settings.dynamic_no_static ? "dynamic" : "static"); + return map; +} + void dabEnsemble::set_parameter(const string& parameter, const string& value) { if (parameter == "localtimeoffset") { @@ -687,6 +735,14 @@ const string dabEnsemble::get_parameter(const string& parameter) const return ss.str(); } +const json::map_t dabEnsemble::get_all_values() const +{ + json::map_t map; + map["localtimeoffset_auto"].v = lto_auto; + map["localtimeoffset"].v = lto; + return map; +} + bool dabEnsemble::validate_linkage_sets() { for (const auto& ls : linkagesets) { diff --git a/src/MuxElements.h b/src/MuxElements.h index 1e9b707..d118df9 100644 --- a/src/MuxElements.h +++ b/src/MuxElements.h @@ -3,7 +3,7 @@ 2011, 2012 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2022 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -37,7 +37,7 @@ #include <exception> #include <algorithm> #include <chrono> -#include <boost/optional.hpp> +#include <optional> #include <stdint.h> #include "dabOutput/dabOutput.h" #include "input/inputs.h" @@ -150,10 +150,10 @@ class AnnouncementCluster : public RemoteControllable { private: mutable std::mutex m_active_mutex; bool m_active = false; - boost::optional< + std::optional< std::chrono::time_point< std::chrono::steady_clock> > m_deferred_start_time; - boost::optional< + std::optional< std::chrono::time_point< std::chrono::steady_clock> > m_deferred_stop_time; @@ -163,6 +163,8 @@ class AnnouncementCluster : public RemoteControllable { /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + + virtual const json::map_t get_all_values() const; }; struct dabOutput { @@ -310,6 +312,8 @@ class dabEnsemble : public RemoteControllable { /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + virtual const json::map_t get_all_values() const; + /* Check if the Linkage Sets are valid */ bool validate_linkage_sets(void); @@ -483,6 +487,8 @@ class DabComponent : public RemoteControllable /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + + virtual const json::map_t get_all_values() const; }; class DabService : public RemoteControllable @@ -536,6 +542,8 @@ class DabService : public RemoteControllable /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + + virtual const json::map_t get_all_values() const; }; /* Represent an entry for FIG0/24 */ diff --git a/src/dabOutput/dabOutputZMQ.cpp b/src/dabOutput/dabOutputZMQ.cpp index 0680f21..9b119f6 100644 --- a/src/dabOutput/dabOutputZMQ.cpp +++ b/src/dabOutput/dabOutputZMQ.cpp @@ -59,7 +59,6 @@ int DabOutputZMQ::Open(const char* endpoint) { // bind to uri string proto_endpoint = zmq_proto_ + "://" + std::string(endpoint); - std::cerr << "ZMQ socket " << proto_endpoint << std::endl; zmq_pub_sock_.bind(proto_endpoint.c_str()); endpoint_ = endpoint; diff --git a/src/input/Edi.cpp b/src/input/Edi.cpp index 692ecb3..3838541 100644 --- a/src/input/Edi.cpp +++ b/src/input/Edi.cpp @@ -2,7 +2,7 @@ Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -470,7 +470,7 @@ const std::string Edi::get_parameter(const std::string& parameter) const ss << "prebuffering"; break; case Inputs::BufferManagement::Timestamped: - ss << "Timestamped"; + ss << "timestamped"; break; } } @@ -483,4 +483,21 @@ const std::string Edi::get_parameter(const std::string& parameter) const return ss.str(); } +const json::map_t Edi::get_all_values() const +{ + json::map_t map; + map["buffer"].v = m_max_frames_overrun; + map["prebuffering"].v = m_num_frames_prebuffering; + switch (getBufferManagement()) { + case Inputs::BufferManagement::Prebuffering: + map["buffermanagement"].v = "prebuffering"; + break; + case Inputs::BufferManagement::Timestamped: + map["buffermanagement"].v = "timestamped"; + break; + } + map["tistdelay"].v = m_tist_delay.count(); + return map; +} + } diff --git a/src/input/Edi.h b/src/input/Edi.h index cf4c576..3de17a7 100644 --- a/src/input/Edi.h +++ b/src/input/Edi.h @@ -2,7 +2,7 @@ Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications Research Center Canada) - Copyright (C) 2019 + Copyright (C) 2024 Matthias P. Braendli, matthias.braendli@mpb.li http://www.opendigitalradio.org @@ -78,6 +78,7 @@ class Edi : public InputBase, public RemoteControllable { /* Remote control */ virtual void set_parameter(const std::string& parameter, const std::string& value); virtual const std::string get_parameter(const std::string& parameter) const; + virtual const json::map_t get_all_values() const; protected: void m_run(); diff --git a/src/input/Zmq.cpp b/src/input/Zmq.cpp index be3fd1f..d5e722e 100644 --- a/src/input/Zmq.cpp +++ b/src/input/Zmq.cpp @@ -614,7 +614,19 @@ const string ZmqBase::get_parameter(const string& parameter) const throw ParameterError(ss.str()); } return ss.str(); +} +const json::map_t ZmqBase::get_all_values() const +{ + json::map_t map; + map["buffer"].v = m_config.buffer_size; + map["prebuffering"].v = m_config.prebuffering; + map["enable"].v = m_enable_input; + map["encryption"].v = m_config.enable_encryption; + map["secretkey"].v = m_config.curve_secret_keyfile; + map["publickey"].v = m_config.curve_public_keyfile; + map["encoderkey"].v = m_config.curve_encoder_keyfile; + return map; } }; diff --git a/src/input/Zmq.h b/src/input/Zmq.h index c101da0..72fccbd 100644 --- a/src/input/Zmq.h +++ b/src/input/Zmq.h @@ -194,6 +194,7 @@ class ZmqBase : public InputBase, public RemoteControllable { /* Getting a parameter always returns a string. */ virtual const std::string get_parameter(const std::string& parameter) const; + virtual const json::map_t get_all_values() const; protected: virtual int readFromSocket(size_t framesize) = 0; |