From 43f4a3a2a695c303bd4fdfbd7fec6def29284f2e Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Tue, 28 May 2019 16:56:43 +0200 Subject: Unify Socket abstractions --- lib/Socket.cpp | 730 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/Socket.h | 240 +++++++++++++++++++ 2 files changed, 970 insertions(+) create mode 100644 lib/Socket.cpp create mode 100644 lib/Socket.h (limited to 'lib') diff --git a/lib/Socket.cpp b/lib/Socket.cpp new file mode 100644 index 0000000..fe3df44 --- /dev/null +++ b/lib/Socket.cpp @@ -0,0 +1,730 @@ +/* + Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the + Queen in Right of Canada (Communications Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include "Socket.h" + +#include +#include +#include +#include +#include +#include + +namespace Socket { + +using namespace std; + +void InetAddress::resolveUdpDestination(const std::string& destination, int port) +{ + char service[NI_MAXSERV]; + snprintf(service, NI_MAXSERV-1, "%d", port); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ + hints.ai_flags = 0; + hints.ai_protocol = 0; + hints.ai_canonname = nullptr; + hints.ai_addr = nullptr; + hints.ai_next = nullptr; + + struct addrinfo *result, *rp; + int s = getaddrinfo(destination.c_str(), service, &hints, &result); + if (s != 0) { + throw runtime_error(string("getaddrinfo failed: ") + gai_strerror(s)); + } + + for (rp = result; rp != nullptr; rp = rp->ai_next) { + // Take the first result + memcpy(&addr, rp->ai_addr, rp->ai_addrlen); + break; + } + + freeaddrinfo(result); + + if (rp == nullptr) { + throw runtime_error("Could not resolve"); + } +} + +UDPPacket::UDPPacket() { } + +UDPPacket::UDPPacket(size_t initSize) : + buffer(initSize) +{ } + + +UDPSocket::UDPSocket() : + listenSocket(INVALID_SOCKET) +{ + reinit(0, ""); +} + +UDPSocket::UDPSocket(int port) : + listenSocket(INVALID_SOCKET) +{ + reinit(port, ""); +} + +UDPSocket::UDPSocket(int port, const std::string& name) : + listenSocket(INVALID_SOCKET) +{ + reinit(port, name); +} + + +void UDPSocket::setBlocking(bool block) +{ + int res = fcntl(listenSocket, F_SETFL, block ? 0 : O_NONBLOCK); + if (res == -1) { + throw runtime_error(string("Can't change blocking state of socket: ") + strerror(errno)); + } +} + +void UDPSocket::reinit(int port, const std::string& name) +{ + if (listenSocket != INVALID_SOCKET) { + ::close(listenSocket); + } + + char service[NI_MAXSERV]; + snprintf(service, NI_MAXSERV-1, "%d", port); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ + hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ + hints.ai_protocol = 0; /* Any protocol */ + hints.ai_canonname = nullptr; + hints.ai_addr = nullptr; + hints.ai_next = nullptr; + + struct addrinfo *result, *rp; + int s = getaddrinfo(name.empty() ? nullptr : name.c_str(), + port == 0 ? nullptr : service, + &hints, &result); + if (s != 0) { + throw runtime_error(string("getaddrinfo failed: ") + gai_strerror(s)); + } + + /* getaddrinfo() returns a list of address structures. + Try each address until we successfully bind(2). + If socket(2) (or bind(2)) fails, we (close the socket + and) try the next address. */ + for (rp = result; rp != nullptr; rp = rp->ai_next) { + int sfd = ::socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (sfd == -1) { + continue; + } + + if (::bind(sfd, rp->ai_addr, rp->ai_addrlen) == 0) { + listenSocket = sfd; + break; + } + + ::close(sfd); + } + + freeaddrinfo(result); + + if (rp == nullptr) { + throw runtime_error("Could not bind"); + } +} + +void UDPSocket::close() +{ + if (listenSocket != INVALID_SOCKET) { + ::close(listenSocket); + } + + listenSocket = INVALID_SOCKET; +} + +UDPSocket::~UDPSocket() +{ + if (listenSocket != INVALID_SOCKET) { + ::close(listenSocket); + } +} + + +UDPPacket UDPSocket::receive(size_t max_size) +{ + UDPPacket packet(max_size); + socklen_t addrSize; + addrSize = sizeof(*packet.address.as_sockaddr()); + ssize_t ret = recvfrom(listenSocket, + packet.buffer.data(), + packet.buffer.size(), + 0, + packet.address.as_sockaddr(), + &addrSize); + + if (ret == SOCKET_ERROR) { + packet.buffer.resize(0); + if (errno == EAGAIN) { + return 0; + } + throw runtime_error(string("Can't receive data: ") + strerror(errno)); + } + + packet.buffer.resize(ret); + return packet; +} + +void UDPSocket::send(UDPPacket& packet) +{ + int ret = sendto(listenSocket, packet.buffer.data(), packet.buffer.size(), 0, + packet.address.as_sockaddr(), sizeof(*packet.address.as_sockaddr())); + if (ret == SOCKET_ERROR && errno != ECONNREFUSED) { + throw runtime_error(string("Can't send UDP packet") + strerror(errno)); + } +} + + +void UDPSocket::send(const std::vector& data, InetAddress destination) +{ + int ret = sendto(listenSocket, &data[0], data.size(), 0, + destination.as_sockaddr(), sizeof(*destination.as_sockaddr())); + if (ret == SOCKET_ERROR && errno != ECONNREFUSED) { + throw runtime_error(string("Can't send UDP packet") + strerror(errno)); + } +} + +void UDPSocket::joinGroup(char* groupname) +{ + ip_mreqn group; + if ((group.imr_multiaddr.s_addr = inet_addr(groupname)) == INADDR_NONE) { + throw runtime_error("Cannot convert multicast group name"); + } + if (!IN_MULTICAST(ntohl(group.imr_multiaddr.s_addr))) { + throw runtime_error("Group name is not a multicast address"); + } + group.imr_address.s_addr = htons(INADDR_ANY);; + group.imr_ifindex = 0; + if (setsockopt(listenSocket, IPPROTO_IP, IP_ADD_MEMBERSHIP, &group, sizeof(group)) + == SOCKET_ERROR) { + throw runtime_error(string("Can't join multicast group") + strerror(errno)); + } +} + +void UDPSocket::setMulticastSource(const char* source_addr) +{ + struct in_addr addr; + if (inet_aton(source_addr, &addr) == 0) { + throw runtime_error(string("Can't parse source address") + strerror(errno)); + } + + if (setsockopt(listenSocket, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) + == SOCKET_ERROR) { + throw runtime_error(string("Can't set source address") + strerror(errno)); + } +} + +void UDPSocket::setMulticastTTL(int ttl) +{ + if (setsockopt(listenSocket, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) + == SOCKET_ERROR) { + throw runtime_error(string("Can't set multicast ttl") + strerror(errno)); + } +} + + +TCPSocket::TCPSocket() +{ + if ((m_sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) { + throw std::runtime_error("Can't create TCP socket"); + } + +#if defined(HAVE_SO_NOSIGPIPE) + int val = 1; + if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, + &val, sizeof(val)) < 0) { + throw std::runtime_error("Can't set SO_NOSIGPIPE"); + } +#endif +} + +TCPSocket::~TCPSocket() +{ + if (m_sock != -1) { + ::close(m_sock); + } +} + +TCPSocket::TCPSocket(TCPSocket&& other) : + m_sock(other.m_sock), + m_remote_address(other.m_remote_address) +{ + if (other.m_sock != -1) { + other.m_sock = -1; + } +} + +TCPSocket& TCPSocket::operator=(TCPSocket&& other) +{ + m_sock = other.m_sock; + m_remote_address = other.m_remote_address; + + if (other.m_sock != -1) { + other.m_sock = -1; + } + + return *this; +} + +bool TCPSocket::valid() const +{ + return m_sock != -1; +} + +void TCPSocket::connect(const std::string& hostname, int port) +{ + if (m_sock != INVALID_SOCKET) { + throw std::logic_error("You may only connect an invalid TCPSocket"); + } + + char service[NI_MAXSERV]; + snprintf(service, NI_MAXSERV-1, "%d", port); + + /* Obtain address(es) matching host/port */ + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = 0; + hints.ai_protocol = 0; + + struct addrinfo *result, *rp; + int s = getaddrinfo(hostname.c_str(), service, &hints, &result); + if (s != 0) { + throw runtime_error(string("getaddrinfo failed: ") + gai_strerror(s)); + } + + /* getaddrinfo() returns a list of address structures. + Try each address until we successfully connect(2). + If socket(2) (or connect(2)) fails, we (close the socket + and) try the next address. */ + + for (rp = result; rp != nullptr; rp = rp->ai_next) { + int sfd = ::socket(rp->ai_family, rp->ai_socktype, + rp->ai_protocol); + if (sfd == -1) + continue; + + int ret = ::connect(sfd, rp->ai_addr, rp->ai_addrlen); + if (ret != -1 or (ret == -1 and errno == EINPROGRESS)) { + // As the TCPClient could set the socket to nonblocking, we + // must handle EINPROGRESS here + m_sock = sfd; + break; + } + + ::close(sfd); + } + + if (m_sock != INVALID_SOCKET) { +#if defined(HAVE_SO_NOSIGPIPE) + int val = 1; + if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, &val, sizeof(val)) + == SOCKET_ERROR) { + throw std::runtime_error("Can't set SO_NOSIGPIPE"); + } +#endif + } + + freeaddrinfo(result); /* No longer needed */ + + if (rp == nullptr) { + throw runtime_error("Could not connect"); + } + +} + +void TCPSocket::listen(int port, const string& name) +{ + if (m_sock != INVALID_SOCKET) { + throw std::logic_error("You may only listen with an invalid TCPSocket"); + } + + char service[NI_MAXSERV]; + snprintf(service, NI_MAXSERV-1, "%d", port); + + struct addrinfo hints; + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ + hints.ai_protocol = 0; + hints.ai_canonname = nullptr; + hints.ai_addr = nullptr; + hints.ai_next = nullptr; + + struct addrinfo *result, *rp; + int s = getaddrinfo(name.empty() ? nullptr : name.c_str(), service, &hints, &result); + if (s != 0) { + throw runtime_error(string("getaddrinfo failed: ") + gai_strerror(s)); + } + + /* getaddrinfo() returns a list of address structures. + Try each address until we successfully bind(2). + If socket(2) (or bind(2)) fails, we (close the socket + and) try the next address. */ + for (rp = result; rp != nullptr; rp = rp->ai_next) { + int sfd = ::socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (sfd == -1) { + continue; + } + + if (::bind(sfd, rp->ai_addr, rp->ai_addrlen) == 0) { + m_sock = sfd; + break; + } + + ::close(sfd); + } + + freeaddrinfo(result); + + if (rp == nullptr) { + throw runtime_error("Could not bind"); + } +} + +void TCPSocket::close() +{ + ::close(m_sock); + m_sock = -1; +} + +TCPSocket TCPSocket::accept(int timeout_ms) +{ + if (timeout_ms == 0) { + InetAddress remote_addr; + socklen_t client_len = sizeof(remote_addr.addr); + int sockfd = ::accept(m_sock, remote_addr.as_sockaddr(), &client_len); + TCPSocket s(sockfd, remote_addr); + return s; + } + else { + struct pollfd fds[1]; + fds[0].fd = m_sock; + fds[0].events = POLLIN; + + int retval = poll(fds, 1, timeout_ms); + + if (retval == -1) { + std::string errstr(strerror(errno)); + throw std::runtime_error("TCP Socket accept error: " + errstr); + } + else if (retval > 0) { + InetAddress remote_addr; + socklen_t client_len = sizeof(remote_addr.addr); + int sockfd = ::accept(m_sock, remote_addr.as_sockaddr(), &client_len); + TCPSocket s(sockfd, remote_addr); + return s; + } + else { + TCPSocket s(-1); + return s; + } + } +} + +ssize_t TCPSocket::sendall(const void *buffer, size_t buflen) +{ + uint8_t *buf = (uint8_t*)buffer; + while (buflen > 0) { + /* On Linux, the MSG_NOSIGNAL flag ensures that the process + * would not receive a SIGPIPE and die. + * Other systems have SO_NOSIGPIPE set on the socket for the + * same effect. */ +#if defined(HAVE_MSG_NOSIGNAL) + const int flags = MSG_NOSIGNAL; +#else + const int flags = 0; +#endif + ssize_t sent = ::send(m_sock, buf, buflen, flags); + if (sent < 0) { + return -1; + } + else { + buf += sent; + buflen -= sent; + } + } + return buflen; +} + +ssize_t TCPSocket::send(const void* data, size_t size, int timeout_ms) +{ + if (timeout_ms) { + struct pollfd fds[1]; + fds[0].fd = m_sock; + fds[0].events = POLLOUT; + + const int retval = poll(fds, 1, timeout_ms); + + if (retval == -1) { + throw std::runtime_error(string("TCP Socket send error on poll(): ") + strerror(errno)); + } + else if (retval == 0) { + // Timed out + return 0; + } + } + + /* On Linux, the MSG_NOSIGNAL flag ensures that the process would not + * receive a SIGPIPE and die. + * Other systems have SO_NOSIGPIPE set on the socket for the same effect. */ +#if defined(HAVE_MSG_NOSIGNAL) + const int flags = MSG_NOSIGNAL; +#else + const int flags = 0; +#endif + const ssize_t ret = ::send(m_sock, (const char*)data, size, flags); + + if (ret == SOCKET_ERROR) { + throw std::runtime_error(string("TCP Socket send error: ") + strerror(errno)); + } + return ret; +} + +ssize_t TCPSocket::recv(void *buffer, size_t length, int flags) +{ + ssize_t ret = ::recv(m_sock, buffer, length, flags); + if (ret == -1) { + std::string errstr(strerror(errno)); + throw std::runtime_error("TCP receive error: " + errstr); + } + return ret; +} + +ssize_t TCPSocket::recv(void *buffer, size_t length, int flags, int timeout_ms) +{ + struct pollfd fds[1]; + fds[0].fd = m_sock; + fds[0].events = POLLIN; + + int retval = poll(fds, 1, timeout_ms); + + if (retval == -1 and errno == EINTR) { + throw Interrupted(); + } + else if (retval == -1) { + std::string errstr(strerror(errno)); + throw std::runtime_error("TCP receive with poll() error: " + errstr); + } + else if (retval > 0 and (fds[0].revents | POLLIN)) { + ssize_t ret = ::recv(m_sock, buffer, length, flags); + if (ret == -1) { + if (errno == ECONNREFUSED) { + return 0; + } + std::string errstr(strerror(errno)); + throw std::runtime_error("TCP receive after poll() error: " + errstr); + } + return ret; + } + else { + throw Timeout(); + } +} + +TCPSocket::TCPSocket(int sockfd) : + m_sock(sockfd), + m_remote_address() +{ } + +TCPSocket::TCPSocket(int sockfd, InetAddress remote_address) : + m_sock(sockfd), + m_remote_address(remote_address) +{ } + +void TCPClient::connect(const std::string& hostname, int port) +{ + m_hostname = hostname; + m_port = port; + reconnect(); +} + +ssize_t TCPClient::recv(void *buffer, size_t length, int flags, int timeout_ms) +{ + try { + ssize_t ret = m_sock.recv(buffer, length, flags, timeout_ms); + + if (ret == 0) { + m_sock.close(); + + TCPSocket newsock; + m_sock = std::move(newsock); + reconnect(); + } + + return ret; + } + catch (const TCPSocket::Interrupted&) { + return -1; + } + catch (const TCPSocket::Timeout&) { + return 0; + } + + return 0; +} + +void TCPClient::reconnect() +{ + int flags = fcntl(m_sock.m_sock, F_GETFL); + if (fcntl(m_sock.m_sock, F_SETFL, flags | O_NONBLOCK) == -1) { + std::string errstr(strerror(errno)); + throw std::runtime_error("TCP: Could not set O_NONBLOCK: " + errstr); + } + + m_sock.connect(m_hostname, m_port); +} + +TCPConnection::TCPConnection(TCPSocket&& sock) : + queue(), + m_running(true), + m_sender_thread(), + m_sock(move(sock)) +{ +#if MISSING_OWN_ADDR + auto own_addr = m_sock.getOwnAddress(); + auto addr = m_sock.getRemoteAddress(); + etiLog.level(debug) << "New TCP Connection on port " << + own_addr.getPort() << " from " << + addr.getHostAddress() << ":" << addr.getPort(); +#endif + m_sender_thread = std::thread(&TCPConnection::process, this); +} + +TCPConnection::~TCPConnection() +{ + m_running = false; + vector termination_marker; + queue.push(termination_marker); + m_sender_thread.join(); +} + +void TCPConnection::process() +{ + while (m_running) { + vector data; + queue.wait_and_pop(data); + + if (data.empty()) { + // empty vector is the termination marker + m_running = false; + break; + } + + try { + ssize_t remaining = data.size(); + const uint8_t *buf = reinterpret_cast(data.data()); + const int timeout_ms = 10; // Less than one ETI frame + + while (m_running and remaining > 0) { + const ssize_t sent = m_sock.send(buf, remaining, timeout_ms); + if (sent < 0 or sent > remaining) { + throw std::logic_error("Invalid TCPSocket::send() return value"); + } + remaining -= sent; + buf += sent; + } + } + catch (const std::runtime_error& e) { + m_running = false; + } + } + +#if MISSING_OWN_ADDR + auto own_addr = m_sock.getOwnAddress(); + auto addr = m_sock.getRemoteAddress(); + etiLog.level(debug) << "Dropping TCP Connection on port " << + own_addr.getPort() << " from " << + addr.getHostAddress() << ":" << addr.getPort(); +#endif +} + + +TCPDataDispatcher::TCPDataDispatcher(size_t max_queue_size) : + m_max_queue_size(max_queue_size) +{ +} + +TCPDataDispatcher::~TCPDataDispatcher() +{ + m_running = false; + m_connections.clear(); + m_listener_socket.close(); + m_listener_thread.join(); +} + +void TCPDataDispatcher::start(int port, const string& address) +{ + m_listener_socket.listen(port, address); + + m_running = true; + m_listener_thread = std::thread(&TCPDataDispatcher::process, this); +} + +void TCPDataDispatcher::write(const vector& data) +{ + if (not m_running) { + throw runtime_error(m_exception_data); + } + + for (auto& connection : m_connections) { + connection.queue.push(data); + } + + m_connections.remove_if( + [&](const TCPConnection& conn){ return conn.queue.size() > m_max_queue_size; }); +} + +void TCPDataDispatcher::process() +{ + try { + const int timeout_ms = 1000; + + while (m_running) { + // Add a new TCPConnection to the list, constructing it from the client socket + auto sock = m_listener_socket.accept(timeout_ms); + if (sock.valid()) { + m_connections.emplace(m_connections.begin(), move(sock)); + } + } + } + catch (const std::runtime_error& e) { + m_exception_data = string("TCPDataDispatcher error: ") + e.what(); + m_running = false; + } +} + +} diff --git a/lib/Socket.h b/lib/Socket.h new file mode 100644 index 0000000..82ff5ad --- /dev/null +++ b/lib/Socket.h @@ -0,0 +1,240 @@ +/* + Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the + Queen in Right of Canada (Communications Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#pragma once + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ThreadsafeQueue.h" +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#define SOCKET int +#define INVALID_SOCKET -1 +#define SOCKET_ERROR -1 + + +namespace Socket { + +struct InetAddress { + struct sockaddr_storage addr; + + struct sockaddr *as_sockaddr() { return reinterpret_cast(&addr); }; + + void resolveUdpDestination(const std::string& destination, int port); +}; + +/** This class represents a UDP packet. + * + * A UDP packet contains a payload (sequence of bytes) and an address. For + * outgoing packets, the address is the destination address. For incoming + * packets, the address tells the user from what source the packet arrived from. + */ +class UDPPacket +{ + public: + UDPPacket(); + UDPPacket(size_t initSize); + + std::vector buffer; + InetAddress address; +}; + +/** + * This class represents a socket for sending and receiving UDP packets. + * + * A UDP socket is the sending or receiving point for a packet delivery service. + * Each packet sent or received on a datagram socket is individually + * addressed and routed. Multiple packets sent from one machine to another may + * be routed differently, and may arrive in any order. + */ +class UDPSocket +{ + public: + /** Create a new socket that will not be bound to any port. To be used + * for data output. + */ + UDPSocket(); + /** Create a new socket. + * @param port The port number on which the socket will be bound + */ + UDPSocket(int port); + /** Create a new socket. + * @param port The port number on which the socket will be bound + * @param name The IP address on which the socket will be bound. + * It is used to bind the socket on a specific interface if + * the computer have many NICs. + */ + UDPSocket(int port, const std::string& name); + ~UDPSocket(); + UDPSocket(const UDPSocket& other) = delete; + const UDPSocket& operator=(const UDPSocket& other) = delete; + + /** Close the already open socket, and create a new one. Throws a runtime_error on error. */ + void reinit(int port, const std::string& name); + + void close(void); + void send(UDPPacket& packet); + void send(const std::vector& data, InetAddress destination); + UDPPacket receive(size_t max_size); + void joinGroup(char* groupname); + void setMulticastSource(const char* source_addr); + void setMulticastTTL(int ttl); + + /** Set blocking mode. By default, the socket is blocking. + * throws a runtime_error on error. + */ + void setBlocking(bool block); + + protected: + SOCKET listenSocket; +}; + + +class TCPSocket { + public: + TCPSocket(); + ~TCPSocket(); + TCPSocket(const TCPSocket& other) = delete; + TCPSocket& operator=(const TCPSocket& other) = delete; + TCPSocket(TCPSocket&& other); + TCPSocket& operator=(TCPSocket&& other); + + bool valid(void) const; + void connect(const std::string& hostname, int port); + void listen(int port, const std::string& name); + void close(void); + + /* throws a runtime_error on failure, an invalid socket on timeout */ + TCPSocket accept(int timeout_ms); + + /* returns -1 on error, doesn't work on nonblocking sockets */ + ssize_t sendall(const void *buffer, size_t buflen); + + /** Send data over the TCP connection. + * @param data The buffer that will be sent. + * @param size Number of bytes to send. + * @param timeout_ms number of milliseconds before timeout, or 0 for infinite timeout + * return number of bytes sent, 0 on timeout, or throws runtime_error. + */ + ssize_t send(const void* data, size_t size, int timeout_ms=0); + + /* Returns number of bytes read, 0 on disconnect. Throws a + * runtime_error on error */ + ssize_t recv(void *buffer, size_t length, int flags); + + class Timeout {}; + class Interrupted {}; + /* Returns number of bytes read, 0 on disconnect or refused connection. + * Throws a Timeout on timeout, Interrupted on EINTR, a runtime_error + * on error + */ + ssize_t recv(void *buffer, size_t length, int flags, int timeout_ms); + + private: + explicit TCPSocket(int sockfd); + explicit TCPSocket(int sockfd, InetAddress remote_address); + SOCKET m_sock = -1; + + InetAddress m_remote_address; + + friend class TCPClient; +}; + +/* Implements a TCP receiver that auto-reconnects on errors */ +class TCPClient { + public: + void connect(const std::string& hostname, int port); + + /* Returns numer of bytes read, 0 on auto-reconnect, -1 + * on interruption. + * Throws a runtime_error on error */ + ssize_t recv(void *buffer, size_t length, int flags, int timeout_ms); + + private: + void reconnect(void); + TCPSocket m_sock; + std::string m_hostname; + int m_port; +}; + +/* Helper class for TCPDataDispatcher, contains a queue of pending data and + * a sender thread. */ +class TCPConnection +{ + public: + TCPConnection(TCPSocket&& sock); + TCPConnection(const TCPConnection&) = delete; + TCPConnection& operator=(const TCPConnection&) = delete; + ~TCPConnection(); + + ThreadsafeQueue > queue; + + private: + std::atomic m_running; + std::thread m_sender_thread; + TCPSocket m_sock; + + void process(void); +}; + +/* Send a TCP stream to several destinations, and automatically disconnect destinations + * whose buffer overflows. + */ +class TCPDataDispatcher +{ + public: + TCPDataDispatcher(size_t max_queue_size); + ~TCPDataDispatcher(); + TCPDataDispatcher(const TCPDataDispatcher&) = delete; + TCPDataDispatcher& operator=(const TCPDataDispatcher&) = delete; + + void start(int port, const std::string& address); + void write(const std::vector& data); + + private: + void process(void); + + size_t m_max_queue_size; + + std::atomic m_running; + std::string m_exception_data; + std::thread m_listener_thread; + TCPSocket m_listener_socket; + std::list m_connections; +}; + +} -- cgit v1.2.3 From 8b0d4b647a3ac2e21e6a8e7a902eea3ae462e213 Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Fri, 7 Jun 2019 10:14:51 +0200 Subject: Work on STI-D/EDI input --- Makefile.am | 21 +++-- doc/advanced.mux | 17 ++-- doc/example.mux | 3 +- lib/ReedSolomon.cpp | 116 ++++++++++++++++++++++++++ lib/ReedSolomon.h | 56 +++++++++++++ lib/Socket.cpp | 165 ++++++++++++++++++++++++++++--------- lib/Socket.h | 32 +++++++- lib/ThreadsafeQueue.h | 176 +++++++++++++++++++++++++++++++++++++++ src/ConfigParser.cpp | 61 ++++++++------ src/DabMux.cpp | 2 +- src/ReedSolomon.cpp | 116 -------------------------- src/ReedSolomon.h | 56 ------------- src/ThreadsafeQueue.h | 178 ---------------------------------------- src/dabOutput/edi/Config.h | 9 +- src/dabOutput/edi/PFT.cpp | 2 +- src/dabOutput/edi/Transport.cpp | 43 ++++++---- src/dabOutput/edi/Transport.h | 4 +- src/input/Udp.cpp | 8 +- 18 files changed, 614 insertions(+), 451 deletions(-) create mode 100644 lib/ReedSolomon.cpp create mode 100644 lib/ReedSolomon.h create mode 100644 lib/ThreadsafeQueue.h delete mode 100644 src/ReedSolomon.cpp delete mode 100644 src/ReedSolomon.h delete mode 100644 src/ThreadsafeQueue.h (limited to 'lib') diff --git a/Makefile.am b/Makefile.am index 2773bbe..80d24e0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -69,6 +69,8 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/input/File.h \ src/input/Udp.cpp \ src/input/Udp.h \ + src/input/Edi.cpp \ + src/input/Edi.h \ src/dabOutput/dabOutput.h \ src/dabOutput/dabOutputFile.cpp \ src/dabOutput/dabOutputFifo.cpp \ @@ -107,11 +109,8 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/MuxElements.cpp \ src/MuxElements.h \ src/PcDebug.h \ - src/ReedSolomon.h \ - src/ReedSolomon.cpp \ src/RemoteControl.cpp \ src/RemoteControl.h \ - src/ThreadsafeQueue.h \ src/crc.h \ src/crc.c \ src/fig/FIG.h \ @@ -161,8 +160,19 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/PrbsGenerator.h \ src/utils.cpp \ src/utils.h \ + lib/edi/STIDecoder.cpp \ + lib/edi/STIDecoder.h \ + lib/edi/STIWriter.cpp \ + lib/edi/STIWriter.h \ + lib/edi/PFT.cpp \ + lib/edi/PFT.h \ + lib/edi/common.cpp \ + lib/edi/common.h \ + lib/ReedSolomon.h \ + lib/ReedSolomon.cpp \ lib/Socket.h \ lib/Socket.cpp \ + lib/ThreadsafeQueue.h \ lib/zmq.hpp \ $(lib_fec_sources) \ $(lib_charset_sources) @@ -201,14 +211,15 @@ odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ src/dabOutput/edi/TagPacket.h \ src/dabOutput/edi/Transport.cpp \ src/dabOutput/edi/Transport.h \ - src/ReedSolomon.h \ - src/ReedSolomon.cpp \ src/Log.h \ src/Log.cpp \ src/crc.h \ src/crc.c \ + lib/ReedSolomon.h \ + lib/ReedSolomon.cpp \ lib/Socket.h \ lib/Socket.cpp \ + lib/ThreadsafeQueue.h \ lib/zmq.hpp \ $(lib_fec_sources) diff --git a/doc/advanced.mux b/doc/advanced.mux index fb67b82..b9cec05 100644 --- a/doc/advanced.mux +++ b/doc/advanced.mux @@ -163,7 +163,8 @@ subchannels { sub-fu { type audio ; example file input - inputfile "funk.mp2" + inputproto zmq + inputuri "funk.mp2" nonblock false bitrate 128 id 10 @@ -188,7 +189,8 @@ subchannels { ; Receive STI-D(LI) carried in STI(PI, X) inside RTP using UDP. ; This is intended to be compatible with AVT audio encoders. ; EXPERIMENTAL! - inputfile "sti-rtp://127.0.0.1:32010" + inputproto sti + inputuri "rtp://127.0.0.1:32010" bitrate 96 id 3 protection 3 @@ -196,11 +198,12 @@ subchannels { sub-ri { type dabplus ; example file input - ;inputfile "rick.dabp" + ;inputuri "rick.dabp" ; example zmq input: ; Accepts connections to port 9000 from any interface. ; Use ODR-AudioEnc as encoder - inputfile "tcp://*:9000" + inputproto zmq + inputuri "tcp://*:9000" bitrate 96 id 1 protection 1 @@ -256,7 +259,8 @@ subchannels { ; for audio types, you can use the ZeroMQ input (if compiled in) ; with the following configuration in combination with ; Toolame-DAB - inputfile "tcp://*:9001" + inputproto zmq + inputuri "tcp://*:9001" bitrate 96 id 1 protection 1 @@ -273,7 +277,8 @@ subchannels { type data ; Use the default PRBS polynomial. - inputfile "prbs://" + inputproto prbs + inputuri "prbs://" ; To use another polynomial, set it in the url as hexadecimal ; The default polynomial is G(x) = x^20 + x^17 + 1, represented as diff --git a/doc/example.mux b/doc/example.mux index 6c2bc18..31e072d 100644 --- a/doc/example.mux +++ b/doc/example.mux @@ -171,7 +171,8 @@ subchannels { type dabplus ; Accepts connections to port 9000 from any interface. ; Use ODR-AudioEnc as encoder - inputfile "tcp://*:9000" + inputproto zmq + inputuri "tcp://*:9000" bitrate 96 id 1 protection 3 diff --git a/lib/ReedSolomon.cpp b/lib/ReedSolomon.cpp new file mode 100644 index 0000000..38d8ea8 --- /dev/null +++ b/lib/ReedSolomon.cpp @@ -0,0 +1,116 @@ +/* + Copyright (C) 2005, 2006, 2007, 2008, 2009 Her Majesty the Queen in Right + of Canada (Communications Research Center Canada) + + Copyright (C) 2016 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#include "ReedSolomon.h" +#include +#include +#include +#include +#include // For galois.h ... +#include // For memcpy + +extern "C" { +#include "fec/fec.h" +} +#include + +#define SYMSIZE 8 + + +ReedSolomon::ReedSolomon(int N, int K, bool reverse, int gfpoly, int firstRoot, int primElem) +{ + setReverse(reverse); + + m_N = N; + m_K = K; + + const int symsize = SYMSIZE; + const int nroots = N - K; // For EDI PFT, this must be 48 + const int pad = ((1 << symsize) - 1) - N; // is 255-N + + rsData = init_rs_char(symsize, gfpoly, firstRoot, primElem, nroots, pad); + + if (rsData == nullptr) { + std::stringstream ss; + ss << "Invalid Reed-Solomon parameters! " << + "N=" << N << " ; K=" << K << " ; pad=" << pad; + throw std::invalid_argument(ss.str()); + } +} + + +ReedSolomon::~ReedSolomon() +{ + free_rs_char(rsData); +} + + +void ReedSolomon::setReverse(bool state) +{ + reverse = state; +} + + +int ReedSolomon::encode(void* data, void* fec, size_t size) +{ + uint8_t* input = reinterpret_cast(data); + uint8_t* output = reinterpret_cast(fec); + int ret = 0; + + if (reverse) { + std::vector buffer(m_N); + + memcpy(&buffer[0], input, m_K); + memcpy(&buffer[m_K], output, m_N - m_K); + + ret = decode_rs_char(rsData, &buffer[0], nullptr, 0); + if ((ret != 0) && (ret != -1)) { + memcpy(input, &buffer[0], m_K); + memcpy(output, &buffer[m_K], m_N - m_K); + } + } + else { + encode_rs_char(rsData, input, output); + } + + return ret; +} + + +int ReedSolomon::encode(void* data, size_t size) +{ + uint8_t* input = reinterpret_cast(data); + int ret = 0; + + if (reverse) { + ret = decode_rs_char(rsData, input, nullptr, 0); + } + else { + encode_rs_char(rsData, input, &input[m_K]); + } + + return ret; +} diff --git a/lib/ReedSolomon.h b/lib/ReedSolomon.h new file mode 100644 index 0000000..abcef62 --- /dev/null +++ b/lib/ReedSolomon.h @@ -0,0 +1,56 @@ +/* + Copyright (C) 2005, 2006, 2007, 2008, 2009 Her Majesty the Queen in Right + of Canada (Communications Research Center Canada) + + Copyright (C) 2016 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#pragma once + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include + +class ReedSolomon +{ +public: + ReedSolomon(int N, int K, + bool reverse = false, + int gfpoly = 0x11d, int firstRoot = 0, int primElem = 1); + ReedSolomon(const ReedSolomon& other) = delete; + ReedSolomon operator=(const ReedSolomon& other) = delete; + ~ReedSolomon(); + + void setReverse(bool state); + int encode(void* data, void* fec, size_t size); + int encode(void* data, size_t size); + +private: + int m_N; + int m_K; + + void* rsData; + bool reverse; +}; + diff --git a/lib/Socket.cpp b/lib/Socket.cpp index fe3df44..9b404eb 100644 --- a/lib/Socket.cpp +++ b/lib/Socket.cpp @@ -42,13 +42,10 @@ void InetAddress::resolveUdpDestination(const std::string& destination, int port struct addrinfo hints; memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ hints.ai_flags = 0; hints.ai_protocol = 0; - hints.ai_canonname = nullptr; - hints.ai_addr = nullptr; - hints.ai_next = nullptr; struct addrinfo *result, *rp; int s = getaddrinfo(destination.c_str(), service, &hints, &result); @@ -77,19 +74,19 @@ UDPPacket::UDPPacket(size_t initSize) : UDPSocket::UDPSocket() : - listenSocket(INVALID_SOCKET) + m_sock(INVALID_SOCKET) { reinit(0, ""); } UDPSocket::UDPSocket(int port) : - listenSocket(INVALID_SOCKET) + m_sock(INVALID_SOCKET) { reinit(port, ""); } UDPSocket::UDPSocket(int port, const std::string& name) : - listenSocket(INVALID_SOCKET) + m_sock(INVALID_SOCKET) { reinit(port, name); } @@ -97,16 +94,28 @@ UDPSocket::UDPSocket(int port, const std::string& name) : void UDPSocket::setBlocking(bool block) { - int res = fcntl(listenSocket, F_SETFL, block ? 0 : O_NONBLOCK); + int res = fcntl(m_sock, F_SETFL, block ? 0 : O_NONBLOCK); if (res == -1) { throw runtime_error(string("Can't change blocking state of socket: ") + strerror(errno)); } } +void UDPSocket::reinit(int port) +{ + return reinit(port, ""); +} + void UDPSocket::reinit(int port, const std::string& name) { - if (listenSocket != INVALID_SOCKET) { - ::close(listenSocket); + if (m_sock != INVALID_SOCKET) { + ::close(m_sock); + } + + if (port == 0) { + // No need to bind to a given port, creating the + // socket is enough + m_sock = ::socket(AF_INET, SOCK_DGRAM, 0); + return; } char service[NI_MAXSERV]; @@ -114,7 +123,7 @@ void UDPSocket::reinit(int port, const std::string& name) struct addrinfo hints; memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */ hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ hints.ai_protocol = 0; /* Any protocol */ @@ -141,7 +150,7 @@ void UDPSocket::reinit(int port, const std::string& name) } if (::bind(sfd, rp->ai_addr, rp->ai_addrlen) == 0) { - listenSocket = sfd; + m_sock = sfd; break; } @@ -157,17 +166,17 @@ void UDPSocket::reinit(int port, const std::string& name) void UDPSocket::close() { - if (listenSocket != INVALID_SOCKET) { - ::close(listenSocket); + if (m_sock != INVALID_SOCKET) { + ::close(m_sock); } - listenSocket = INVALID_SOCKET; + m_sock = INVALID_SOCKET; } UDPSocket::~UDPSocket() { - if (listenSocket != INVALID_SOCKET) { - ::close(listenSocket); + if (m_sock != INVALID_SOCKET) { + ::close(m_sock); } } @@ -177,7 +186,7 @@ UDPPacket UDPSocket::receive(size_t max_size) UDPPacket packet(max_size); socklen_t addrSize; addrSize = sizeof(*packet.address.as_sockaddr()); - ssize_t ret = recvfrom(listenSocket, + ssize_t ret = recvfrom(m_sock, packet.buffer.data(), packet.buffer.size(), 0, @@ -186,7 +195,13 @@ UDPPacket UDPSocket::receive(size_t max_size) if (ret == SOCKET_ERROR) { packet.buffer.resize(0); + + // This suppresses the -Wlogical-op warning +#if EAGAIN == EWOULDBLOCK if (errno == EAGAIN) { +#else + if (errno == EAGAIN or errno == EWOULDBLOCK) { +#endif return 0; } throw runtime_error(string("Can't receive data: ") + strerror(errno)); @@ -198,24 +213,24 @@ UDPPacket UDPSocket::receive(size_t max_size) void UDPSocket::send(UDPPacket& packet) { - int ret = sendto(listenSocket, packet.buffer.data(), packet.buffer.size(), 0, + const int ret = sendto(m_sock, packet.buffer.data(), packet.buffer.size(), 0, packet.address.as_sockaddr(), sizeof(*packet.address.as_sockaddr())); if (ret == SOCKET_ERROR && errno != ECONNREFUSED) { - throw runtime_error(string("Can't send UDP packet") + strerror(errno)); + throw runtime_error(string("Can't send UDP packet: ") + strerror(errno)); } } void UDPSocket::send(const std::vector& data, InetAddress destination) { - int ret = sendto(listenSocket, &data[0], data.size(), 0, + const int ret = sendto(m_sock, data.data(), data.size(), 0, destination.as_sockaddr(), sizeof(*destination.as_sockaddr())); if (ret == SOCKET_ERROR && errno != ECONNREFUSED) { - throw runtime_error(string("Can't send UDP packet") + strerror(errno)); + throw runtime_error(string("Can't send UDP packet: ") + strerror(errno)); } } -void UDPSocket::joinGroup(char* groupname) +void UDPSocket::joinGroup(const char* groupname, const char* if_addr) { ip_mreqn group; if ((group.imr_multiaddr.s_addr = inet_addr(groupname)) == INADDR_NONE) { @@ -224,9 +239,15 @@ void UDPSocket::joinGroup(char* groupname) if (!IN_MULTICAST(ntohl(group.imr_multiaddr.s_addr))) { throw runtime_error("Group name is not a multicast address"); } - group.imr_address.s_addr = htons(INADDR_ANY);; + + if (if_addr) { + group.imr_address.s_addr = inet_addr(if_addr); + } + else { + group.imr_address.s_addr = htons(INADDR_ANY); + } group.imr_ifindex = 0; - if (setsockopt(listenSocket, IPPROTO_IP, IP_ADD_MEMBERSHIP, &group, sizeof(group)) + if (setsockopt(m_sock, IPPROTO_IP, IP_ADD_MEMBERSHIP, &group, sizeof(group)) == SOCKET_ERROR) { throw runtime_error(string("Can't join multicast group") + strerror(errno)); } @@ -239,7 +260,7 @@ void UDPSocket::setMulticastSource(const char* source_addr) throw runtime_error(string("Can't parse source address") + strerror(errno)); } - if (setsockopt(listenSocket, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) + if (setsockopt(m_sock, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == SOCKET_ERROR) { throw runtime_error(string("Can't set source address") + strerror(errno)); } @@ -247,26 +268,82 @@ void UDPSocket::setMulticastSource(const char* source_addr) void UDPSocket::setMulticastTTL(int ttl) { - if (setsockopt(listenSocket, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) + if (setsockopt(m_sock, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) == SOCKET_ERROR) { throw runtime_error(string("Can't set multicast ttl") + strerror(errno)); } } +UDPReceiver::~UDPReceiver() { + m_stop = true; + m_sock.close(); + if (m_thread.joinable()) { + m_thread.join(); + } +} + +void UDPReceiver::start(int port, const string& bindto, const string& mcastaddr, size_t max_packets_queued) { + m_port = port; + m_bindto = bindto; + m_mcastaddr = mcastaddr; + m_max_packets_queued = max_packets_queued; + m_thread = std::thread(&UDPReceiver::m_run, this); +} -TCPSocket::TCPSocket() +std::vector UDPReceiver::get_packet_buffer() { - if ((m_sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) { - throw std::runtime_error("Can't create TCP socket"); + if (m_stop) { + throw runtime_error("UDP Receiver not running"); } -#if defined(HAVE_SO_NOSIGPIPE) - int val = 1; - if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, - &val, sizeof(val)) < 0) { - throw std::runtime_error("Can't set SO_NOSIGPIPE"); + UDPPacket p; + m_packets.wait_and_pop(p); + + return p.buffer; +} + +void UDPReceiver::m_run() +{ + // Ensure that stop is set to true in case of exception or return + struct SetStopOnDestruct { + SetStopOnDestruct(atomic& stop) : m_stop(stop) {} + ~SetStopOnDestruct() { m_stop = true; } + private: atomic& m_stop; + } autoSetStop(m_stop); + + if (IN_MULTICAST(ntohl(inet_addr(m_mcastaddr.c_str())))) { + m_sock.reinit(m_port, m_mcastaddr); + m_sock.setMulticastSource(m_bindto.c_str()); + m_sock.joinGroup(m_mcastaddr.c_str(), m_bindto.c_str()); } -#endif + else { + m_sock.reinit(m_port, m_bindto); + } + + while (not m_stop) { + constexpr size_t packsize = 8192; + try { + auto packet = m_sock.receive(packsize); + if (packet.buffer.size() == packsize) { + // TODO replace fprintf + fprintf(stderr, "Warning, possible UDP truncation\n"); + } + + // If this blocks, the UDP socket will lose incoming packets + m_packets.push_wait_if_full(packet, m_max_packets_queued); + } + catch (const std::runtime_error& e) { + // TODO replace fprintf + // TODO handle intr + fprintf(stderr, "Socket error: %s\n", e.what()); + m_stop = true; + } + } +} + + +TCPSocket::TCPSocket() +{ } TCPSocket::~TCPSocket() @@ -314,7 +391,7 @@ void TCPSocket::connect(const std::string& hostname, int port) /* Obtain address(es) matching host/port */ struct addrinfo hints; memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = 0; hints.ai_protocol = 0; @@ -376,7 +453,7 @@ void TCPSocket::listen(int port, const string& name) struct addrinfo hints; memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = AF_UNSPEC; + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; /* For wildcard IP address */ hints.ai_protocol = 0; @@ -410,6 +487,14 @@ void TCPSocket::listen(int port, const string& name) freeaddrinfo(result); +#if defined(HAVE_SO_NOSIGPIPE) + int val = 1; + if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, + &val, sizeof(val)) < 0) { + throw std::runtime_error("Can't set SO_NOSIGPIPE"); + } +#endif + if (rp == nullptr) { throw runtime_error("Could not bind"); } @@ -683,7 +768,9 @@ TCPDataDispatcher::~TCPDataDispatcher() m_running = false; m_connections.clear(); m_listener_socket.close(); - m_listener_thread.join(); + if (m_listener_thread.joinable()) { + m_listener_thread.join(); + } } void TCPDataDispatcher::start(int port, const string& address) diff --git a/lib/Socket.h b/lib/Socket.h index 82ff5ad..2393584 100644 --- a/lib/Socket.h +++ b/lib/Socket.h @@ -104,13 +104,14 @@ class UDPSocket const UDPSocket& operator=(const UDPSocket& other) = delete; /** Close the already open socket, and create a new one. Throws a runtime_error on error. */ + void reinit(int port); void reinit(int port, const std::string& name); void close(void); void send(UDPPacket& packet); void send(const std::vector& data, InetAddress destination); UDPPacket receive(size_t max_size); - void joinGroup(char* groupname); + void joinGroup(const char* groupname, const char* if_addr = nullptr); void setMulticastSource(const char* source_addr); void setMulticastTTL(int ttl); @@ -120,9 +121,36 @@ class UDPSocket void setBlocking(bool block); protected: - SOCKET listenSocket; + SOCKET m_sock; }; +/* Threaded UDP receiver */ +class UDPReceiver { + public: + UDPReceiver() : m_port(0), m_thread(), m_stop(false), m_packets() {} + ~UDPReceiver(); + UDPReceiver(const UDPReceiver&) = delete; + UDPReceiver operator=(const UDPReceiver&) = delete; + + // Start the receiver in a separate thread + void start(int port, const std::string& bindto, const std::string& mcastaddr, size_t max_packets_queued); + + // Get the data contained in a UDP packet, blocks if none available + // In case of error, throws a runtime_error + std::vector get_packet_buffer(void); + + private: + void m_run(void); + + int m_port; + std::string m_bindto; + std::string m_mcastaddr; + size_t m_max_packets_queued; + std::thread m_thread; + std::atomic m_stop; + ThreadsafeQueue m_packets; + UDPSocket m_sock; +}; class TCPSocket { public: diff --git a/lib/ThreadsafeQueue.h b/lib/ThreadsafeQueue.h new file mode 100644 index 0000000..62f4c96 --- /dev/null +++ b/lib/ThreadsafeQueue.h @@ -0,0 +1,176 @@ +/* + Copyright (C) 2007, 2008, 2009, 2010, 2011 Her Majesty the Queen in + Right of Canada (Communications Research Center Canada) + + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + + An implementation for a threadsafe queue, depends on C++11 + + When creating a ThreadsafeQueue, one can specify the minimal number + of elements it must contain before it is possible to take one + element out. + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +/* This queue is meant to be used by two threads. One producer + * that pushes elements into the queue, and one consumer that + * retrieves the elements. + * + * The queue can make the consumer block until an element + * is available, or a wakeup requested. + */ + +/* Class thrown by blocking pop to tell the consumer + * that there's a wakeup requested. */ +class ThreadsafeQueueWakeup {}; + +template +class ThreadsafeQueue +{ +public: + /* Push one element into the queue, and notify another thread that + * might be waiting. + * + * returns the new queue size. + */ + size_t push(T const& val) + { + std::unique_lock lock(the_mutex); + the_queue.push(val); + size_t queue_size = the_queue.size(); + lock.unlock(); + + the_rx_notification.notify_one(); + + return queue_size; + } + + size_t push(T&& val) + { + std::unique_lock lock(the_mutex); + the_queue.emplace(std::move(val)); + size_t queue_size = the_queue.size(); + lock.unlock(); + + the_rx_notification.notify_one(); + + return queue_size; + } + + /* Push one element into the queue, but wait until the + * queue size goes below the threshold. + * + * Notify waiting thread. + * + * returns the new queue size. + */ + size_t push_wait_if_full(T const& val, size_t threshold) + { + std::unique_lock lock(the_mutex); + while (the_queue.size() >= threshold) { + the_tx_notification.wait(lock); + } + the_queue.push(val); + size_t queue_size = the_queue.size(); + lock.unlock(); + + the_rx_notification.notify_one(); + + return queue_size; + } + + /* Trigger a wakeup event on a blocking consumer, which + * will receive a ThreadsafeQueueWakeup exception. + */ + void trigger_wakeup(void) + { + std::unique_lock lock(the_mutex); + wakeup_requested = true; + lock.unlock(); + the_rx_notification.notify_one(); + } + + /* Send a notification for the receiver thread */ + void notify(void) + { + the_rx_notification.notify_one(); + } + + bool empty() const + { + std::unique_lock lock(the_mutex); + return the_queue.empty(); + } + + size_t size() const + { + std::unique_lock lock(the_mutex); + return the_queue.size(); + } + + bool try_pop(T& popped_value) + { + std::unique_lock lock(the_mutex); + if (the_queue.empty()) { + return false; + } + + popped_value = the_queue.front(); + the_queue.pop(); + + lock.unlock(); + the_tx_notification.notify_one(); + + return true; + } + + void wait_and_pop(T& popped_value, size_t prebuffering = 1) + { + std::unique_lock lock(the_mutex); + while (the_queue.size() < prebuffering and + not wakeup_requested) { + the_rx_notification.wait(lock); + } + + if (wakeup_requested) { + wakeup_requested = false; + throw ThreadsafeQueueWakeup(); + } + else { + std::swap(popped_value, the_queue.front()); + the_queue.pop(); + + lock.unlock(); + the_tx_notification.notify_one(); + } + } + +private: + std::queue the_queue; + mutable std::mutex the_mutex; + std::condition_variable the_rx_notification; + std::condition_variable the_tx_notification; + bool wakeup_requested = false; +}; + diff --git a/src/ConfigParser.cpp b/src/ConfigParser.cpp index fb49efc..3142bb3 100644 --- a/src/ConfigParser.cpp +++ b/src/ConfigParser.cpp @@ -40,6 +40,7 @@ #include "utils.h" #include "DabMux.h" #include "ManagementServer.h" +#include "input/Edi.h" #include "input/Prbs.h" #include "input/Zmq.h" #include "input/File.h" @@ -876,34 +877,46 @@ static void setup_subchannel_from_ptree(shared_ptr& subchan, type = pt.get("type"); } catch (const ptree_error &e) { - stringstream ss; - ss << "Subchannel with uid " << subchanuid << " has no type defined!"; - throw runtime_error(ss.str()); + throw runtime_error("Subchannel with uid " + subchanuid + " has no type defined!"); } - /* Both inputfile and inputuri are supported, and are equivalent. - * inputuri has precedence + /* Up to v2.3.1, both inputfile and inputuri are supported, and are + * equivalent. inputuri has precedence. + * + * After that, either inputfile or the (inputproto, inputuri) pair must be given, but not both. */ string inputUri = pt.get("inputuri", ""); + string proto = pt.get("inputproto", ""); - if (inputUri == "") { + if (inputUri.empty() and proto.empty()) { try { + /* Old approach, derives proto from scheme used in the URL. + * This makes it impossible to distinguish between ZMQ tcp:// and + * EDI tcp:// + */ inputUri = pt.get("inputfile"); + size_t protopos = inputUri.find("://"); + + if (protopos == string::npos) { + proto = "file"; + } + else { + proto = inputUri.substr(0, protopos); + + if (proto == "tcp" or proto == "epgm" or proto == "ipc") { + proto = "zmq"; + } + else if (proto == "sti-rtp") { + proto = "sti"; + } + } } catch (const ptree_error &e) { - stringstream ss; - ss << "Subchannel with uid " << subchanuid << " has no inputUri defined!"; - throw runtime_error(ss.str()); + throw runtime_error("Subchannel with uid " + subchanuid + " has no input defined!"); } } - - string proto; - size_t protopos = inputUri.find("://"); - if (protopos == string::npos) { - proto = "file"; - } - else { - proto = inputUri.substr(0, protopos); + else if (inputUri.empty() or proto.empty()) { + throw runtime_error("Must define both inputuri and inputproto for uid " + subchanuid); } subchan->inputUri = inputUri; @@ -928,7 +941,7 @@ static void setup_subchannel_from_ptree(shared_ptr& subchan, throw logic_error("Incomplete handling of file input"); } } - else if (proto == "tcp" or proto == "epgm" or proto == "ipc") { + else if (proto == "zmq") { auto zmqconfig = setup_zmq_input(pt, subchanuid); if (type == "audio") { @@ -941,15 +954,11 @@ static void setup_subchannel_from_ptree(shared_ptr& subchan, rcs.enrol(inzmq.get()); subchan->input = inzmq; } - - if (proto == "epgm") { - etiLog.level(warn) << "Using untested epgm:// zeromq input"; - } - else if (proto == "ipc") { - etiLog.level(warn) << "Using untested ipc:// zeromq input"; - } } - else if (proto == "sti-rtp") { + else if (proto == "edi") { + subchan->input = make_shared(); + } + else if (proto == "stp") { subchan->input = make_shared(); } else { diff --git a/src/DabMux.cpp b/src/DabMux.cpp index d749ed3..e726fd3 100644 --- a/src/DabMux.cpp +++ b/src/DabMux.cpp @@ -304,7 +304,7 @@ int main(int argc, char *argv[]) edi_conf.destinations.push_back(dest); } else if (proto == "tcp") { - auto dest = make_shared(); + auto dest = make_shared(); dest->listen_port = pt_edi_dest.second.get("listenport"); dest->max_frames_queued = pt_edi_dest.second.get("max_frames_queued", 500); edi_conf.destinations.push_back(dest); diff --git a/src/ReedSolomon.cpp b/src/ReedSolomon.cpp deleted file mode 100644 index 38d8ea8..0000000 --- a/src/ReedSolomon.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - Copyright (C) 2005, 2006, 2007, 2008, 2009 Her Majesty the Queen in Right - of Canada (Communications Research Center Canada) - - Copyright (C) 2016 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "ReedSolomon.h" -#include -#include -#include -#include -#include // For galois.h ... -#include // For memcpy - -extern "C" { -#include "fec/fec.h" -} -#include - -#define SYMSIZE 8 - - -ReedSolomon::ReedSolomon(int N, int K, bool reverse, int gfpoly, int firstRoot, int primElem) -{ - setReverse(reverse); - - m_N = N; - m_K = K; - - const int symsize = SYMSIZE; - const int nroots = N - K; // For EDI PFT, this must be 48 - const int pad = ((1 << symsize) - 1) - N; // is 255-N - - rsData = init_rs_char(symsize, gfpoly, firstRoot, primElem, nroots, pad); - - if (rsData == nullptr) { - std::stringstream ss; - ss << "Invalid Reed-Solomon parameters! " << - "N=" << N << " ; K=" << K << " ; pad=" << pad; - throw std::invalid_argument(ss.str()); - } -} - - -ReedSolomon::~ReedSolomon() -{ - free_rs_char(rsData); -} - - -void ReedSolomon::setReverse(bool state) -{ - reverse = state; -} - - -int ReedSolomon::encode(void* data, void* fec, size_t size) -{ - uint8_t* input = reinterpret_cast(data); - uint8_t* output = reinterpret_cast(fec); - int ret = 0; - - if (reverse) { - std::vector buffer(m_N); - - memcpy(&buffer[0], input, m_K); - memcpy(&buffer[m_K], output, m_N - m_K); - - ret = decode_rs_char(rsData, &buffer[0], nullptr, 0); - if ((ret != 0) && (ret != -1)) { - memcpy(input, &buffer[0], m_K); - memcpy(output, &buffer[m_K], m_N - m_K); - } - } - else { - encode_rs_char(rsData, input, output); - } - - return ret; -} - - -int ReedSolomon::encode(void* data, size_t size) -{ - uint8_t* input = reinterpret_cast(data); - int ret = 0; - - if (reverse) { - ret = decode_rs_char(rsData, input, nullptr, 0); - } - else { - encode_rs_char(rsData, input, &input[m_K]); - } - - return ret; -} diff --git a/src/ReedSolomon.h b/src/ReedSolomon.h deleted file mode 100644 index abcef62..0000000 --- a/src/ReedSolomon.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - Copyright (C) 2005, 2006, 2007, 2008, 2009 Her Majesty the Queen in Right - of Canada (Communications Research Center Canada) - - Copyright (C) 2016 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include - -class ReedSolomon -{ -public: - ReedSolomon(int N, int K, - bool reverse = false, - int gfpoly = 0x11d, int firstRoot = 0, int primElem = 1); - ReedSolomon(const ReedSolomon& other) = delete; - ReedSolomon operator=(const ReedSolomon& other) = delete; - ~ReedSolomon(); - - void setReverse(bool state); - int encode(void* data, void* fec, size_t size); - int encode(void* data, size_t size); - -private: - int m_N; - int m_K; - - void* rsData; - bool reverse; -}; - diff --git a/src/ThreadsafeQueue.h b/src/ThreadsafeQueue.h deleted file mode 100644 index ab287b2..0000000 --- a/src/ThreadsafeQueue.h +++ /dev/null @@ -1,178 +0,0 @@ -/* - Copyright (C) 2007, 2008, 2009, 2010, 2011 Her Majesty the Queen in - Right of Canada (Communications Research Center Canada) - - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - - An implementation for a threadsafe queue, depends on C++11 - - When creating a ThreadsafeQueue, one can specify the minimal number - of elements it must contain before it is possible to take one - element out. - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -/* This queue is meant to be used by two threads. One producer - * that pushes elements into the queue, and one consumer that - * retrieves the elements. - * - * The queue can make the consumer block until an element - * is available, or a wakeup requested. - */ - -/* Class thrown by blocking pop to tell the consumer - * that there's a wakeup requested. */ -class ThreadsafeQueueWakeup {}; - -template -class ThreadsafeQueue -{ -public: - /* Push one element into the queue, and notify another thread that - * might be waiting. - * - * returns the new queue size. - */ - size_t push(T const& val) - { - std::unique_lock lock(the_mutex); - the_queue.push(val); - size_t queue_size = the_queue.size(); - lock.unlock(); - - the_rx_notification.notify_one(); - - return queue_size; - } - - size_t push(T&& val) - { - std::unique_lock lock(the_mutex); - the_queue.emplace(std::move(val)); - size_t queue_size = the_queue.size(); - lock.unlock(); - - the_rx_notification.notify_one(); - - return queue_size; - } - - /* Push one element into the queue, but wait until the - * queue size goes below the threshold. - * - * Notify waiting thread. - * - * returns the new queue size. - */ - size_t push_wait_if_full(T const& val, size_t threshold) - { - std::unique_lock lock(the_mutex); - while (the_queue.size() >= threshold) { - the_tx_notification.wait(lock); - } - the_queue.push(val); - size_t queue_size = the_queue.size(); - lock.unlock(); - - the_rx_notification.notify_one(); - - return queue_size; - } - - /* Trigger a wakeup event on a blocking consumer, which - * will receive a ThreadsafeQueueWakeup exception. - */ - void trigger_wakeup(void) - { - std::unique_lock lock(the_mutex); - wakeup_requested = true; - lock.unlock(); - the_rx_notification.notify_one(); - } - - /* Send a notification for the receiver thread */ - void notify(void) - { - the_rx_notification.notify_one(); - } - - bool empty() const - { - std::unique_lock lock(the_mutex); - return the_queue.empty(); - } - - size_t size() const - { - std::unique_lock lock(the_mutex); - return the_queue.size(); - } - - bool try_pop(T& popped_value) - { - std::unique_lock lock(the_mutex); - if (the_queue.empty()) { - return false; - } - - popped_value = the_queue.front(); - the_queue.pop(); - - lock.unlock(); - the_tx_notification.notify_one(); - - return true; - } - - void wait_and_pop(T& popped_value, size_t prebuffering = 1) - { - std::unique_lock lock(the_mutex); - while (the_queue.size() < prebuffering and - not wakeup_requested) { - the_rx_notification.wait(lock); - } - - if (wakeup_requested) { - wakeup_requested = false; - throw ThreadsafeQueueWakeup(); - } - else { - std::swap(popped_value, the_queue.front()); - the_queue.pop(); - - lock.unlock(); - the_tx_notification.notify_one(); - } - } - -private: - std::queue the_queue; - mutable std::mutex the_mutex; - std::condition_variable the_rx_notification; - std::condition_variable the_tx_notification; - bool wakeup_requested = false; -}; - diff --git a/src/dabOutput/edi/Config.h b/src/dabOutput/edi/Config.h index 55d5f0f..0c7dce8 100644 --- a/src/dabOutput/edi/Config.h +++ b/src/dabOutput/edi/Config.h @@ -50,11 +50,18 @@ struct udp_destination_t : public destination_t { }; // TCP server that can accept multiple connections -struct tcp_destination_t : public destination_t { +struct tcp_server_t : public destination_t { unsigned int listen_port = 0; size_t max_frames_queued = 1024; }; +// TCP client that connects to one endpoint +struct tcp_client_t : public destination_t { + std::string dest_addr; + unsigned int dest_port = 0; + size_t max_frames_queued = 1024; +}; + struct configuration_t { unsigned chunk_len = 207; // RSk, data length of each chunk unsigned fec = 0; // number of fragments that can be recovered diff --git a/src/dabOutput/edi/PFT.cpp b/src/dabOutput/edi/PFT.cpp index 5b93016..63dfa34 100644 --- a/src/dabOutput/edi/PFT.cpp +++ b/src/dabOutput/edi/PFT.cpp @@ -314,7 +314,7 @@ std::vector< PFTFragment > PFT::Assemble(AFPacket af_packet) #if 0 fprintf(stderr, "* PFT pseq %d, findex %d, fcount %d, plen %d\n", - m_pseq, findex, fcount, plen & ~0x8000); + m_pseq, findex, fcount, plen & ~0xC000); #endif } diff --git a/src/dabOutput/edi/Transport.cpp b/src/dabOutput/edi/Transport.cpp index 6d3950f..187aabe 100644 --- a/src/dabOutput/edi/Transport.cpp +++ b/src/dabOutput/edi/Transport.cpp @@ -45,12 +45,16 @@ void configuration_t::print() const } etiLog.level(info) << " source port " << udp_dest->source_port; } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { etiLog.level(info) << " TCP listening on port " << tcp_dest->listen_port; etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + etiLog.level(info) << " TCP client connecting to " << tcp_dest->dest_addr << ":" << tcp_dest->dest_port; + etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; + } else { - throw std::logic_error("EDI destination not implemented"); + throw logic_error("EDI destination not implemented"); } } if (interleaver_enabled()) { @@ -78,13 +82,18 @@ Sender::Sender(const configuration_t& conf) : udp_sockets.emplace(udp_dest.get(), udp_socket); } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { auto dispatcher = make_shared(tcp_dest->max_frames_queued); dispatcher->start(tcp_dest->listen_port, "0.0.0.0"); tcp_dispatchers.emplace(tcp_dest.get(), dispatcher); } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + auto tcp_socket = make_shared(); + tcp_socket->connect(tcp_dest->dest_addr, tcp_dest->dest_port); + tcp_senders.emplace(tcp_dest.get(), tcp_socket); + } else { - throw std::logic_error("EDI destination not implemented"); + throw logic_error("EDI destination not implemented"); } } @@ -111,7 +120,7 @@ void Sender::write(const TagPacket& tagpacket) vector edi_fragments = edi_pft.Assemble(af_packet); if (m_conf.verbose) { - fprintf(stderr, "EDI number of PFT fragment before interleaver %zu", + fprintf(stderr, "EDI number of PFT fragment before interleaver %zu\n", edi_fragments.size()); } @@ -128,22 +137,25 @@ void Sender::write(const TagPacket& tagpacket) udp_sockets.at(udp_dest.get())->send(edi_frag, addr); } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { tcp_dispatchers.at(tcp_dest.get())->write(edi_frag); } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_senders.at(tcp_dest.get())->sendall(edi_frag.data(), edi_frag.size()); + } else { - throw std::logic_error("EDI destination not implemented"); + throw logic_error("EDI destination not implemented"); } } if (m_conf.dump) { - std::ostream_iterator debug_iterator(edi_debug_file); - std::copy(edi_frag.begin(), edi_frag.end(), debug_iterator); + ostream_iterator debug_iterator(edi_debug_file); + copy(edi_frag.begin(), edi_frag.end(), debug_iterator); } } if (m_conf.verbose) { - fprintf(stderr, "EDI number of PFT fragments %zu", + fprintf(stderr, "EDI number of PFT fragments %zu\n", edi_fragments.size()); } } @@ -156,17 +168,20 @@ void Sender::write(const TagPacket& tagpacket) udp_sockets.at(udp_dest.get())->send(af_packet, addr); } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { tcp_dispatchers.at(tcp_dest.get())->write(af_packet); } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_senders.at(tcp_dest.get())->sendall(af_packet.data(), af_packet.size()); + } else { - throw std::logic_error("EDI destination not implemented"); + throw logic_error("EDI destination not implemented"); } } if (m_conf.dump) { - std::ostream_iterator debug_iterator(edi_debug_file); - std::copy(af_packet.begin(), af_packet.end(), debug_iterator); + ostream_iterator debug_iterator(edi_debug_file); + copy(af_packet.begin(), af_packet.end(), debug_iterator); } } } diff --git a/src/dabOutput/edi/Transport.h b/src/dabOutput/edi/Transport.h index 74126d1..9633275 100644 --- a/src/dabOutput/edi/Transport.h +++ b/src/dabOutput/edi/Transport.h @@ -36,6 +36,7 @@ #include #include #include +#include #include namespace edi { @@ -62,7 +63,8 @@ class Sender { edi::Interleaver edi_interleaver; std::unordered_map> udp_sockets; - std::unordered_map> tcp_dispatchers; + std::unordered_map> tcp_dispatchers; + std::unordered_map> tcp_senders; }; } diff --git a/src/input/Udp.cpp b/src/input/Udp.cpp index b4cced0..5d4f964 100644 --- a/src/input/Udp.cpp +++ b/src/input/Udp.cpp @@ -151,8 +151,8 @@ static uint16_t unpack2(const uint8_t *buf) int Sti_d_Rtp::open(const std::string& name) { - // Skip the sti-rtp:// part if it is present - const string endpoint = (name.substr(0, 10) == "sti-rtp://") ? + // Skip the rtp:// part if it is present + const string endpoint = (name.substr(0, 10) == "rtp://") ? name.substr(10) : name; // The endpoint should be address:port @@ -160,8 +160,8 @@ int Sti_d_Rtp::open(const std::string& name) if (colon_pos == string::npos) { stringstream ss; ss << "'" << name << - " is an invalid format for sti-rtp address: " - "expected [sti-rtp://]address:port"; + " is an invalid format for rtp address: " + "expected [rtp://]address:port"; throw invalid_argument(ss.str()); } -- cgit v1.2.3 From 03967733d70220e2de7af3cdad320aec5c82ede1 Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Tue, 25 Jun 2019 10:50:23 +0200 Subject: Add more EDI input improvements --- Makefile.am | 1 + lib/Socket.cpp | 97 +++++++- lib/Socket.h | 36 ++- lib/edi/PFT.cpp | 574 ++++++++++++++++++++++++++++++++++++++++++++++ lib/edi/PFT.hpp | 166 ++++++++++++++ lib/edi/STIDecoder.cpp | 191 +++++++++++++++ lib/edi/STIDecoder.hpp | 122 ++++++++++ lib/edi/STIWriter.cpp | 138 +++++++++++ lib/edi/STIWriter.hpp | 84 +++++++ lib/edi/buffer_unpack.hpp | 62 +++++ lib/edi/common.cpp | 300 ++++++++++++++++++++++++ lib/edi/common.hpp | 88 +++++++ src/input/Edi.cpp | 189 +++++++++++++++ src/input/Edi.h | 82 +++++++ 14 files changed, 2117 insertions(+), 13 deletions(-) create mode 100644 lib/edi/PFT.cpp create mode 100644 lib/edi/PFT.hpp create mode 100644 lib/edi/STIDecoder.cpp create mode 100644 lib/edi/STIDecoder.hpp create mode 100644 lib/edi/STIWriter.cpp create mode 100644 lib/edi/STIWriter.hpp create mode 100644 lib/edi/buffer_unpack.hpp create mode 100644 lib/edi/common.cpp create mode 100644 lib/edi/common.hpp create mode 100644 src/input/Edi.cpp create mode 100644 src/input/Edi.h (limited to 'lib') diff --git a/Makefile.am b/Makefile.am index 80d24e0..6e5aa71 100644 --- a/Makefile.am +++ b/Makefile.am @@ -168,6 +168,7 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ lib/edi/PFT.h \ lib/edi/common.cpp \ lib/edi/common.h \ + lib/edi/buffer_unpack.hpp \ lib/ReedSolomon.h \ lib/ReedSolomon.cpp \ lib/Socket.h \ diff --git a/lib/Socket.cpp b/lib/Socket.cpp index 9b404eb..cd70a8e 100644 --- a/lib/Socket.cpp +++ b/lib/Socket.cpp @@ -274,6 +274,8 @@ void UDPSocket::setMulticastTTL(int ttl) } } +UDPReceiver::UDPReceiver() { } + UDPReceiver::~UDPReceiver() { m_stop = true; m_sock.close(); @@ -355,7 +357,7 @@ TCPSocket::~TCPSocket() TCPSocket::TCPSocket(TCPSocket&& other) : m_sock(other.m_sock), - m_remote_address(other.m_remote_address) + m_remote_address(move(other.m_remote_address)) { if (other.m_sock != -1) { other.m_sock = -1; @@ -364,9 +366,9 @@ TCPSocket::TCPSocket(TCPSocket&& other) : TCPSocket& TCPSocket::operator=(TCPSocket&& other) { - m_sock = other.m_sock; - m_remote_address = other.m_remote_address; + swap(m_remote_address, other.m_remote_address); + m_sock = other.m_sock; if (other.m_sock != -1) { other.m_sock = -1; } @@ -487,14 +489,21 @@ void TCPSocket::listen(int port, const string& name) freeaddrinfo(result); + if (m_sock != INVALID_SOCKET) { #if defined(HAVE_SO_NOSIGPIPE) - int val = 1; - if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, - &val, sizeof(val)) < 0) { - throw std::runtime_error("Can't set SO_NOSIGPIPE"); - } + int val = 1; + if (setsockopt(m_sock, SOL_SOCKET, SO_NOSIGPIPE, + &val, sizeof(val)) < 0) { + throw std::runtime_error("Can't set SO_NOSIGPIPE"); + } #endif + int ret = ::listen(m_sock, 0); + if (ret == -1) { + throw std::runtime_error(string("Could not listen: ") + strerror(errno)); + } + } + if (rp == nullptr) { throw runtime_error("Could not bind"); } @@ -814,4 +823,76 @@ void TCPDataDispatcher::process() } } +TCPReceiveServer::TCPReceiveServer(size_t blocksize) : + m_blocksize(blocksize) +{ +} + +void TCPReceiveServer::start(int listen_port, const std::string& address) +{ + m_listener_socket.listen(listen_port, address); + + m_running = true; + m_listener_thread = std::thread(&TCPReceiveServer::process, this); +} + +TCPReceiveServer::~TCPReceiveServer() +{ + m_running = false; + if (m_listener_thread.joinable()) { + m_listener_thread.join(); + } +} + +vector TCPReceiveServer::receive() +{ + vector buffer; + m_queue.try_pop(buffer); + + // we can ignore try_pop()'s return value, because + // if it is unsuccessful the buffer is not touched. + return buffer; +} + +void TCPReceiveServer::process() +{ + constexpr int timeout_ms = 1000; + constexpr int disconnect_timeout_ms = 10000; + constexpr int max_num_timeouts = disconnect_timeout_ms / timeout_ms; + + while (m_running) { + auto sock = m_listener_socket.accept(timeout_ms); + + int num_timeouts = 0; + + while (m_running and sock.valid()) { + try { + vector buf(m_blocksize); + ssize_t r = sock.recv(buf.data(), buf.size(), 0, timeout_ms); + if (r < 0) { + throw logic_error("Invalid recv return value"); + } + else if (r == 0) { + sock.close(); + break; + } + else { + buf.resize(r); + m_queue.push(move(buf)); + } + } + catch (const TCPSocket::Interrupted&) { + break; + } + catch (const TCPSocket::Timeout&) { + num_timeouts++; + } + + if (num_timeouts > max_num_timeouts) { + sock.close(); + } + } + } +} + } diff --git a/lib/Socket.h b/lib/Socket.h index 2393584..8bb7fe1 100644 --- a/lib/Socket.h +++ b/lib/Socket.h @@ -127,7 +127,7 @@ class UDPSocket /* Threaded UDP receiver */ class UDPReceiver { public: - UDPReceiver() : m_port(0), m_thread(), m_stop(false), m_packets() {} + UDPReceiver(); ~UDPReceiver(); UDPReceiver(const UDPReceiver&) = delete; UDPReceiver operator=(const UDPReceiver&) = delete; @@ -142,12 +142,12 @@ class UDPReceiver { private: void m_run(void); - int m_port; + int m_port = 0; std::string m_bindto; std::string m_mcastaddr; - size_t m_max_packets_queued; + size_t m_max_packets_queued = 1; std::thread m_thread; - std::atomic m_stop; + std::atomic m_stop = ATOMIC_VAR_INIT(false); ThreadsafeQueue m_packets; UDPSocket m_sock; }; @@ -254,7 +254,7 @@ class TCPDataDispatcher void write(const std::vector& data); private: - void process(void); + void process(); size_t m_max_queue_size; @@ -265,4 +265,30 @@ class TCPDataDispatcher std::list m_connections; }; +/* A TCP Server to receive data, which abstracts the handling of connects and disconnects. + */ +class TCPReceiveServer { + public: + TCPReceiveServer(size_t blocksize); + ~TCPReceiveServer(); + TCPReceiveServer(const TCPReceiveServer&) = delete; + TCPReceiveServer& operator=(const TCPReceiveServer&) = delete; + + void start(int listen_port, const std::string& address); + + // Return a vector that contains up to blocksize bytes of data, or + // and empty vector if no data is available. + std::vector receive(); + + private: + void process(); + + size_t m_blocksize = 0; + ThreadsafeQueue > m_queue; + std::atomic m_running; + std::string m_exception_data; + std::thread m_listener_thread; + TCPSocket m_listener_socket; +}; + } diff --git a/lib/edi/PFT.cpp b/lib/edi/PFT.cpp new file mode 100644 index 0000000..aff7929 --- /dev/null +++ b/lib/edi/PFT.cpp @@ -0,0 +1,574 @@ +/* ------------------------------------------------------------------ + * Copyright (C) 2017 AVT GmbH - Fabien Vercasson + * Copyright (C) 2017 Matthias P. Braendli + * matthias.braendli@mpb.li + * + * http://opendigitalradio.org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * ------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include "crc.h" +#include "PFT.hpp" +#include "Log.h" +#include "buffer_unpack.hpp" +extern "C" { +#include "fec/fec.h" +} + +namespace EdiDecoder { +namespace PFT { + +using namespace std; + +const findex_t NUM_AFBUILDERS_TO_KEEP = 10; + +static bool checkCRC(const uint8_t *buf, size_t size) +{ + const uint16_t crc_from_packet = read_16b(buf + size - 2); + uint16_t crc_calc = 0xffff; + crc_calc = crc16(crc_calc, buf, size - 2); + crc_calc ^= 0xffff; + + return crc_from_packet == crc_calc; +} + +class FECDecoder { + public: + FECDecoder() { + m_rs_handler = init_rs_char( + symsize, gfPoly, firstRoot, primElem, nroots, pad); + } + FECDecoder(const FECDecoder& other) = delete; + FECDecoder& operator=(const FECDecoder& other) = delete; + ~FECDecoder() { + free_rs_char(m_rs_handler); + } + + // return -1 in case of failure, non-negative value if errors + // were corrected. + // Known positions of erasures should be given in eras_pos to + // improve decoding probability. After calling this function + // eras_pos will contain the positions of the corrected errors. + int decode(vector &data, vector &eras_pos) { + assert(data.size() == N); + const size_t no_eras = eras_pos.size(); + + eras_pos.resize(nroots); + int num_err = decode_rs_char(m_rs_handler, data.data(), + eras_pos.data(), no_eras); + if (num_err > 0) { + eras_pos.resize(num_err); + } + return num_err; + } + + // return -1 in case of failure, non-negative value if errors + // were corrected. No known erasures. + int decode(vector &data) { + assert(data.size() == N); + int num_err = decode_rs_char(m_rs_handler, data.data(), nullptr, 0); + return num_err; + } + + private: + void* m_rs_handler; + + const int firstRoot = 1; // Discovered by analysing EDI dump + const int gfPoly = 0x11d; + + // The encoding has to be 255, 207 always, because the chunk has to + // be padded at the end, and not at the beginning as libfec would + // do + const size_t N = 255; + const size_t K = 207; + const int primElem = 1; + const int symsize = 8; + const size_t nroots = N - K; // For EDI PFT, this must be 48 + const size_t pad = ((1 << symsize) - 1) - N; // is 255-N + +}; + +size_t Fragment::loadData(const std::vector &buf) +{ + const size_t header_len = 14; + if (buf.size() < header_len) { + return 0; + } + + size_t index = 0; + + // Parse PFT Fragment Header (ETSI TS 102 821 V1.4.1 ch7.1) + if (not (buf[0] == 'P' and buf[1] == 'F') ) { + throw invalid_argument("Invalid PFT SYNC bytes"); + } + index += 2; // Psync + + _Pseq = read_16b(buf.begin()+index); index += 2; + _Findex = read_24b(buf.begin()+index); index += 3; + _Fcount = read_24b(buf.begin()+index); index += 3; + _FEC = unpack1bit(buf[index], 0); + _Addr = unpack1bit(buf[index], 1); + _Plen = read_16b(buf.begin()+index) & 0x3FFF; index += 2; + + const size_t required_len = header_len + + (_FEC ? 1 : 0) + + (_Addr ? 2 : 0) + + 2; // CRC + if (buf.size() < required_len) { + return 0; + } + + // Optional RS Header + _RSk = 0; + _RSz = 0; + if (_FEC) { + _RSk = buf[index]; index += 1; + _RSz = buf[index]; index += 1; + } + + // Optional transport header + _Source = 0; + _Dest = 0; + if (_Addr) { + _Source = read_16b(buf.begin()+index); index += 2; + _Dest = read_16b(buf.begin()+index); index += 2; + } + + index += 2; + const bool crc_valid = checkCRC(buf.data(), index); + const bool buf_has_enough_data = (buf.size() >= index + _Plen); + + if (not buf_has_enough_data) { + return 0; + } + + _valid = ((not _FEC) or crc_valid) and buf_has_enough_data; + +#if 0 + if (!_valid) { + stringstream ss; + ss << "Invalid PF fragment: "; + if (_FEC) { + ss << " RSk=" << (uint32_t)_RSk << " RSz=" << (uint32_t)_RSz; + } + + if (_Addr) { + ss << " Source=" << _Source << " Dest=" << _Dest; + } + etiLog.log(debug, "%s\n", ss.str().c_str()); + } +#endif + + _payload.clear(); + if (_valid) { + copy( buf.begin()+index, + buf.begin()+index+_Plen, + back_inserter(_payload)); + index += _Plen; + } + + return index; +} + + +AFBuilder::AFBuilder(pseq_t Pseq, findex_t Fcount, size_t lifetime) +{ + _Pseq = Pseq; + _Fcount = Fcount; + assert(lifetime > 0); + lifeTime = lifetime; +} + +void AFBuilder::pushPFTFrag(const Fragment &frag) +{ + if (_Pseq != frag.Pseq() or _Fcount != frag.Fcount()) { + throw invalid_argument("Invalid PFT fragment Pseq or Fcount"); + } + const auto Findex = frag.Findex(); + const bool fragment_already_received = _fragments.count(Findex); + + if (not fragment_already_received) + { + _fragments[Findex] = frag; + } +} + +bool Fragment::checkConsistency(const Fragment& other) const +{ + /* Consistency check, TS 102 821 Clause 7.3.2. + * + * Every PFT Fragment produced from a single AF or RS Packet shall have + * the same values in all of the PFT Header fields except for the Findex, + * Plen and HCRC fields. + */ + + return other._Fcount == _Fcount and + other._FEC == _FEC and + other._RSk == _RSk and + other._RSz == _RSz and + other._Addr == _Addr and + other._Source == _Source and + other._Dest == _Dest and + + /* The Plen field of all fragments shall be the s for the initial f-1 + * fragments and s - (L%f) for the final fragment. + * Note that when Reed Solomon has been used, all fragments will be of + * length s. + */ + (_FEC ? other._Plen == _Plen : true); +} + + +AFBuilder::decode_attempt_result_t AFBuilder::canAttemptToDecode() const +{ + if (_fragments.empty()) { + return AFBuilder::decode_attempt_result_t::no; + } + + if (_fragments.size() == _Fcount) { + return AFBuilder::decode_attempt_result_t::yes; + } + + /* Check that all fragments are consistent */ + const Fragment& first = _fragments.begin()->second; + if (not std::all_of(_fragments.begin(), _fragments.end(), + [&](const pair& pair) { + const Fragment& frag = pair.second; + return first.checkConsistency(frag) and _Pseq == frag.Pseq(); + }) ) { + throw invalid_argument("Inconsistent PFT fragments"); + } + + // Calculate the minimum number of fragments necessary to apply FEC. + // This can't be done with the last fragment that may have a + // smaller size + // ETSI TS 102 821 V1.4.1 ch 7.4.4 + auto frag_it = _fragments.begin(); + if (frag_it->second.Fcount() == _Fcount - 1) { + frag_it++; + + if (frag_it == _fragments.end()) { + return AFBuilder::decode_attempt_result_t::no; + } + } + + const Fragment& frag = frag_it->second; + + if ( frag.FEC() ) + { + const uint16_t _Plen = frag.Plen(); + + /* max number of RS chunks that may have been sent */ + const uint32_t _cmax = (_Fcount*_Plen) / (frag.RSk()+48); + assert(_cmax > 0); + + /* Receiving _rxmin fragments does not guarantee that decoding + * will succeed! */ + const uint32_t _rxmin = _Fcount - (_cmax*48)/_Plen; + + if (_fragments.size() >= _rxmin) { + return AFBuilder::decode_attempt_result_t::maybe; + } + } + + return AFBuilder::decode_attempt_result_t::no; +} + +std::vector AFBuilder::extractAF() const +{ + if (not _af_packet.empty()) { + return _af_packet; + } + + bool ok = false; + + if (canAttemptToDecode() != AFBuilder::decode_attempt_result_t::no) { + + auto frag_it = _fragments.begin(); + if (frag_it->second.Fcount() == _Fcount - 1) { + frag_it++; + + if (frag_it == _fragments.end()) { + throw std::runtime_error("Invalid attempt at extracting AF"); + } + } + + const Fragment& ref_frag = frag_it->second; + const auto RSk = ref_frag.RSk(); + const auto RSz = ref_frag.RSz(); + const auto Plen = ref_frag.Plen(); + + if ( ref_frag.FEC() ) + { + const uint32_t cmax = (_Fcount*Plen) / (RSk+48); + + // Keep track of erasures (missing fragments) for + // every chunk + map > erasures; + + + // Assemble fragments into a RS block, immediately + // deinterleaving it. + vector rs_block(Plen * _Fcount); + for (size_t j = 0; j < _Fcount; j++) { + const bool fragment_present = _fragments.count(j); + if (fragment_present) { + const auto& fragment = _fragments.at(j).payload(); + + if (j != _Fcount - 1 and fragment.size() != Plen) { + throw runtime_error("Incorrect fragment length " + + to_string(fragment.size()) + " " + + to_string(Plen)); + } + + if (j == _Fcount - 1 and fragment.size() > Plen) { + throw runtime_error("Incorrect last fragment length " + + to_string(fragment.size()) + " " + + to_string(Plen)); + } + + size_t k = 0; + for (; k < fragment.size(); k++) { + rs_block[k * _Fcount + j] = fragment[k]; + } + + for (; k < Plen; k++) { + rs_block[k * _Fcount + j] = 0x00; + } + } + else { + // fill with zeros if fragment is missing + for (size_t k = 0; k < Plen; k++) { + rs_block[k * _Fcount + j] = 0x00; + + const size_t chunk_ix = (k * _Fcount + j) / (RSk + 48); + const size_t chunk_offset = (k * _Fcount + j) % (RSk + 48); + erasures[chunk_ix].push_back(chunk_offset); + } + } + } + + // The RS block is a concatenation of chunks of RSk bytes + 48 parity + // followed by RSz padding + + FECDecoder fec; + for (size_t i = 0; i < cmax; i++) { + // We need to pad the chunk ourself + vector chunk(255); + const auto& block_begin = rs_block.begin() + (RSk + 48) * i; + copy(block_begin, block_begin + RSk, chunk.begin()); + // bytes between RSk and 207 are 0x00 already + copy(block_begin + RSk, block_begin + RSk + 48, + chunk.begin() + 207); + + int errors_corrected = -1; + if (erasures.count(i)) { + errors_corrected = fec.decode(chunk, erasures[i]); + } + else { + errors_corrected = fec.decode(chunk); + } + + if (errors_corrected == -1) { + _af_packet.clear(); + return {}; + } + +#if 0 + if (errors_corrected > 0) { + etiLog.log(debug, "Corrected %d errors at ", errors_corrected); + for (const auto &index : erasures[i]) { + etiLog.log(debug, " %d", index); + } + etiLog.log(debug, "\n"); + } +#endif + + _af_packet.insert(_af_packet.end(), chunk.begin(), chunk.begin() + RSk); + } + + _af_packet.resize(_af_packet.size() - RSz); + } + else { + // No FEC: just assemble fragments + + for (size_t j = 0; j < _Fcount; ++j) { + const bool fragment_present = _fragments.count(j); + if (fragment_present) + { + const auto& fragment = _fragments.at(j); + + _af_packet.insert(_af_packet.end(), + fragment.payload().begin(), + fragment.payload().end()); + } + else { + throw logic_error("Missing fragment"); + } + } + } + + // EDI specific, must have a CRC. + if( _af_packet.size() >= 12 ) { + ok = checkCRC(_af_packet.data(), _af_packet.size()); + + if (not ok) { + etiLog.log(debug, "Too many errors to reconstruct AF from %zu/%u" + " PFT fragments\n", _fragments.size(), _Fcount); + } + } + } + + if (not ok) { + _af_packet.clear(); + } + + return _af_packet; +} + +std::string AFBuilder::visualise() const +{ + stringstream ss; + ss << "|"; + for (size_t i = 0; i < _Fcount; i++) { + if (_fragments.count(i)) { + ss << "."; + } + else { + ss << " "; + } + } + ss << "| " << AFBuilder::dar_to_string(canAttemptToDecode()) << " " << lifeTime; + return ss.str(); +} + +void PFT::pushPFTFrag(const Fragment &fragment) +{ + // Start decoding the first pseq we receive. In normal + // operation without interruptions, the map should + // never become empty + if (m_afbuilders.empty()) { + m_next_pseq = fragment.Pseq(); + etiLog.log(debug,"Initialise next_pseq to %u\n", m_next_pseq); + } + + if (m_afbuilders.count(fragment.Pseq()) == 0) { + // The AFBuilder wants to know the lifetime in number of fragments, + // we know the delay in number of AF packets. Every AF packet + // is cut into Fcount fragments. + const size_t lifetime = fragment.Fcount() * m_max_delay; + + // Build the afbuilder in the map in-place + m_afbuilders.emplace(std::piecewise_construct, + /* key */ + std::forward_as_tuple(fragment.Pseq()), + /* builder */ + std::forward_as_tuple(fragment.Pseq(), fragment.Fcount(), lifetime)); + } + + auto& p = m_afbuilders.at(fragment.Pseq()); + p.pushPFTFrag(fragment); + + if (m_verbose) { + etiLog.log(debug, "Got frag %u:%u, afbuilders: ", + fragment.Pseq(), fragment.Findex()); + for (const auto &k : m_afbuilders) { + const bool isNextPseq = (m_next_pseq == k.first); + etiLog.level(debug) << (isNextPseq ? "->" : " ") << + k.first << " " << k.second.visualise(); + } + } +} + + +std::vector PFT::getNextAFPacket() +{ + if (m_afbuilders.count(m_next_pseq) == 0) { + if (m_afbuilders.size() > m_max_delay) { + m_afbuilders.clear(); + etiLog.level(debug) << " Reinit"; + } + + return {}; + } + + auto &builder = m_afbuilders.at(m_next_pseq); + + using dar_t = AFBuilder::decode_attempt_result_t; + + if (builder.canAttemptToDecode() == dar_t::yes) { + auto afpacket = builder.extractAF(); + assert(not afpacket.empty()); + incrementNextPseq(); + return afpacket; + } + else if (builder.canAttemptToDecode() == dar_t::maybe) { + if (builder.lifeTime > 0) { + builder.lifeTime--; + } + + if (builder.lifeTime == 0) { + // Attempt Reed-Solomon decoding + auto afpacket = builder.extractAF(); + + if (afpacket.empty()) { + etiLog.log(debug,"pseq %d timed out after RS", m_next_pseq); + } + incrementNextPseq(); + return afpacket; + } + } + else { + if (builder.lifeTime > 0) { + builder.lifeTime--; + } + + if (builder.lifeTime == 0) { + etiLog.log(debug, "pseq %d timed out\n", m_next_pseq); + incrementNextPseq(); + } + } + + return {}; +} + +void PFT::setMaxDelay(size_t num_af_packets) +{ + m_max_delay = num_af_packets; +} + +void PFT::setVerbose(bool enable) +{ + m_verbose = enable; +} + +void PFT::incrementNextPseq() +{ + if (m_afbuilders.count(m_next_pseq - NUM_AFBUILDERS_TO_KEEP) > 0) { + m_afbuilders.erase(m_next_pseq - NUM_AFBUILDERS_TO_KEEP); + } + + m_next_pseq++; +} + +} +} diff --git a/lib/edi/PFT.hpp b/lib/edi/PFT.hpp new file mode 100644 index 0000000..779509b --- /dev/null +++ b/lib/edi/PFT.hpp @@ -0,0 +1,166 @@ +/* ------------------------------------------------------------------ + * Copyright (C) 2017 AVT GmbH - Fabien Vercasson + * Copyright (C) 2017 Matthias P. Braendli + * matthias.braendli@mpb.li + * + * http://opendigitalradio.org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. + * See the License for the specific language governing permissions + * and limitations under the License. + * ------------------------------------------------------------------- + */ + +#pragma once +#include +#include +#include +#include + +namespace EdiDecoder { +namespace PFT { + +using pseq_t = uint16_t; +using findex_t = uint32_t; // findex is a 24-bit value + +class Fragment +{ + public: + // Load the data for one fragment from buf into + // the Fragment. + // \returns the number of bytes of useful data found in buf + // A non-zero return value doesn't imply a valid fragment + // the isValid() method must be used to verify this. + size_t loadData(const std::vector &buf); + + bool isValid() const { return _valid; } + pseq_t Pseq() const { return _Pseq; } + findex_t Findex() const { return _Findex; } + findex_t Fcount() const { return _Fcount; } + bool FEC() const { return _FEC; } + uint16_t Plen() const { return _Plen; } + uint8_t RSk() const { return _RSk; } + uint8_t RSz() const { return _RSz; } + const std::vector& payload() const + { return _payload; } + + bool checkConsistency(const Fragment& other) const; + + private: + std::vector _payload; + + pseq_t _Pseq = 0; + findex_t _Findex = 0; + findex_t _Fcount = 0; + bool _FEC = false; + bool _Addr = false; + uint16_t _Plen = 0; + uint8_t _RSk = 0; + uint8_t _RSz = 0; + uint16_t _Source = 0; + uint16_t _Dest = 0; + bool _valid = false; +}; + +/* The AFBuilder collects Fragments and builds an Application Frame + * out of them. It does error correction if necessary + */ +class AFBuilder +{ + public: + enum class decode_attempt_result_t { + yes, // The AF packet can be build because all fragments are present + maybe, // RS decoding may correctly decode the AF packet + no, // Not enough fragments present to permit RS + }; + + static std::string dar_to_string(decode_attempt_result_t dar) { + switch (dar) { + case decode_attempt_result_t::yes: return "y"; + case decode_attempt_result_t::no: return "n"; + case decode_attempt_result_t::maybe: return "m"; + } + return "?"; + } + + AFBuilder(pseq_t Pseq, findex_t Fcount, size_t lifetime); + + void pushPFTFrag(const Fragment &frag); + + /* Assess if it may be possible to decode this AF packet */ + decode_attempt_result_t canAttemptToDecode() const; + + /* Try to build the AF with received fragments. + * Apply error correction if necessary (missing packets/CRC errors) + * \return an empty vector if building the AF is not possible + */ + std::vector extractAF(void) const; + + std::pair + numberOfFragments(void) const { + return {_fragments.size(), _Fcount}; + } + + std::string visualise(void) const; + + /* The user of this instance can keep track of the lifetime of this + * builder + */ + size_t lifeTime; + + private: + + // A map from fragment index to fragment + std::map _fragments; + + // cached version of decoded AF packet + mutable std::vector _af_packet; + + pseq_t _Pseq; + findex_t _Fcount; +}; + +class PFT +{ + public: + void pushPFTFrag(const Fragment &fragment); + + /* Try to build the AF packet for the next pseq. This might + * skip one or more pseq according to the maximum delay setting. + * + * \return an empty vector if building the AF is not possible + */ + std::vector getNextAFPacket(void); + + /* Set the maximum delay in number of AF Packets before we + * abandon decoding a given pseq. + */ + void setMaxDelay(size_t num_af_packets); + + /* Enable verbose fprintf */ + void setVerbose(bool enable); + + private: + void incrementNextPseq(void); + + pseq_t m_next_pseq; + size_t m_max_delay = 10; // in AF packets + + // Keep one AFBuilder for each Pseq + std::map m_afbuilders; + + bool m_verbose = 0; +}; + +} + +} diff --git a/lib/edi/STIDecoder.cpp b/lib/edi/STIDecoder.cpp new file mode 100644 index 0000000..ca8cead --- /dev/null +++ b/lib/edi/STIDecoder.cpp @@ -0,0 +1,191 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "STIDecoder.hpp" +#include "buffer_unpack.hpp" +#include "crc.h" +#include "Log.h" +#include +#include +#include + +namespace EdiDecoder { + +using namespace std; + +STIDecoder::STIDecoder(STIDataCollector& data_collector, bool verbose) : + m_data_collector(data_collector), + m_dispatcher(std::bind(&STIDecoder::packet_completed, this), verbose) +{ + using std::placeholders::_1; + using std::placeholders::_2; + m_dispatcher.register_tag("*ptr", + std::bind(&STIDecoder::decode_starptr, this, _1, _2)); + m_dispatcher.register_tag("dsti", + std::bind(&STIDecoder::decode_dsti, this, _1, _2)); + m_dispatcher.register_tag("ss", + std::bind(&STIDecoder::decode_ssn, this, _1, _2)); + m_dispatcher.register_tag("*dmy", + std::bind(&STIDecoder::decode_stardmy, this, _1, _2)); +} + +void STIDecoder::push_bytes(const vector &buf) +{ + m_dispatcher.push_bytes(buf); +} + +void STIDecoder::push_packet(const vector &buf) +{ + m_dispatcher.push_packet(buf); +} + +void STIDecoder::setMaxDelay(int num_af_packets) +{ + m_dispatcher.setMaxDelay(num_af_packets); +} + +#define AFPACKET_HEADER_LEN 10 // includes SYNC + +bool STIDecoder::decode_starptr(const vector &value, uint16_t) +{ + if (value.size() != 0x40 / 8) { + etiLog.log(warn, "Incorrect length %02lx for *PTR", value.size()); + return false; + } + + char protocol_sz[5]; + protocol_sz[4] = '\0'; + copy(value.begin(), value.begin() + 4, protocol_sz); + string protocol(protocol_sz); + + uint16_t major = read_16b(value.begin() + 4); + uint16_t minor = read_16b(value.begin() + 6); + + m_data_collector.update_protocol(protocol, major, minor); + + return true; +} + +bool STIDecoder::decode_dsti(const vector &value, uint16_t) +{ + size_t offset = 0; + + const uint16_t dstiHeader = read_16b(value.begin() + offset); + offset += 2; + + sti_management_data md; + + md.stihf = (dstiHeader >> 15) & 0x1; + md.atstf = (dstiHeader >> 14) & 0x1; + md.rfadf = (dstiHeader >> 13) & 0x1; + uint8_t dfcth = (dstiHeader >> 8) & 0x1F; + uint8_t dfctl = dstiHeader & 0xFF; + + md.dflc = dfcth * 250 + dfctl; // modulo 5000 counter + + const size_t expected_length = 2 + + (md.stihf ? 3 : 0) + + (md.atstf ? 1 + 4 + 3 : 0) + + (md.rfadf ? 9 : 0); + + if (value.size() != expected_length) { + throw std::logic_error("EDI dsti: Assertion error:" + "value.size() != expected_length: " + + to_string(value.size()) + " " + + to_string(expected_length)); + } + + if (md.stihf) { + const uint8_t stat = value[offset++]; + const uint16_t spid = read_16b(value.begin() + offset); + m_data_collector.update_stat(stat, spid); + offset += 2; + } + + if (md.atstf) { + uint8_t utco = value[offset]; + offset++; + + uint32_t seconds = read_32b(value.begin() + offset); + offset += 4; + + m_data_collector.update_edi_time(utco, seconds); + + md.tsta = read_24b(value.begin() + offset); + offset += 3; + } + else { + // Null timestamp, ETSI ETS 300 799, C.2.2 + md.tsta = 0xFFFFFF; + } + + + if (md.rfadf) { + std::array rfad; + copy(value.cbegin() + offset, + value.cbegin() + offset + 9, + rfad.begin()); + offset += 9; + + m_data_collector.update_rfad(rfad); + } + + m_data_collector.update_sti_management(md); + + return true; +} + +bool STIDecoder::decode_ssn(const vector &value, uint16_t n) +{ + sti_payload_data sti; + + sti.stream_index = n - 1; // n is 1-indexed + sti.rfa = value[0] >> 3; + sti.tid = value[0] & 0x07; + + uint16_t istc = read_24b(value.begin() + 1); + sti.tidext = istc >> 13; + sti.crcstf = (istc >> 12) & 0x1; + sti.stid = istc & 0xFFF; + + if (sti.rfa != 0) { + etiLog.level(warn) << "EDI: rfa field in SSnn tag non-null"; + } + + copy( value.cbegin() + 3, + value.cend(), + back_inserter(sti.istd)); + + m_data_collector.add_payload(move(sti)); + + return true; +} + +bool STIDecoder::decode_stardmy(const vector& /*value*/, uint16_t) +{ + return true; +} + +void STIDecoder::packet_completed() +{ + m_data_collector.assemble(); +} + +} diff --git a/lib/edi/STIDecoder.hpp b/lib/edi/STIDecoder.hpp new file mode 100644 index 0000000..201a176 --- /dev/null +++ b/lib/edi/STIDecoder.hpp @@ -0,0 +1,122 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "common.hpp" +#include +#include +#include +#include + +namespace EdiDecoder { + +// Information for STI-D Management +struct sti_management_data { + bool stihf; + bool atstf; + bool rfadf; + uint16_t dflc; + + uint32_t tsta; +}; + +// Information for a subchannel available in EDI +struct sti_payload_data { + uint16_t stream_index; + uint8_t rfa; + uint8_t tid; + uint8_t tidext; + bool crcstf; + uint16_t stid; + std::vector istd; + + // Return the length of ISTD in bytes + uint16_t stl(void) const { return istd.size(); } +}; + +/* A class that receives STI data must implement the interface described + * in the STIDataCollector. This can be e.g. a converter to ETI, or something that + * prepares data structures for a modulator. + */ +class STIDataCollector { + public: + // Tell the ETIWriter what EDI protocol we receive in *ptr. + // This is not part of the ETI data, but is used as check + virtual void update_protocol( + const std::string& proto, + uint16_t major, + uint16_t minor) = 0; + + // STAT error field and service provider ID + virtual void update_stat(uint8_t stat, uint16_t spid) = 0; + + // In addition to TSTA in ETI, EDI also transports more time + // stamp information. + virtual void update_edi_time(uint32_t utco, uint32_t seconds) = 0; + + virtual void update_rfad(std::array rfad) = 0; + virtual void update_sti_management(const sti_management_data& data) = 0; + + virtual void add_payload(sti_payload_data&& payload) = 0; + + virtual void assemble() = 0; +}; + +/* The STIDecoder takes care of decoding the EDI TAGs related to the transport + * of ETI(NI) data inside AF and PF packets. + * + * PF packets are handed over to the PFT decoder, which will in turn return + * AF packets. AF packets are directly handled (TAG extraction) here. + */ +class STIDecoder { + public: + STIDecoder(STIDataCollector& data_collector, bool verbose); + + /* Push bytes into the decoder. The buf can contain more + * than a single packet. This is useful when reading from streams + * (files, TCP) + */ + void push_bytes(const std::vector &buf); + + /* Push a complete packet into the decoder. Useful for UDP and other + * datagram-oriented protocols. + */ + void push_packet(const std::vector &buf); + + /* Set the maximum delay in number of AF Packets before we + * abandon decoding a given pseq. + */ + void setMaxDelay(int num_af_packets); + + private: + bool decode_starptr(const std::vector &value, uint16_t); + bool decode_dsti(const std::vector &value, uint16_t); + bool decode_ssn(const std::vector &value, uint16_t n); + bool decode_stardmy(const std::vector &value, uint16_t); + + void packet_completed(); + + STIDataCollector& m_data_collector; + TagDispatcher m_dispatcher; + +}; + +} diff --git a/lib/edi/STIWriter.cpp b/lib/edi/STIWriter.cpp new file mode 100644 index 0000000..6964eb1 --- /dev/null +++ b/lib/edi/STIWriter.cpp @@ -0,0 +1,138 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "STIWriter.hpp" +#include "crc.h" +#include "Log.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace EdiDecoder { + +using namespace std; + +void STIWriter::update_protocol( + const std::string& proto, + uint16_t major, + uint16_t minor) +{ + m_proto_valid = (proto == "DSTI" and major == 0 and minor == 0); + + if (not m_proto_valid) { + throw std::invalid_argument("Wrong EDI protocol"); + } +} + +void STIWriter::reinit() +{ + m_proto_valid = false; + m_management_data_valid = false; + m_stat_valid = false; + m_time_valid = false; + m_payload_valid = false; + m_stiFrame.frame.clear(); +} + +void STIWriter::update_stat(uint8_t stat, uint16_t spid) +{ + m_stat = stat; + m_spid = spid; + m_stat_valid = true; + + if (m_stat != 0xFF) { + etiLog.log(warn, "STI errorlevel %02x", m_stat); + } +} + +void STIWriter::update_rfad(std::array rfad) +{ + (void)rfad; +} + +void STIWriter::update_sti_management(const sti_management_data& data) +{ + m_management_data = data; + m_management_data_valid = true; +} + +void STIWriter::add_payload(sti_payload_data&& payload) +{ + m_payload = move(payload); + m_payload_valid = true; +} + +void STIWriter::update_edi_time( + uint32_t utco, + uint32_t seconds) +{ + if (not m_proto_valid) { + throw std::logic_error("Cannot update time before protocol"); + } + + m_utco = utco; + m_seconds = seconds; + + // TODO check validity + m_time_valid = true; +} + + +void STIWriter::assemble() +{ + if (not m_proto_valid) { + throw std::logic_error("Cannot assemble STI before protocol"); + } + + if (not m_management_data_valid) { + throw std::logic_error("Cannot assemble STI before management data"); + } + + if (not m_payload_valid) { + throw std::logic_error("Cannot assemble STI without frame data"); + } + + // TODO check time validity + + // Do copies so as to preserve existing payload data + m_stiFrame.frame = m_payload.istd; + m_stiFrame.timestamp.seconds = m_seconds; + m_stiFrame.timestamp.utco = m_utco; +} + +sti_frame_t STIWriter::getFrame() +{ + if (m_stiFrame.frame.empty()) { + return {}; + } + + sti_frame_t sti; + swap(sti, m_stiFrame); + reinit(); + return sti; +} + +} + diff --git a/lib/edi/STIWriter.hpp b/lib/edi/STIWriter.hpp new file mode 100644 index 0000000..a75cb69 --- /dev/null +++ b/lib/edi/STIWriter.hpp @@ -0,0 +1,84 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "common.hpp" +#include "STIDecoder.hpp" +#include +#include +#include +#include + +namespace EdiDecoder { + +struct sti_frame_t { + std::vector frame; + frame_timestamp_t timestamp; +}; + +class STIWriter : public STIDataCollector { + public: + // Tell the ETIWriter what EDI protocol we receive in *ptr. + // This is not part of the ETI data, but is used as check + virtual void update_protocol( + const std::string& proto, + uint16_t major, + uint16_t minor); + + virtual void update_stat(uint8_t stat, uint16_t spid); + + virtual void update_edi_time( + uint32_t utco, + uint32_t seconds); + + virtual void update_rfad(std::array rfad); + virtual void update_sti_management(const sti_management_data& data); + virtual void add_payload(sti_payload_data&& payload); + + virtual void assemble(void); + + // Return the assembled frame or an empty frame if not ready + sti_frame_t getFrame(); + + private: + void reinit(void); + + bool m_proto_valid = false; + + bool m_management_data_valid = false; + sti_management_data m_management_data; + + bool m_stat_valid = false; + uint8_t m_stat = 0; + uint16_t m_spid = 0; + + bool m_time_valid = false; + uint32_t m_utco = 0; + uint32_t m_seconds = 0; + + bool m_payload_valid = false; + sti_payload_data m_payload; + + sti_frame_t m_stiFrame; +}; + +} + diff --git a/lib/edi/buffer_unpack.hpp b/lib/edi/buffer_unpack.hpp new file mode 100644 index 0000000..05a1534 --- /dev/null +++ b/lib/edi/buffer_unpack.hpp @@ -0,0 +1,62 @@ +/* + Copyright (C) 2016 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once +#include + +namespace EdiDecoder { + +template +uint16_t read_16b(T buf) +{ + uint16_t value = 0; + value = (uint16_t)(buf[0]) << 8; + value |= (uint16_t)(buf[1]); + return value; +} + +template +uint32_t read_24b(T buf) +{ + uint32_t value = 0; + value = (uint32_t)(buf[0]) << 16; + value |= (uint32_t)(buf[1]) << 8; + value |= (uint32_t)(buf[2]); + return value; +} + +template +uint32_t read_32b(T buf) +{ + uint32_t value = 0; + value = (uint32_t)(buf[0]) << 24; + value |= (uint32_t)(buf[1]) << 16; + value |= (uint32_t)(buf[2]) << 8; + value |= (uint32_t)(buf[3]); + return value; +} + +inline uint32_t unpack1bit(uint8_t byte, int bitpos) +{ + return (byte & 1 << (7-bitpos)) > (7-bitpos); +} + +} diff --git a/lib/edi/common.cpp b/lib/edi/common.cpp new file mode 100644 index 0000000..bc0fa1b --- /dev/null +++ b/lib/edi/common.cpp @@ -0,0 +1,300 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "common.hpp" +#include "buffer_unpack.hpp" +#include "Log.h" +#include "crc.h" +#include +#include +#include +#include + +namespace EdiDecoder { + +using namespace std; + +string frame_timestamp_t::to_string() const +{ + const time_t seconds_in_unix_epoch = to_unix_epoch(); + + stringstream ss; + ss << "Timestamp: " << std::put_time(std::gmtime(&seconds_in_unix_epoch), "%c %Z"); + return ss.str(); +} + +time_t frame_timestamp_t::to_unix_epoch() const +{ + // EDI epoch: 2000-01-01T00:00:00Z + // Convert using + // TZ=UTC python -c 'import datetime; print(datetime.datetime(2000,1,1,0,0,0,0).strftime("%s"))' + return 946684800 + seconds - utco; +} + + +TagDispatcher::TagDispatcher( + std::function&& af_packet_completed, bool verbose) : + m_af_packet_completed(move(af_packet_completed)) +{ + m_pft.setVerbose(verbose); +} + +void TagDispatcher::push_bytes(const vector &buf) +{ + copy(buf.begin(), buf.end(), back_inserter(m_input_data)); + + while (m_input_data.size() > 2) { + if (m_input_data[0] == 'A' and m_input_data[1] == 'F') { + const decode_state_t st = decode_afpacket(m_input_data); + + if (st.num_bytes_consumed == 0 and not st.complete) { + // We need to refill our buffer + break; + } + + if (st.num_bytes_consumed) { + vector remaining_data; + copy(m_input_data.begin() + st.num_bytes_consumed, + m_input_data.end(), + back_inserter(remaining_data)); + m_input_data = remaining_data; + } + + if (st.complete) { + m_af_packet_completed(); + } + } + else if (m_input_data[0] == 'P' and m_input_data[1] == 'F') { + PFT::Fragment fragment; + const size_t fragment_bytes = fragment.loadData(m_input_data); + + if (fragment_bytes == 0) { + // We need to refill our buffer + break; + } + + vector remaining_data; + copy(m_input_data.begin() + fragment_bytes, + m_input_data.end(), + back_inserter(remaining_data)); + m_input_data = remaining_data; + + if (fragment.isValid()) { + m_pft.pushPFTFrag(fragment); + } + + auto af = m_pft.getNextAFPacket(); + if (not af.empty()) { + decode_state_t st = decode_afpacket(af); + + if (st.complete) { + m_af_packet_completed(); + } + } + } + else { + etiLog.log(warn,"Unknown %c!", *m_input_data.data()); + m_input_data.erase(m_input_data.begin()); + } + } +} + +void TagDispatcher::push_packet(const vector &buf) +{ + if (buf.size() < 2) { + throw std::invalid_argument("Not enough bytes to read EDI packet header"); + } + + if (buf[0] == 'A' and buf[1] == 'F') { + const decode_state_t st = decode_afpacket(buf); + + if (st.complete) { + m_af_packet_completed(); + } + + } + else if (buf[0] == 'P' and buf[1] == 'F') { + PFT::Fragment fragment; + fragment.loadData(buf); + + if (fragment.isValid()) { + m_pft.pushPFTFrag(fragment); + } + + auto af = m_pft.getNextAFPacket(); + if (not af.empty()) { + const decode_state_t st = decode_afpacket(af); + + if (st.complete) { + m_af_packet_completed(); + } + } + } + else { + const char packettype[3] = {(char)buf[0], (char)buf[1], '\0'}; + std::stringstream ss; + ss << "Unknown EDI packet "; + ss << packettype; + throw std::invalid_argument(ss.str()); + } +} + +void TagDispatcher::setMaxDelay(int num_af_packets) +{ + m_pft.setMaxDelay(num_af_packets); +} + + +#define AFPACKET_HEADER_LEN 10 // includes SYNC +decode_state_t TagDispatcher::decode_afpacket( + const std::vector &input_data) +{ + if (input_data.size() < AFPACKET_HEADER_LEN) { + return {false, 0}; + } + + // read length from packet + uint32_t taglength = read_32b(input_data.begin() + 2); + uint16_t seq = read_16b(input_data.begin() + 6); + + const size_t crclength = 2; + if (input_data.size() < AFPACKET_HEADER_LEN + taglength + crclength) { + return {false, 0}; + } + + if (m_last_seq + 1 != seq) { + etiLog.level(warn) << "EDI AF Packet sequence error, " << seq; + } + m_last_seq = seq; + + bool has_crc = (input_data[8] & 0x80) ? true : false; + uint8_t major_revision = (input_data[8] & 0x70) >> 4; + uint8_t minor_revision = input_data[8] & 0x0F; + if (major_revision != 1 or minor_revision != 0) { + throw invalid_argument("EDI AF Packet has wrong revision " + + to_string(major_revision) + "." + to_string(minor_revision)); + } + uint8_t pt = input_data[9]; + if (pt != 'T') { + // only support Tag + return {false, 0}; + } + + + if (not has_crc) { + throw invalid_argument("AF packet not supported, has no CRC"); + } + + uint16_t crc = 0xffff; + for (size_t i = 0; i < AFPACKET_HEADER_LEN + taglength; i++) { + crc = crc16(crc, &input_data[i], 1); + } + crc ^= 0xffff; + + uint16_t packet_crc = read_16b(input_data.begin() + AFPACKET_HEADER_LEN + taglength); + + if (packet_crc != crc) { + throw invalid_argument( + "AF Packet crc wrong"); + } + else { + vector payload(taglength); + copy(input_data.begin() + AFPACKET_HEADER_LEN, + input_data.begin() + AFPACKET_HEADER_LEN + taglength, + payload.begin()); + + return {decode_tagpacket(payload), + AFPACKET_HEADER_LEN + taglength + 2}; + } +} + +void TagDispatcher::register_tag(const std::string& tag, tag_handler&& h) +{ + m_handlers[tag] = move(h); +} + + +bool TagDispatcher::decode_tagpacket(const vector &payload) +{ + size_t length = 0; + + bool success = true; + + for (size_t i = 0; i + 8 < payload.size(); i += 8 + length) { + char tag_sz[5]; + tag_sz[4] = '\0'; + copy(payload.begin() + i, payload.begin() + i + 4, tag_sz); + + string tag(tag_sz); + + uint32_t taglength = read_32b(payload.begin() + i + 4); + + if (taglength % 8 != 0) { + etiLog.log(warn, "Invalid tag length!"); + break; + } + taglength /= 8; + + length = taglength; + + vector tag_value(taglength); + copy( payload.begin() + i+8, + payload.begin() + i+8+taglength, + tag_value.begin()); + + bool tagsuccess = false; + bool found = false; + for (auto tag_handler : m_handlers) { + if (tag_handler.first.size() == 4 and tag_handler.first == tag) { + found = true; + tagsuccess = tag_handler.second(tag_value, 0); + } + else if (tag_handler.first.size() == 3 and + tag.substr(0, 3) == tag_handler.first) { + found = true; + uint8_t n = tag_sz[3]; + tagsuccess = tag_handler.second(tag_value, n); + } + else if (tag_handler.first.size() == 2 and + tag.substr(0, 2) == tag_handler.first) { + found = true; + uint16_t n = 0; + n = (uint16_t)(tag_sz[2]) << 8; + n |= (uint16_t)(tag_sz[3]); + tagsuccess = tag_handler.second(tag_value, n); + } + } + + if (not found) { + etiLog.log(warn, "Ignoring unknown TAG %s", tag.c_str()); + break; + } + + if (not tagsuccess) { + etiLog.log(warn, "Error decoding TAG %s", tag.c_str()); + success = tagsuccess; + break; + } + } + + return success; +} + +} diff --git a/lib/edi/common.hpp b/lib/edi/common.hpp new file mode 100644 index 0000000..1433004 --- /dev/null +++ b/lib/edi/common.hpp @@ -0,0 +1,88 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://opendigitalradio.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "PFT.hpp" +#include +#include +#include +#include +#include +#include + +namespace EdiDecoder { + +struct frame_timestamp_t { + uint32_t seconds = 0; + uint32_t utco = 0; + + std::string to_string() const; + time_t to_unix_epoch() const; +}; + +struct decode_state_t { + decode_state_t(bool _complete, size_t _num_bytes_consumed) : + complete(_complete), num_bytes_consumed(_num_bytes_consumed) {} + bool complete; + size_t num_bytes_consumed; +}; + +/* The TagDispatcher takes care of decoding EDI, with or without PFT, and + * will call functions when TAGs are encountered. + * + * PF packets are handed over to the PFT decoder, which will in turn return + * AF packets. AF packets are directly dispatched to the TAG functions. + */ +class TagDispatcher { + public: + TagDispatcher(std::function&& af_packet_completed, bool verbose); + + /* Push bytes into the decoder. The buf can contain more + * than a single packet. This is useful when reading from streams + * (files, TCP) + */ + void push_bytes(const std::vector &buf); + + /* Push a complete packet into the decoder. Useful for UDP and other + * datagram-oriented protocols. + */ + void push_packet(const std::vector &buf); + + /* Set the maximum delay in number of AF Packets before we + * abandon decoding a given pseq. + */ + void setMaxDelay(int num_af_packets); + + using tag_handler = std::function, uint16_t)>; + void register_tag(const std::string& tag, tag_handler&& h); + + private: + decode_state_t decode_afpacket(const std::vector &input_data); + bool decode_tagpacket(const std::vector &payload); + + PFT::PFT m_pft; + uint16_t m_last_seq = 0; + std::vector m_input_data; + std::map m_handlers; + std::function m_af_packet_completed; +}; + +} diff --git a/src/input/Edi.cpp b/src/input/Edi.cpp new file mode 100644 index 0000000..8aee296 --- /dev/null +++ b/src/input/Edi.cpp @@ -0,0 +1,189 @@ +/* + Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications + Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#include "input/Edi.h" + +#include +#include +#include +#include +#include +#include +#include +#include "utils.h" + +using namespace std; + +namespace Inputs { + +constexpr bool VERBOSE = false; +constexpr size_t TCP_BLOCKSIZE = 2048; +constexpr size_t MAX_FRAMES_QUEUED = 10; + +Edi::Edi() : + m_tcp_receive_server(TCP_BLOCKSIZE), + m_sti_writer(), + m_sti_decoder(m_sti_writer, VERBOSE) +{ } + +Edi::~Edi() { + m_running = false; + if (m_thread.joinable()) { + m_thread.join(); + } +} + +int Edi::open(const std::string& name) +{ + const std::regex re_udp("udp://:([0-9]+)"); + const std::regex re_tcp("tcp://(.*):([0-9]+)"); + + lock_guard lock(m_mutex); + + m_running = false; + if (m_thread.joinable()) { + m_thread.join(); + } + + std::smatch m; + if (std::regex_match(name, m, re_udp)) { + const int udp_port = std::stoi(m[1].str()); + m_input_used = InputUsed::UDP; + m_udp_sock.reinit(udp_port); + m_udp_sock.setBlocking(false); + // TODO multicast + } + else if (std::regex_match(name, m, re_tcp)) { + m_input_used = InputUsed::TCP; + const string addr = m[1].str(); + const int tcp_port = std::stoi(m[2].str()); + m_tcp_receive_server.start(tcp_port, addr); + } + else { + throw runtime_error("Cannot parse EDI input URI"); + } + + m_name = name; + + m_running = true; + m_thread = std::thread(&Edi::m_run, this); + + return 0; +} + +int Edi::readFrame(uint8_t* buffer, size_t size) +{ + vector frame; + if (m_is_prebuffering) { + m_is_prebuffering = m_frames.size() < 10; + if (not m_is_prebuffering) { + etiLog.level(info) << "EDI input " << m_name << " pre-buffering complete."; + } + } + else if (m_frames.try_pop(frame)) { + if (frame.size() == 0) { + etiLog.level(debug) << "EDI input " << m_name << " empty frame"; + return 0; + } + else if (frame.size() == size) { + std::copy(frame.cbegin(), frame.cend(), buffer); + } + else { + etiLog.level(debug) << "EDI input " << m_name << " size mismatch: " << frame.size() << + " received, " << size << " requested"; + memset(buffer, 0, size * sizeof(*buffer)); + } + } + else { + memset(buffer, 0, size * sizeof(*buffer)); + m_is_prebuffering = true; + etiLog.level(info) << "EDI input " << m_name << " re-enabling pre-buffering"; + } + return size; +} + +void Edi::m_run() +{ + while (m_running) { + bool work_done = false; + + switch (m_input_used) { + case InputUsed::UDP: + { + constexpr size_t packsize = 2048; + const auto packet = m_udp_sock.receive(packsize); + if (packet.buffer.size() == packsize) { + fprintf(stderr, "Warning, possible UDP truncation\n"); + } + if (not packet.buffer.empty()) { + m_sti_decoder.push_packet(packet.buffer); + work_done = true; + } + } + break; + case InputUsed::TCP: + { + auto packet = m_tcp_receive_server.receive(); + if (not packet.empty()) { + m_sti_decoder.push_bytes(packet); + work_done = true; + } + } + break; + default: + throw logic_error("unimplemented input"); + } + + const auto sti = m_sti_writer.getFrame(); + if (not sti.frame.empty()) { + m_frames.push_wait_if_full(move(sti.frame), MAX_FRAMES_QUEUED); + work_done = true; + } + + if (not work_done) { + // Avoid fast loop + this_thread::sleep_for(chrono::milliseconds(12)); + } + } +} + +int Edi::setBitrate(int bitrate) +{ + if (bitrate <= 0) { + etiLog.level(error) << "Invalid bitrate (" << bitrate << ") for " << m_name; + return -1; + } + + return bitrate; +} + +int Edi::close() +{ + m_udp_sock.close(); + return 0; +} + +} diff --git a/src/input/Edi.h b/src/input/Edi.h new file mode 100644 index 0000000..7b3dc04 --- /dev/null +++ b/src/input/Edi.h @@ -0,0 +1,82 @@ +/* + Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications + Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include "Socket.h" +#include "input/inputs.h" +#include "edi/STIDecoder.hpp" +#include "edi/STIWriter.hpp" +#include "ThreadsafeQueue.h" + +namespace Inputs { + +/* + * Receives EDI from UDP or TCP in a separate thread and pushes that data + * into the STIDecoder. Complete frames are then put into a queue for the consumer. + * + * This way, the EDI decoding happens in a separate thread. + */ +class Edi : public InputBase { + public: + Edi(); + Edi(const Edi&) = delete; + Edi& operator=(const Edi&) = delete; + ~Edi(); + + virtual int open(const std::string& name); + virtual int readFrame(uint8_t* buffer, size_t size); + virtual int setBitrate(int bitrate); + virtual int close(); + + protected: + void m_run(); + + std::mutex m_mutex; + + enum class InputUsed { Invalid, UDP, TCP }; + InputUsed m_input_used = InputUsed::Invalid; + Socket::UDPSocket m_udp_sock; + Socket::TCPReceiveServer m_tcp_receive_server; + + EdiDecoder::STIWriter m_sti_writer; + EdiDecoder::STIDecoder m_sti_decoder; + std::thread m_thread; + std::atomic m_running = ATOMIC_VAR_INIT(false); + ThreadsafeQueue > m_frames; + + bool m_is_prebuffering = true; + + std::string m_name; +}; + +}; + -- cgit v1.2.3 From 14f69f9c915cf644147a52b803d79ff8f40a4ea1 Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Mon, 29 Jul 2019 15:27:32 +0200 Subject: First prototype taking EDI TIST into account for contribution --- lib/edi/STIWriter.cpp | 1 + lib/edi/common.cpp | 25 ++++++++++++++++++++++++- lib/edi/common.hpp | 4 ++++ src/input/Edi.cpp | 48 ++++++++++++++++++++++++++++++++++++++++-------- src/input/Edi.h | 3 ++- src/utils.cpp | 2 +- 6 files changed, 72 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/edi/STIWriter.cpp b/lib/edi/STIWriter.cpp index 6964eb1..399922a 100644 --- a/lib/edi/STIWriter.cpp +++ b/lib/edi/STIWriter.cpp @@ -120,6 +120,7 @@ void STIWriter::assemble() m_stiFrame.frame = m_payload.istd; m_stiFrame.timestamp.seconds = m_seconds; m_stiFrame.timestamp.utco = m_utco; + m_stiFrame.timestamp.tsta = m_management_data.tsta; } sti_frame_t STIWriter::getFrame() diff --git a/lib/edi/common.cpp b/lib/edi/common.cpp index bc0fa1b..b4b0c79 100644 --- a/lib/edi/common.cpp +++ b/lib/edi/common.cpp @@ -25,18 +25,31 @@ #include #include #include +#include #include namespace EdiDecoder { using namespace std; +bool frame_timestamp_t::valid() const +{ + return tsta != 0xFFFFFF; +} + string frame_timestamp_t::to_string() const { const time_t seconds_in_unix_epoch = to_unix_epoch(); stringstream ss; - ss << "Timestamp: " << std::put_time(std::gmtime(&seconds_in_unix_epoch), "%c %Z"); + if (valid()) { + ss << "Timestamp: "; + } + else { + ss << "Timestamp not valid: "; + } + ss << std::put_time(std::gmtime(&seconds_in_unix_epoch), "%c %Z") << + " + " << ((double)tsta / 16384000.0); return ss.str(); } @@ -48,6 +61,16 @@ time_t frame_timestamp_t::to_unix_epoch() const return 946684800 + seconds - utco; } +std::chrono::system_clock::time_point frame_timestamp_t::to_system_clock() const +{ + auto ts = chrono::system_clock::from_time_t(to_unix_epoch()); + + // PPS offset in seconds = tsta / 16384000 + ts += chrono::nanoseconds(std::lrint(tsta / 0.016384)); + + return ts; +} + TagDispatcher::TagDispatcher( std::function&& af_packet_completed, bool verbose) : diff --git a/lib/edi/common.hpp b/lib/edi/common.hpp index 1433004..887bc3d 100644 --- a/lib/edi/common.hpp +++ b/lib/edi/common.hpp @@ -23,6 +23,7 @@ #include "PFT.hpp" #include #include +#include #include #include #include @@ -33,9 +34,12 @@ namespace EdiDecoder { struct frame_timestamp_t { uint32_t seconds = 0; uint32_t utco = 0; + uint32_t tsta = 0; // According to EN 300 797 Annex B + bool valid() const; std::string to_string() const; time_t to_unix_epoch() const; + std::chrono::system_clock::time_point to_system_clock() const; }; struct decode_state_t { diff --git a/src/input/Edi.cpp b/src/input/Edi.cpp index 8aee296..765a355 100644 --- a/src/input/Edi.cpp +++ b/src/input/Edi.cpp @@ -27,6 +27,7 @@ #include "input/Edi.h" #include +#include #include #include #include @@ -41,7 +42,7 @@ namespace Inputs { constexpr bool VERBOSE = false; constexpr size_t TCP_BLOCKSIZE = 2048; -constexpr size_t MAX_FRAMES_QUEUED = 10; +constexpr size_t MAX_FRAMES_QUEUED = 1000; Edi::Edi() : m_tcp_receive_server(TCP_BLOCKSIZE), @@ -96,23 +97,53 @@ int Edi::open(const std::string& name) int Edi::readFrame(uint8_t* buffer, size_t size) { - vector frame; + if (m_pending_sti_frame.frame.empty()) { + m_frames.try_pop(m_pending_sti_frame); + } + + if (not m_pending_sti_frame.frame.empty()) { + if (m_pending_sti_frame.frame.size() != size) { + etiLog.level(debug) << "EDI input " << m_name << " size mismatch: " << + m_pending_sti_frame.frame.size() << " received, " << size << " requested"; + memset(buffer, 0, size * sizeof(*buffer)); + } + else { + const auto now = chrono::system_clock::now(); + + if (m_pending_sti_frame.timestamp.to_system_clock() <= now) { + etiLog.level(debug) << "EDI input take frame with TS " << + m_pending_sti_frame.timestamp.to_string() << " queue size " << m_frames.size(); + + std::copy(m_pending_sti_frame.frame.cbegin(), m_pending_sti_frame.frame.cend(), buffer); + m_pending_sti_frame.frame.clear(); + } + else { + etiLog.level(debug) << "EDI input skip frame with TS " << + m_pending_sti_frame.timestamp.to_string() << " queue size " << m_frames.size(); + } + } + } + + return size; + +#if 0 + EdiDecoder::sti_frame_t sti; if (m_is_prebuffering) { m_is_prebuffering = m_frames.size() < 10; if (not m_is_prebuffering) { etiLog.level(info) << "EDI input " << m_name << " pre-buffering complete."; } } - else if (m_frames.try_pop(frame)) { - if (frame.size() == 0) { + else if (m_frames.try_pop(sti)) { + if (sti.frame.size() == 0) { etiLog.level(debug) << "EDI input " << m_name << " empty frame"; return 0; } - else if (frame.size() == size) { - std::copy(frame.cbegin(), frame.cend(), buffer); + else if (sti.frame.size() == size) { + std::copy(sti.frame.cbegin(), sti.frame.cend(), buffer); } else { - etiLog.level(debug) << "EDI input " << m_name << " size mismatch: " << frame.size() << + etiLog.level(debug) << "EDI input " << m_name << " size mismatch: " << sti.frame.size() << " received, " << size << " requested"; memset(buffer, 0, size * sizeof(*buffer)); } @@ -123,6 +154,7 @@ int Edi::readFrame(uint8_t* buffer, size_t size) etiLog.level(info) << "EDI input " << m_name << " re-enabling pre-buffering"; } return size; +#endif } void Edi::m_run() @@ -159,7 +191,7 @@ void Edi::m_run() const auto sti = m_sti_writer.getFrame(); if (not sti.frame.empty()) { - m_frames.push_wait_if_full(move(sti.frame), MAX_FRAMES_QUEUED); + m_frames.push_wait_if_full(move(sti), MAX_FRAMES_QUEUED); work_done = true; } diff --git a/src/input/Edi.h b/src/input/Edi.h index 7b3dc04..66ff682 100644 --- a/src/input/Edi.h +++ b/src/input/Edi.h @@ -71,7 +71,8 @@ class Edi : public InputBase { EdiDecoder::STIDecoder m_sti_decoder; std::thread m_thread; std::atomic m_running = ATOMIC_VAR_INIT(false); - ThreadsafeQueue > m_frames; + ThreadsafeQueue m_frames; + EdiDecoder::sti_frame_t m_pending_sti_frame; bool m_is_prebuffering = true; diff --git a/src/utils.cpp b/src/utils.cpp index 721c145..3e3e86e 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -328,7 +328,7 @@ void printSubchannels(const vec_sp_subchannel& subchannels) etiLog.level(info) << " URI: " << subchannel->inputUri; switch (subchannel->type) { case subchannel_type_t::DABAudio: - etiLog.log(info, " type: DAbAudio"); + etiLog.log(info, " type: DABAudio"); break; case subchannel_type_t::DABPlusAudio: etiLog.log(info, " type: DABPlusAudio"); -- cgit v1.2.3 From ef536ba0e190df286d273ded5ba93f4a2f39b45e Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Tue, 13 Aug 2019 10:48:49 +0200 Subject: Pull in files from odr-mmbtools-common Telnet RC is now single-user only. --- Makefile.am | 28 +-- configure.ac | 2 +- lib/ClockTAI.cpp | 562 +++++++++++++++++++++++++++++++++++++++++++++++ lib/ClockTAI.h | 102 +++++++++ lib/Log.cpp | 194 ++++++++++++++++ lib/Log.h | 204 +++++++++++++++++ lib/ReedSolomon.cpp | 4 +- lib/RemoteControl.cpp | 581 ++++++++++++++++++++++++++++++++++++++++++++++++ lib/RemoteControl.h | 249 +++++++++++++++++++++ lib/crc.c | 266 ++++++++++++++++++++++ lib/crc.h | 59 +++++ src/ClockTAI.cpp | 562 ----------------------------------------------- src/ClockTAI.h | 102 --------- src/Log.cpp | 143 ------------ src/Log.h | 157 ------------- src/RemoteControl.cpp | 595 -------------------------------------------------- src/RemoteControl.h | 263 ---------------------- src/crc.c | 266 ---------------------- src/crc.h | 59 ----- 19 files changed, 2235 insertions(+), 2163 deletions(-) create mode 100644 lib/ClockTAI.cpp create mode 100644 lib/ClockTAI.h create mode 100644 lib/Log.cpp create mode 100644 lib/Log.h create mode 100644 lib/RemoteControl.cpp create mode 100644 lib/RemoteControl.h create mode 100644 lib/crc.c create mode 100644 lib/crc.h delete mode 100644 src/ClockTAI.cpp delete mode 100644 src/ClockTAI.h delete mode 100644 src/Log.cpp delete mode 100644 src/Log.h delete mode 100644 src/RemoteControl.cpp delete mode 100644 src/RemoteControl.h delete mode 100644 src/crc.c delete mode 100644 src/crc.h (limited to 'lib') diff --git a/Makefile.am b/Makefile.am index 6e5aa71..216f7c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -94,25 +94,17 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/dabOutput/edi/TagPacket.h \ src/dabOutput/edi/Transport.cpp \ src/dabOutput/edi/Transport.h \ - src/ClockTAI.h \ - src/ClockTAI.cpp \ src/ConfigParser.cpp \ src/ConfigParser.h \ src/Eti.h \ src/Eti.cpp \ src/Interleaver.h \ src/Interleaver.cpp \ - src/Log.h \ - src/Log.cpp \ src/ManagementServer.h \ src/ManagementServer.cpp \ src/MuxElements.cpp \ src/MuxElements.h \ src/PcDebug.h \ - src/RemoteControl.cpp \ - src/RemoteControl.h \ - src/crc.h \ - src/crc.c \ src/fig/FIG.h \ src/fig/FIG.cpp \ src/fig/FIG0.h \ @@ -160,6 +152,14 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/PrbsGenerator.h \ src/utils.cpp \ src/utils.h \ + lib/crc.h \ + lib/crc.c \ + lib/ClockTAI.h \ + lib/ClockTAI.cpp \ + lib/Log.h \ + lib/Log.cpp \ + lib/RemoteControl.cpp \ + lib/RemoteControl.h \ lib/edi/STIDecoder.cpp \ lib/edi/STIDecoder.h \ lib/edi/STIWriter.cpp \ @@ -185,8 +185,8 @@ zmqinput_keygen_CFLAGS = -Wall $(GITVERSION_FLAGS) $(ZMQ_CPPFLAGS) odr_zmq2farsync_SOURCES = src/zmq2farsync/zmq2farsync.cpp \ src/dabOutput/dabOutput.h \ src/dabOutput/dabOutputRaw.cpp \ - src/Log.h \ - src/Log.cpp \ + lib/Log.h \ + lib/Log.cpp \ lib/zmq.hpp odr_zmq2farsync_LDADD = $(ZMQ_LIBS) @@ -212,10 +212,10 @@ odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ src/dabOutput/edi/TagPacket.h \ src/dabOutput/edi/Transport.cpp \ src/dabOutput/edi/Transport.h \ - src/Log.h \ - src/Log.cpp \ - src/crc.h \ - src/crc.c \ + lib/Log.h \ + lib/Log.cpp \ + lib/crc.h \ + lib/crc.c \ lib/ReedSolomon.h \ lib/ReedSolomon.cpp \ lib/Socket.h \ diff --git a/configure.ac b/configure.ac index 50623a2..405b6f0 100644 --- a/configure.ac +++ b/configure.ac @@ -128,7 +128,7 @@ AX_ZMQ([4.0.0], [], AC_MSG_ERROR(ZeroMQ 4.0.0 is required)) AC_DEFINE([HAVE_INPUT_ZEROMQ], [1], [Define if ZeroMQ input is enabled]) AC_DEFINE([HAVE_OUTPUT_ZEROMQ], [1], [Define if ZeroMQ output is enabled]) -AC_DEFINE([HAVE_RC_ZEROMQ], [1], [Define if ZeroMQ enabled for rc]) +AC_DEFINE([HAVE_ZEROMQ], [1], [Define if ZeroMQ enabled for rc]) # Do not build odr-zmq2farsync if no RAW output AM_CONDITIONAL([HAVE_OUTPUT_RAW_TEST], diff --git a/lib/ClockTAI.cpp b/lib/ClockTAI.cpp new file mode 100644 index 0000000..42497f4 --- /dev/null +++ b/lib/ClockTAI.cpp @@ -0,0 +1,562 @@ +/* + Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012 Her Majesty the Queen in Right of Canada (Communications + Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . +*/ + +/* This file downloads the TAI-UTC bulletins from the from IETF and parses them + * so that correct time can be communicated in EDI timestamps. + * + * This file contains self-test code that can be executed by running + * g++ -g -Wall -DTEST -DHAVE_CURL -std=c++11 -lcurl -pthread \ + * ClockTAI.cpp Log.cpp RemoteControl.cpp -lboost_system -o taitest && ./taitest + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "ClockTAI.h" +#include "Log.h" + +#include +#include +#include +#if SUPPORT_SETTING_CLOCK_TAI +# include +#endif +#ifdef HAVE_CURL +# include +#endif +#include +#include +#include +#include +#include + +using namespace std; + +#ifdef TEST +static bool wait_longer = true; +#endif + +constexpr int download_retry_interval_hours = 1; + +// Offset between NTP time and POSIX time: +// timestamp_unix = timestamp_ntp - ntp_unix_offset +const int64_t ntp_unix_offset = 2208988800L; + +// leap seconds insertion bulletin is available from the IETF and in the TZ +// distribution +static array default_tai_urls = { + "https://www.ietf.org/timezones/data/leap-seconds.list", + "https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list", +}; + +// According to the Filesystem Hierarchy Standard, the data in +// /var/tmp "must not be deleted when the system is booted." +static const char *tai_cache_location = "/var/tmp/odr-dabmux-leap-seconds.cache"; + +// read TAI offset from a valid bulletin in IETF format +static int parse_ietf_bulletin(const std::string& bulletin) +{ + // Example Line: + // 3692217600 37 # 1 Jan 2017 + // + // NTP timestampleap seconds# some comment + // The NTP timestamp starts at epoch 1.1.1900. + // The difference between NTP timestamps and unix epoch is 70 + // years i.e. 2208988800 seconds + + std::regex regex_bulletin(R"(([0-9]+)\s+([0-9]+)\s+#.*)"); + + time_t now = time(nullptr); + + int tai_utc_offset = 0; + + int tai_utc_offset_valid = false; + + stringstream ss(bulletin); + + /* We cannot just take the last line, because it might + * be in the future, announcing an upcoming leap second. + * + * So we need to look at the current date, and compare it + * with the date of the leap second. + */ + for (string line; getline(ss, line); ) { + + std::smatch bulletin_entry; + + bool is_match = std::regex_search(line, bulletin_entry, regex_bulletin); + if (is_match) { + if (bulletin_entry.size() != 3) { + throw runtime_error( + "Incorrect number of matched TAI IETF bulletin entries"); + } + const string bulletin_ntp_timestamp(bulletin_entry[1]); + const string bulletin_offset(bulletin_entry[2]); + + const int64_t timestamp_unix = + std::atoll(bulletin_ntp_timestamp.c_str()) - ntp_unix_offset; + + const int offset = std::atoi(bulletin_offset.c_str()); + // Ignore entries announcing leap seconds in the future + if (timestamp_unix < now) { + tai_utc_offset = offset; + tai_utc_offset_valid = true; + } +#if TEST + else { + cerr << "IETF Ignoring offset " << bulletin_offset << + " at TS " << bulletin_ntp_timestamp << + " in the future" << endl; + } +#endif + } + } + + if (not tai_utc_offset_valid) { + throw runtime_error("No data in TAI bulletin"); + } + + return tai_utc_offset; +} + + +struct bulletin_state { + bool valid = false; + int64_t expiry = 0; + int offset = 0; + + bool usable() const { return valid and expiry > 0; } +}; + +static bulletin_state parse_bulletin(const string& bulletin) +{ + // The bulletin contains one line that specifies an expiration date + // in NTP time. If that point in time is in the future, we consider + // the bulletin valid. + // + // The entry looks like this: + //#@ 3707596800 + + bulletin_state ret; + + std::regex regex_expiration(R"(#@\s+([0-9]+))"); + + time_t now = time(nullptr); + + stringstream ss(bulletin); + + for (string line; getline(ss, line); ) { + std::smatch bulletin_entry; + + bool is_match = std::regex_search(line, bulletin_entry, regex_expiration); + if (is_match) { + if (bulletin_entry.size() != 2) { + throw runtime_error( + "Incorrect number of matched TAI IETF bulletin expiration"); + } + const string expiry_data_str(bulletin_entry[1]); + const int64_t expiry_unix = + std::atoll(expiry_data_str.c_str()) - ntp_unix_offset; + +#ifdef TEST + etiLog.level(info) << "Bulletin expires in " << expiry_unix - now; +#endif + ret.expiry = expiry_unix - now; + try { + ret.offset = parse_ietf_bulletin(bulletin); + ret.valid = true; + } + catch (const runtime_error& e) { + etiLog.level(warn) << "Bulletin expiry ok but parse error: " << e.what(); + } + break; + } + } + return ret; +} + + +// callback that receives data from cURL +static size_t fill_bulletin(char *ptr, size_t size, size_t nmemb, void *ctx) +{ + auto *bulletin = reinterpret_cast(ctx); + + size_t len = size * nmemb; + for (size_t i = 0; i < len; i++) { + *bulletin << ptr[i]; + } + return len; +} + +static string download_tai_utc_bulletin(const char* url) +{ + stringstream bulletin; + +#ifdef HAVE_CURL + CURL *curl; + CURLcode res; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url); + /* Tell libcurl to follow redirection */ + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_bulletin); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &bulletin); + + res = curl_easy_perform(curl); + /* always cleanup ! */ + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + throw runtime_error( "TAI-UTC bulletin download failed: " + + string(curl_easy_strerror(res))); + } + } + return bulletin.str(); +#else + throw runtime_error("Cannot download TAI Clock information without cURL"); +#endif // HAVE_CURL +} + +static string load_bulletin_from_file(const char* cache_filename) +{ + // Clear the bulletin + ifstream f(cache_filename); + if (not f.good()) { + return {}; + } + + stringstream ss; + ss << f.rdbuf(); + f.close(); + + return ss.str(); +} + +ClockTAI::ClockTAI(const std::vector& bulletin_urls) : + RemoteControllable("clocktai") +{ + RC_ADD_PARAMETER(expiry, "Number of seconds until TAI Bulletin expires"); + + if (bulletin_urls.empty()) { + etiLog.level(debug) << "Initialising default TAI Bulletin URLs"; + for (const auto url : default_tai_urls) { + m_bulletin_urls.push_back(url); + } + } + else { + etiLog.level(debug) << "Initialising user-configured TAI Bulletin URLs"; + m_bulletin_urls = bulletin_urls; + } + + for (const auto url : m_bulletin_urls) { + etiLog.level(info) << "TAI Bulletin URL: '" << url << "'"; + } +} + +int ClockTAI::get_valid_offset() +{ + int offset = 0; + bool offset_valid = false; + + std::unique_lock lock(m_data_mutex); + + const auto state = parse_bulletin(m_bulletin); + if (state.usable()) { +#if TEST + etiLog.level(info) << "Bulletin already valid"; +#endif + offset = state.offset; + offset_valid = true; + } + else { + const auto cache_bulletin = load_bulletin_from_file(tai_cache_location); + const auto cache_state = parse_bulletin(cache_bulletin); + + if (cache_state.usable()) { + m_bulletin = cache_bulletin; + offset = cache_state.offset; + offset_valid = true; +#if TEST + etiLog.level(info) << "Bulletin from cache valid with offset=" << offset; +#endif + } + else { + for (const auto url : m_bulletin_urls) { + try { +#if TEST + etiLog.level(info) << "Load bulletin from " << url; +#endif + const auto new_bulletin = download_tai_utc_bulletin(url.c_str()); + const auto new_state = parse_bulletin(new_bulletin); + if (new_state.usable()) { + m_bulletin = new_bulletin; + offset = new_state.offset; + offset_valid = true; + + etiLog.level(debug) << "Loaded valid TAI Bulletin from " << + url << " giving offset=" << offset; + } + else { + etiLog.level(debug) << "Skipping invalid TAI bulletin from " + << url; + } + } + catch (const runtime_error& e) { + etiLog.level(warn) << + "TAI-UTC offset could not be retrieved from " << + url << " : " << e.what(); + } + + if (offset_valid) { + update_cache(tai_cache_location); + break; + } + } + } + } + + if (offset_valid) { + // With the current evolution of the offset, we're probably going + // to reach 500 long after DAB gets replaced by another standard. + if (offset < 0 or offset > 500) { + stringstream ss; + ss << "TAI offset " << offset << " out of range"; + throw range_error(ss.str()); + } + + return offset; + } + else { + // Try again later + throw download_failed(); + } +} + + +int ClockTAI::get_offset() +{ + using namespace std::chrono; + const auto time_now = system_clock::now(); + + std::unique_lock lock(m_data_mutex); + + if (not m_offset_valid) { +#ifdef TEST + // Assume we've downloaded it in the past: + + m_offset = 37; // Valid in early 2017 + m_offset_valid = true; + + // Simulate requiring a new download + m_bulletin_download_time = time_now - hours(24 * 40); +#else + // First time we run we must block until we know + // the offset + lock.unlock(); + try { + m_offset = get_valid_offset(); + } + catch (const download_failed&) { + throw runtime_error("Unable to download TAI bulletin"); + } + lock.lock(); + m_offset_valid = true; + m_bulletin_download_time = time_now; +#endif + etiLog.level(info) << + "Initialised TAI-UTC offset to " << m_offset << "s."; + } + + if (time_now - m_bulletin_download_time > hours(24 * 31)) { + // Refresh if it's older than one month. Leap seconds are + // announced several months in advance + etiLog.level(debug) << "Trying to refresh TAI bulletin"; + + if (m_offset_future.valid()) { + auto state = m_offset_future.wait_for(seconds(0)); + switch (state) { + case future_status::ready: + try { + m_offset = m_offset_future.get(); + m_offset_valid = true; + m_bulletin_download_time = time_now; + + etiLog.level(info) << + "Updated TAI-UTC offset to " << m_offset << "s."; + } + catch (const download_failed&) { + etiLog.level(warn) << + "TAI-UTC download failed, will retry in " << + download_retry_interval_hours << " hour(s)"; + + m_bulletin_download_time += hours(download_retry_interval_hours); + } +#ifdef TEST + wait_longer = false; +#endif + break; + + case future_status::deferred: + case future_status::timeout: + // Not ready yet +#ifdef TEST + etiLog.level(debug) << " async not ready yet"; +#endif + break; + } + } + else { +#ifdef TEST + etiLog.level(debug) << " Launch async"; +#endif + m_offset_future = async(launch::async, &ClockTAI::get_valid_offset, this); + } + } + + return m_offset; +} + +#if SUPPORT_SETTING_CLOCK_TAI +int ClockTAI::update_local_tai_clock(int offset) +{ + struct timex timex_request; + timex_request.modes = ADJ_TAI; + timex_request.constant = offset; + + int err = adjtimex(&timex_request); + if (err == -1) { + perror("adjtimex"); + } + + printf("adjtimex: %d, tai %d\n", err, timex_request.tai); + + return err; +} +#endif + +void ClockTAI::update_cache(const char* cache_filename) +{ + ofstream f(cache_filename); + if (not f.good()) { + throw runtime_error("TAI-UTC bulletin open cache for writing"); + } + + f << m_bulletin; + f.close(); +} + + +void ClockTAI::set_parameter(const string& parameter, const string& value) +{ + if (parameter == "expiry") { + throw ParameterError("Parameter '" + parameter + + "' is read-only in controllable " + get_rc_name()); + } + else { + throw ParameterError("Parameter '" + parameter + + "' is not exported by controllable " + get_rc_name()); + } +} + +const string ClockTAI::get_parameter(const string& parameter) const +{ + if (parameter == "expiry") { + std::unique_lock lock(m_data_mutex); + const int64_t expiry = parse_bulletin(m_bulletin).expiry; + if (expiry > 0) { + return to_string(expiry); + } + else { + return "Bulletin expired or invalid!"; + } + } + else { + throw ParameterError("Parameter '" + parameter + + "' is not exported by controllable " + get_rc_name()); + } +} + +#if 0 +// Example testing code +void debug_tai_clk() +{ + struct timespec rt_clk; + + int err = clock_gettime(CLOCK_REALTIME, &rt_clk); + if (err) { + perror("REALTIME clock_gettime failed"); + } + + struct timespec tai_clk; + + err = clock_gettime(CLOCK_TAI, &tai_clk); + if (err) { + perror("TAI clock_gettime failed"); + } + + printf("RT - TAI = %ld\n", rt_clk.tv_sec - tai_clk.tv_sec); + + + struct timex timex_request; + timex_request.modes = 0; // Do not set anything + + err = adjtimex(&timex_request); + if (err == -1) { + perror("adjtimex"); + } + + printf("adjtimex: %d, tai %d\n", err, timex_request.tai); +} +#endif + +#if TEST +int main(int argc, char **argv) +{ + using namespace std; + + ClockTAI tai({}); + + while (wait_longer) { + try { + etiLog.level(info) << + "Offset is " << tai.get_offset(); + } + catch (const exception &e) { + etiLog.level(error) << + "Exception " << e.what(); + } + + this_thread::sleep_for(chrono::seconds(2)); + } + + return 0; +} +#endif + diff --git a/lib/ClockTAI.h b/lib/ClockTAI.h new file mode 100644 index 0000000..bb85815 --- /dev/null +++ b/lib/ClockTAI.h @@ -0,0 +1,102 @@ +/* + Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012 Her Majesty the Queen in Right of Canada (Communications + Research Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . +*/ + +/* The EDI output needs TAI clock, according to ETSI TS 102 693 Annex F + * "EDI Timestamps". This module can set the local CLOCK_TAI clock by + * setting the TAI-UTC offset using adjtimex. + * + * This functionality requires Linux 3.10 (30 Jun 2013) or newer. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "RemoteControl.h" + +// EDI needs to know UTC-TAI, but doesn't need the CLOCK_TAI to be set. +// We can keep this code, maybe for future use +#define SUPPORT_SETTING_CLOCK_TAI 0 + +/* Loads, parses and represents TAI-UTC offset information from the IETF bulletin */ +class ClockTAI : public RemoteControllable { + public: + ClockTAI(const std::vector& bulletin_urls); + + // Fetch the bulletin from the IETF website and return the current + // TAI-UTC offset. + // Throws runtime_error on failure. + int get_offset(void); + +#if SUPPORT_SETTING_CLOCK_TAI + // Update the local TAI clock according to the TAI-UTC offset + // return 0 on success + int update_local_tai_clock(int offset); +#endif + + private: + class download_failed {}; + + // Either retrieve the bulletin from the cache or if necessarly + // download it, and calculate the TAI-UTC offset. + // Returns the offset or throws download_failed or a range_error + // if the offset is out of bounds. + int get_valid_offset(void); + + // Download of new bulletin is done asynchronously + std::future m_offset_future; + + // Protect all data members, as RC functions are in another thread + mutable std::mutex m_data_mutex; + + // The currently used TAI-UTC offset + int m_offset = 0; + int m_offset_valid = false; + + std::vector m_bulletin_urls; + + std::string m_bulletin; + std::chrono::system_clock::time_point m_bulletin_download_time; + + // Update the cache file with the current m_bulletin + void update_cache(const char* cache_filename); + + + /* Remote control */ + virtual void set_parameter(const std::string& parameter, + const std::string& value); + + /* Getting a parameter always returns a string. */ + virtual const std::string get_parameter(const std::string& parameter) const; +}; + diff --git a/lib/Log.cpp b/lib/Log.cpp new file mode 100644 index 0000000..2417f3a --- /dev/null +++ b/lib/Log.cpp @@ -0,0 +1,194 @@ +/* + Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "Log.h" + +using namespace std; + +/* etiLog is a singleton used in all parts of the program to output log messages. + */ +Logger etiLog; + +void Logger::register_backend(std::shared_ptr backend) +{ + backends.push_back(backend); +} + + +void Logger::log(log_level_t level, const char* fmt, ...) +{ + if (level == discard) { + return; + } + + int size = 100; + std::string str; + va_list ap; + while (1) { + str.resize(size); + va_start(ap, fmt); + int n = vsnprintf((char *)str.c_str(), size, fmt, ap); + va_end(ap); + if (n > -1 && n < size) { + str.resize(n); + break; + } + if (n > -1) + size = n + 1; + else + size *= 2; + } + + logstr(level, move(str)); +} + +void Logger::logstr(log_level_t level, std::string&& message) +{ + if (level == discard) { + return; + } + + log_message_t m(level, move(message)); + m_message_queue.push(move(m)); +} + +void Logger::io_process() +{ + while (1) { + log_message_t m; + try { + m_message_queue.wait_and_pop(m); + } + catch (const ThreadsafeQueueWakeup&) { + break; + } + + auto message = m.message; + + /* Remove a potential trailing newline. + * It doesn't look good in syslog + */ + if (message[message.length()-1] == '\n') { + message.resize(message.length()-1); + } + + for (auto &backend : backends) { + backend->log(m.level, message); + } + + if (m.level != log_level_t::trace) { + std::lock_guard guard(m_cerr_mutex); + std::cerr << levels_as_str[m.level] << " " << message << std::endl; + } + } +} + + +LogLine Logger::level(log_level_t level) +{ + return LogLine(this, level); +} + +LogToFile::LogToFile(const std::string& filename) : name("FILE") +{ + FILE* fd = fopen(filename.c_str(), "a"); + if (fd == nullptr) { + fprintf(stderr, "Cannot open log file !"); + throw std::runtime_error("Cannot open log file !"); + } + + log_file.reset(fd); +} + +void LogToFile::log(log_level_t level, const std::string& message) +{ + if (not (level == log_level_t::trace or level == log_level_t::discard)) { + const char* log_level_text[] = { + "DEBUG", "INFO", "WARN", "ERROR", "ALERT", "EMERG"}; + + // fprintf is thread-safe + fprintf(log_file.get(), SYSLOG_IDENT ": %s: %s\n", + log_level_text[(size_t)level], message.c_str()); + fflush(log_file.get()); + } +} + +void LogToSyslog::log(log_level_t level, const std::string& message) +{ + if (not (level == log_level_t::trace or level == log_level_t::discard)) { + int syslog_level = LOG_EMERG; + switch (level) { + case debug: syslog_level = LOG_DEBUG; break; + case info: syslog_level = LOG_INFO; break; + /* we don't have the notice level */ + case warn: syslog_level = LOG_WARNING; break; + case error: syslog_level = LOG_ERR; break; + default: syslog_level = LOG_CRIT; break; + case alert: syslog_level = LOG_ALERT; break; + case emerg: syslog_level = LOG_EMERG; break; + } + + syslog(syslog_level, SYSLOG_IDENT " %s", message.c_str()); + } +} + +LogTracer::LogTracer(const string& trace_filename) : name("TRACE") +{ + etiLog.level(info) << "Setting up TRACE to " << trace_filename; + + FILE* fd = fopen(trace_filename.c_str(), "a"); + if (fd == nullptr) { + fprintf(stderr, "Cannot open trace file !"); + throw std::runtime_error("Cannot open trace file !"); + } + m_trace_file.reset(fd); + + using namespace std::chrono; + auto now = steady_clock::now().time_since_epoch(); + m_trace_micros_startup = duration_cast(now).count(); + + fprintf(m_trace_file.get(), + "0,TRACER,startup at %" PRIu64 "\n", m_trace_micros_startup); +} + +void LogTracer::log(log_level_t level, const std::string& message) +{ + if (level == log_level_t::trace) { + using namespace std::chrono; + const auto now = steady_clock::now().time_since_epoch(); + const auto micros = duration_cast(now).count(); + + fprintf(m_trace_file.get(), "%" PRIu64 ",%s\n", + micros - m_trace_micros_startup, + message.c_str()); + } +} diff --git a/lib/Log.h b/lib/Log.h new file mode 100644 index 0000000..d5c39e0 --- /dev/null +++ b/lib/Log.h @@ -0,0 +1,204 @@ +/* + Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2018 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ThreadsafeQueue.h" + +#define SYSLOG_IDENT PACKAGE_NAME +#define SYSLOG_FACILITY LOG_LOCAL0 + +enum log_level_t {debug = 0, info, warn, error, alert, emerg, trace, discard}; + +static const std::string levels_as_str[] = + { " ", " ", "WARN ", "ERROR", "ALERT", "EMERG", "TRACE", "-----"} ; + +/** Abstract class all backends must inherit from */ +class LogBackend { + public: + virtual ~LogBackend() {}; + virtual void log(log_level_t level, const std::string& message) = 0; + virtual std::string get_name() const = 0; +}; + +/** A Logging backend for Syslog */ +class LogToSyslog : public LogBackend { + public: + LogToSyslog() : name("SYSLOG") { + openlog(SYSLOG_IDENT, LOG_PID, SYSLOG_FACILITY); + } + + virtual ~LogToSyslog() { + closelog(); + } + + void log(log_level_t level, const std::string& message); + + std::string get_name() const { return name; } + + private: + const std::string name; + + LogToSyslog(const LogToSyslog& other) = delete; + const LogToSyslog& operator=(const LogToSyslog& other) = delete; +}; + +class LogToFile : public LogBackend { + public: + LogToFile(const std::string& filename); + void log(log_level_t level, const std::string& message); + std::string get_name() const { return name; } + + private: + const std::string name; + + struct FILEDeleter{ void operator()(FILE* fd){ if(fd) fclose(fd);}}; + std::unique_ptr log_file; + + LogToFile(const LogToFile& other) = delete; + const LogToFile& operator=(const LogToFile& other) = delete; +}; + +class LogTracer : public LogBackend { + public: + LogTracer(const std::string& filename); + void log(log_level_t level, const std::string& message); + std::string get_name() const { return name; } + private: + std::string name; + uint64_t m_trace_micros_startup = 0; + + struct FILEDeleter{ void operator()(FILE* fd){ if(fd) fclose(fd);}}; + std::unique_ptr m_trace_file; + + LogTracer(const LogTracer& other) = delete; + const LogTracer& operator=(const LogTracer& other) = delete; +}; + +class LogLine; + +struct log_message_t { + log_message_t(log_level_t _level, std::string&& _message) : + level(_level), + message(move(_message)) {} + + log_message_t() : + level(debug), + message("") {} + + log_level_t level; + std::string message; +}; + +class Logger { + public: + Logger() { + m_io_thread = std::thread(&Logger::io_process, this); + } + + Logger(const Logger& other) = delete; + const Logger& operator=(const Logger& other) = delete; + ~Logger() { + m_message_queue.trigger_wakeup(); + m_io_thread.join(); + } + + void register_backend(std::shared_ptr backend); + + /* Log the message to all backends */ + void log(log_level_t level, const char* fmt, ...); + + void logstr(log_level_t level, std::string&& message); + + /* All logging IO is done in another thread */ + void io_process(void); + + /* Return a LogLine for the given level + * so that you can write etiLog.level(info) << "stuff = " << 21 */ + LogLine level(log_level_t level); + + private: + std::list > backends; + + ThreadsafeQueue m_message_queue; + std::thread m_io_thread; + std::mutex m_cerr_mutex; +}; + +extern Logger etiLog; + +// Accumulate a line of logs, using same syntax as stringstream +// The line is logged when the LogLine gets destroyed +class LogLine { + public: + LogLine(const LogLine& logline); + const LogLine& operator=(const LogLine& other) = delete; + LogLine(Logger* logger, log_level_t level) : + logger_(logger) + { + level_ = level; + } + + // Push the new element into the stringstream + template + LogLine& operator<<(T s) { + if (level_ != discard) { + os << s; + } + return *this; + } + + ~LogLine() + { + if (level_ != discard) { + logger_->logstr(level_, os.str()); + } + } + + private: + std::ostringstream os; + log_level_t level_; + Logger* logger_; +}; + diff --git a/lib/ReedSolomon.cpp b/lib/ReedSolomon.cpp index 38d8ea8..1bf0b24 100644 --- a/lib/ReedSolomon.cpp +++ b/lib/ReedSolomon.cpp @@ -64,7 +64,9 @@ ReedSolomon::ReedSolomon(int N, int K, bool reverse, int gfpoly, int firstRoot, ReedSolomon::~ReedSolomon() { - free_rs_char(rsData); + if (rsData != nullptr) { + free_rs_char(rsData); + } } diff --git a/lib/RemoteControl.cpp b/lib/RemoteControl.cpp new file mode 100644 index 0000000..878af59 --- /dev/null +++ b/lib/RemoteControl.cpp @@ -0,0 +1,581 @@ +/* + Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include +#include +#include +#include +#include + +#include "RemoteControl.h" + +using namespace std; + +RemoteControllers rcs; + +RemoteControllerTelnet::~RemoteControllerTelnet() +{ + m_active = false; + + if (m_restarter_thread.joinable()) { + m_restarter_thread.join(); + } + + if (m_child_thread.joinable()) { + m_child_thread.join(); + } +} + +void RemoteControllerTelnet::restart() +{ + if (m_restarter_thread.joinable()) { + m_restarter_thread.join(); + } + + m_restarter_thread = std::thread( + &RemoteControllerTelnet::restart_thread, + this, 0); +} + +RemoteControllable::~RemoteControllable() { + rcs.remove_controllable(this); +} + +std::list RemoteControllable::get_supported_parameters() const { + std::list parameterlist; + for (const auto& param : m_parameters) { + parameterlist.push_back(param[0]); + } + return parameterlist; +} + +void RemoteControllers::add_controller(std::shared_ptr rc) { + m_controllers.push_back(rc); +} + +void RemoteControllers::enrol(RemoteControllable *rc) { + controllables.push_back(rc); +} + +void RemoteControllers::remove_controllable(RemoteControllable *rc) { + controllables.remove(rc); +} + +std::list< std::vector > RemoteControllers::get_param_list_values(const std::string& name) { + RemoteControllable* controllable = get_controllable_(name); + + std::list< std::vector > allparams; + for (auto ¶m : controllable->get_supported_parameters()) { + std::vector item; + item.push_back(param); + try { + item.push_back(controllable->get_parameter(param)); + } + catch (const ParameterError &e) { + item.push_back(std::string("error: ") + e.what()); + } + + allparams.push_back(item); + } + return allparams; +} + +std::string RemoteControllers::get_param(const std::string& name, const std::string& param) { + RemoteControllable* controllable = get_controllable_(name); + return controllable->get_parameter(param); +} + +void RemoteControllers::check_faults() { + for (auto &controller : m_controllers) { + if (controller->fault_detected()) { + etiLog.level(warn) << + "Detected Remote Control fault, restarting it"; + controller->restart(); + } + } +} + +RemoteControllable* RemoteControllers::get_controllable_(const std::string& name) +{ + auto rc = std::find_if(controllables.begin(), controllables.end(), + [&](RemoteControllable* r) { return r->get_rc_name() == name; }); + + if (rc == controllables.end()) { + throw ParameterError("Module name unknown"); + } + else { + return *rc; + } +} + +void RemoteControllers::set_param( + const std::string& name, + const std::string& param, + const std::string& value) +{ + etiLog.level(info) << "RC: Setting " << name << " " << param + << " to " << value; + RemoteControllable* controllable = get_controllable_(name); + try { + return controllable->set_parameter(param, value); + } + catch (const ios_base::failure& e) { + etiLog.level(info) << "RC: Failed to set " << name << " " << param + << " to " << value << ": " << e.what(); + throw ParameterError("Cannot understand value"); + } +} + +// This runs in a separate thread, because +// it would take too long to be done in the main loop +// thread. +void RemoteControllerTelnet::restart_thread(long) +{ + m_active = false; + + if (m_child_thread.joinable()) { + m_child_thread.join(); + } + + m_child_thread = std::thread(&RemoteControllerTelnet::process, this, 0); +} + +void RemoteControllerTelnet::handle_accept(Socket::TCPSocket&& socket) +{ + const std::string welcome = PACKAGE_NAME " Remote Control CLI\n" + "Write 'help' for help.\n" + "**********\n"; + const std::string prompt = "> "; + + std::string in_message; + + try { + etiLog.level(info) << "RC: Accepted"; + + socket.sendall(welcome.data(), welcome.size()); + + while (m_active and in_message != "quit") { + socket.sendall(prompt.data(), prompt.size()); + + stringstream in_message_stream; + + char last_char = '\0'; + try { + while (last_char != '\n') { + try { + auto ret = socket.recv(&last_char, 1, 0, 1000); + if (ret == 1) { + in_message_stream << last_char; + } + else { + break; + } + } + catch (const Socket::TCPSocket::Timeout&) { + if (not m_active) { + break; + } + } + } + } + catch (const Socket::TCPSocket::Interrupted&) { + in_message_stream.clear(); + } + + + if (in_message_stream.str().size() == 0) { + etiLog.level(info) << "RC: Connection terminated"; + break; + } + + std::getline(in_message_stream, in_message); + + while (in_message.length() > 0 && + (in_message[in_message.length()-1] == '\r' || + in_message[in_message.length()-1] == '\n')) { + in_message.erase(in_message.length()-1, 1); + } + + if (in_message.length() == 0) { + continue; + } + + etiLog.level(info) << "RC: Got message '" << in_message << "'"; + + dispatch_command(socket, in_message); + } + etiLog.level(info) << "RC: Closing socket"; + socket.close(); + } + catch (const std::exception& e) { + etiLog.level(error) << "Remote control caught exception: " << e.what(); + } +} + +void RemoteControllerTelnet::process(long) +{ + try { + m_active = true; + + m_socket.listen(m_port, "localhost"); + + etiLog.level(info) << "RC: Waiting for connection on port " << m_port; + while (m_active) { + auto sock = m_socket.accept(1000); + + if (sock.valid()) { + handle_accept(move(sock)); + etiLog.level(info) << "RC: Connection closed. Waiting for connection on port " << m_port; + } + } + } + catch (const runtime_error& e) { + etiLog.level(warn) << "RC: Encountered error: " << e.what(); + } + + etiLog.level(info) << "RC: Leaving"; + m_fault = true; +} + +static std::vector tokenise(const std::string& message) { + stringstream ss(message); + std::vector all_tokens; + std::string item; + + while (std::getline(ss, item, ' ')) { + all_tokens.push_back(move(item)); + } + return all_tokens; +} + + +void RemoteControllerTelnet::dispatch_command(Socket::TCPSocket& socket, string command) +{ + vector cmd = tokenise(command); + + if (cmd[0] == "help") { + reply(socket, + "The following commands are supported:\n" + " list\n" + " * Lists the modules that are loaded and their parameters\n" + " show MODULE\n" + " * Lists all parameters and their values from module MODULE\n" + " get MODULE PARAMETER\n" + " * Gets the value for the specified PARAMETER from module MODULE\n" + " set MODULE PARAMETER VALUE\n" + " * Sets the value for the PARAMETER ofr module MODULE\n" + " quit\n" + " * Terminate this session\n" + "\n"); + } + else if (cmd[0] == "list") { + stringstream ss; + + if (cmd.size() == 1) { + for (auto &controllable : rcs.controllables) { + ss << controllable->get_rc_name() << endl; + + list< vector > params = controllable->get_parameter_descriptions(); + for (auto ¶m : params) { + ss << "\t" << param[0] << " : " << param[1] << endl; + } + } + } + else { + reply(socket, "Too many arguments for command 'list'"); + } + + reply(socket, ss.str()); + } + else if (cmd[0] == "show") { + if (cmd.size() == 2) { + try { + stringstream ss; + list< vector > r = rcs.get_param_list_values(cmd[1]); + for (auto ¶m_val : r) { + ss << param_val[0] << ": " << param_val[1] << endl; + } + reply(socket, ss.str()); + + } + catch (const ParameterError &e) { + reply(socket, e.what()); + } + } + else { + reply(socket, "Incorrect parameters for command 'show'"); + } + } + else if (cmd[0] == "get") { + if (cmd.size() == 3) { + try { + string r = rcs.get_param(cmd[1], cmd[2]); + reply(socket, r); + } + catch (const ParameterError &e) { + reply(socket, e.what()); + } + } + else { + reply(socket, "Incorrect parameters for command 'get'"); + } + } + else if (cmd[0] == "set") { + if (cmd.size() >= 4) { + try { + stringstream new_param_value; + for (size_t i = 3; i < cmd.size(); i++) { + new_param_value << cmd[i]; + + if (i+1 < cmd.size()) { + new_param_value << " "; + } + } + + rcs.set_param(cmd[1], cmd[2], new_param_value.str()); + reply(socket, "ok"); + } + catch (const ParameterError &e) { + reply(socket, e.what()); + } + catch (const exception &e) { + reply(socket, "Error: Invalid parameter value. "); + } + } + else { + reply(socket, "Incorrect parameters for command 'set'"); + } + } + else if (cmd[0] == "quit") { + reply(socket, "Goodbye"); + } + else { + reply(socket, "Message not understood"); + } +} + +void RemoteControllerTelnet::reply(Socket::TCPSocket& socket, string message) +{ + stringstream ss; + ss << message << "\r\n"; + socket.sendall(message.data(), message.size()); +} + + +#if defined(HAVE_ZEROMQ) + +RemoteControllerZmq::~RemoteControllerZmq() { + m_active = false; + m_fault = false; + + if (m_restarter_thread.joinable()) { + m_restarter_thread.join(); + } + + if (m_child_thread.joinable()) { + m_child_thread.join(); + } +} + +void RemoteControllerZmq::restart() +{ + if (m_restarter_thread.joinable()) { + m_restarter_thread.join(); + } + + m_restarter_thread = std::thread(&RemoteControllerZmq::restart_thread, this); +} + +// This runs in a separate thread, because +// it would take too long to be done in the main loop +// thread. +void RemoteControllerZmq::restart_thread() +{ + m_active = false; + + if (m_child_thread.joinable()) { + m_child_thread.join(); + } + + m_child_thread = std::thread(&RemoteControllerZmq::process, this); +} + +void RemoteControllerZmq::recv_all(zmq::socket_t& pSocket, std::vector &message) +{ + bool more = true; + do { + zmq::message_t msg; + pSocket.recv(&msg); + std::string incoming((char*)msg.data(), msg.size()); + message.push_back(incoming); + more = msg.more(); + } while (more); +} + +void RemoteControllerZmq::send_ok_reply(zmq::socket_t &pSocket) +{ + zmq::message_t msg(2); + char repCode[2] = {'o', 'k'}; + memcpy ((void*) msg.data(), repCode, 2); + pSocket.send(msg, 0); +} + +void RemoteControllerZmq::send_fail_reply(zmq::socket_t &pSocket, const std::string &error) +{ + zmq::message_t msg1(4); + char repCode[4] = {'f', 'a', 'i', 'l'}; + memcpy ((void*) msg1.data(), repCode, 4); + pSocket.send(msg1, ZMQ_SNDMORE); + + zmq::message_t msg2(error.length()); + memcpy ((void*) msg2.data(), error.c_str(), error.length()); + pSocket.send(msg2, 0); +} + +void RemoteControllerZmq::process() +{ + m_fault = false; + + // create zmq reply socket for receiving ctrl parameters + try { + zmq::socket_t repSocket(m_zmqContext, ZMQ_REP); + + // connect the socket + int hwm = 100; + int linger = 0; + repSocket.setsockopt(ZMQ_RCVHWM, &hwm, sizeof(hwm)); + repSocket.setsockopt(ZMQ_SNDHWM, &hwm, sizeof(hwm)); + repSocket.setsockopt(ZMQ_LINGER, &linger, sizeof(linger)); + repSocket.bind(m_endpoint.c_str()); + + // create pollitem that polls the ZMQ sockets + zmq::pollitem_t pollItems[] = { {repSocket, 0, ZMQ_POLLIN, 0} }; + while (m_active) { + zmq::poll(pollItems, 1, 100); + std::vector msg; + + if (pollItems[0].revents & ZMQ_POLLIN) { + recv_all(repSocket, msg); + + std::string command((char*)msg[0].data(), msg[0].size()); + + if (msg.size() == 1 && command == "ping") { + send_ok_reply(repSocket); + } + else if (msg.size() == 1 && command == "list") { + size_t cohort_size = rcs.controllables.size(); + for (auto &controllable : rcs.controllables) { + std::stringstream ss; + ss << "{ \"name\": \"" << controllable->get_rc_name() << "\"," << + " \"params\": { "; + + list< vector > params = controllable->get_parameter_descriptions(); + size_t i = 0; + for (auto ¶m : params) { + if (i > 0) { + ss << ", "; + } + + ss << "\"" << param[0] << "\": " << + "\"" << param[1] << "\""; + + i++; + } + + ss << " } }"; + + std::string msg_s = ss.str(); + + zmq::message_t zmsg(ss.str().size()); + memcpy ((void*) zmsg.data(), msg_s.data(), msg_s.size()); + + int flag = (--cohort_size > 0) ? ZMQ_SNDMORE : 0; + repSocket.send(zmsg, flag); + } + } + else if (msg.size() == 2 && command == "show") { + std::string module((char*) msg[1].data(), msg[1].size()); + try { + list< vector > r = rcs.get_param_list_values(module); + size_t r_size = r.size(); + for (auto ¶m_val : r) { + std::stringstream ss; + ss << param_val[0] << ": " << param_val[1] << endl; + zmq::message_t zmsg(ss.str().size()); + memcpy(zmsg.data(), ss.str().data(), ss.str().size()); + + int flag = (--r_size > 0) ? ZMQ_SNDMORE : 0; + repSocket.send(zmsg, flag); + } + } + catch (const ParameterError &err) { + send_fail_reply(repSocket, err.what()); + } + } + else if (msg.size() == 3 && command == "get") { + std::string module((char*) msg[1].data(), msg[1].size()); + std::string parameter((char*) msg[2].data(), msg[2].size()); + + try { + std::string value = rcs.get_param(module, parameter); + zmq::message_t zmsg(value.size()); + memcpy ((void*) zmsg.data(), value.data(), value.size()); + repSocket.send(zmsg, 0); + } + catch (const ParameterError &err) { + send_fail_reply(repSocket, err.what()); + } + } + else if (msg.size() == 4 && command == "set") { + std::string module((char*) msg[1].data(), msg[1].size()); + std::string parameter((char*) msg[2].data(), msg[2].size()); + std::string value((char*) msg[3].data(), msg[3].size()); + + try { + rcs.set_param(module, parameter, value); + send_ok_reply(repSocket); + } + catch (const ParameterError &err) { + send_fail_reply(repSocket, err.what()); + } + } + else { + send_fail_reply(repSocket, + "Unsupported command. commands: list, show, get, set"); + } + } + } + repSocket.close(); + } + catch (const zmq::error_t &e) { + etiLog.level(error) << "ZMQ RC error: " << std::string(e.what()); + } + catch (const std::exception& e) { + etiLog.level(error) << "ZMQ RC caught exception: " << e.what(); + m_fault = true; + } +} + +#endif + diff --git a/lib/RemoteControl.h b/lib/RemoteControl.h new file mode 100644 index 0000000..bd88f82 --- /dev/null +++ b/lib/RemoteControl.h @@ -0,0 +1,249 @@ +/* + Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 + Her Majesty the Queen in Right of Canada (Communications Research + Center Canada) + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + This module adds remote-control capability to some of the dabmux/dabmod modules. + */ +/* + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#if defined(HAVE_ZEROMQ) +# include "zmq.hpp" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Log.h" +#include "Socket.h" + +#define RC_ADD_PARAMETER(p, desc) { \ + std::vector p; \ + p.push_back(#p); \ + p.push_back(desc); \ + m_parameters.push_back(p); \ +} + +class ParameterError : public std::exception +{ + public: + ParameterError(std::string message) : m_message(message) {} + ~ParameterError() throw() {} + const char* what() const throw() { return m_message.c_str(); } + + private: + std::string m_message; +}; + +class RemoteControllable; + +/* Remote controllers (that recieve orders from the user) + * must implement BaseRemoteController + */ +class BaseRemoteController { + public: + /* When this returns one, the remote controller cannot be + * used anymore, and must be restarted + */ + virtual bool fault_detected() = 0; + + /* In case of a fault, the remote controller can be + * restarted. + */ + virtual void restart() = 0; + + virtual ~BaseRemoteController() {} +}; + +/* Objects that support remote control must implement the following class */ +class RemoteControllable { + public: + RemoteControllable(const std::string& name) : + m_rc_name(name) {} + + RemoteControllable(const RemoteControllable& other) = delete; + RemoteControllable& operator=(const RemoteControllable& other) = delete; + + virtual ~RemoteControllable(); + + /* return a short name used to identify the controllable. + * It might be used in the commands the user has to type, so keep + * it short + */ + virtual std::string get_rc_name() const { return m_rc_name; } + + /* Return a list of possible parameters that can be set */ + virtual std::list get_supported_parameters() const; + + /* Return a mapping of the descriptions of all parameters */ + virtual std::list< std::vector > + get_parameter_descriptions() const + { + return m_parameters; + } + + /* Base function to set parameters. */ + virtual void set_parameter( + const std::string& parameter, + const std::string& value) = 0; + + /* Getting a parameter always returns a string. */ + virtual const std::string get_parameter(const std::string& parameter) const = 0; + + protected: + std::string m_rc_name; + std::list< std::vector > m_parameters; +}; + +/* Holds all our remote controllers and controlled object. + */ +class RemoteControllers { + public: + void add_controller(std::shared_ptr rc); + void enrol(RemoteControllable *rc); + void remove_controllable(RemoteControllable *rc); + void check_faults(); + std::list< std::vector > get_param_list_values(const std::string& name); + std::string get_param(const std::string& name, const std::string& param); + + void set_param( + const std::string& name, + const std::string& param, + const std::string& value); + + std::list controllables; + + private: + RemoteControllable* get_controllable_(const std::string& name); + + std::list > m_controllers; +}; + +extern RemoteControllers rcs; + +/* Implements a Remote controller based on a simple telnet CLI + * that listens on localhost + */ +class RemoteControllerTelnet : public BaseRemoteController { + public: + RemoteControllerTelnet() + : m_active(false), + m_fault(false), + m_port(0) { } + + RemoteControllerTelnet(int port) + : m_active(port > 0), + m_fault(false), + m_port(port) + { + restart(); + } + + + RemoteControllerTelnet& operator=(const RemoteControllerTelnet& other) = delete; + RemoteControllerTelnet(const RemoteControllerTelnet& other) = delete; + + ~RemoteControllerTelnet(); + + virtual bool fault_detected() { return m_fault; } + + virtual void restart(); + + private: + void restart_thread(long); + + void process(long); + + void dispatch_command(Socket::TCPSocket& socket, std::string command); + void reply(Socket::TCPSocket& socket, std::string message); + void handle_accept(Socket::TCPSocket&& socket); + + std::atomic m_active; + + /* This is set to true if a fault occurred */ + std::atomic m_fault; + std::thread m_restarter_thread; + + std::thread m_child_thread; + + Socket::TCPSocket m_socket; + int m_port; +}; + +#if defined(HAVE_ZEROMQ) +/* Implements a Remote controller using ZMQ transportlayer + * that listens on localhost + */ +class RemoteControllerZmq : public BaseRemoteController { + public: + RemoteControllerZmq() + : m_active(false), m_fault(false), + m_zmqContext(1), + m_endpoint("") { } + + RemoteControllerZmq(const std::string& endpoint) + : m_active(not endpoint.empty()), m_fault(false), + m_zmqContext(1), + m_endpoint(endpoint), + m_child_thread(&RemoteControllerZmq::process, this) { } + + RemoteControllerZmq& operator=(const RemoteControllerZmq& other) = delete; + RemoteControllerZmq(const RemoteControllerZmq& other) = delete; + + ~RemoteControllerZmq(); + + virtual bool fault_detected() { return m_fault; } + + virtual void restart(); + + private: + void restart_thread(); + + void recv_all(zmq::socket_t &pSocket, std::vector &message); + void send_ok_reply(zmq::socket_t &pSocket); + void send_fail_reply(zmq::socket_t &pSocket, const std::string &error); + void process(); + + std::atomic m_active; + + /* This is set to true if a fault occurred */ + std::atomic m_fault; + std::thread m_restarter_thread; + + zmq::context_t m_zmqContext; + + std::string m_endpoint; + std::thread m_child_thread; +}; +#endif + diff --git a/lib/crc.c b/lib/crc.c new file mode 100644 index 0000000..cc02473 --- /dev/null +++ b/lib/crc.c @@ -0,0 +1,266 @@ +/* + Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the + Queen in Right of Canada (Communications Research Center Canada) + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#include "crc.h" +#ifndef _WIN32 +# include +# include +#endif +#include +#include + +//#define CCITT 0x1021 + +uint8_t crc8tab[256] = { + 0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, + 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d, + 0x70, 0x77, 0x7e, 0x79, 0x6c, 0x6b, 0x62, 0x65, + 0x48, 0x4f, 0x46, 0x41, 0x54, 0x53, 0x5a, 0x5d, + 0xe0, 0xe7, 0xee, 0xe9, 0xfc, 0xfb, 0xf2, 0xf5, + 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 0xc3, 0xca, 0xcd, + 0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 0x82, 0x85, + 0xa8, 0xaf, 0xa6, 0xa1, 0xb4, 0xb3, 0xba, 0xbd, + 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2, + 0xff, 0xf8, 0xf1, 0xf6, 0xe3, 0xe4, 0xed, 0xea, + 0xb7, 0xb0, 0xb9, 0xbe, 0xab, 0xac, 0xa5, 0xa2, + 0x8f, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9d, 0x9a, + 0x27, 0x20, 0x29, 0x2e, 0x3b, 0x3c, 0x35, 0x32, + 0x1f, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0d, 0x0a, + 0x57, 0x50, 0x59, 0x5e, 0x4b, 0x4c, 0x45, 0x42, + 0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7d, 0x7a, + 0x89, 0x8e, 0x87, 0x80, 0x95, 0x92, 0x9b, 0x9c, + 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4, + 0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, + 0xc1, 0xc6, 0xcf, 0xc8, 0xdd, 0xda, 0xd3, 0xd4, + 0x69, 0x6e, 0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c, + 0x51, 0x56, 0x5f, 0x58, 0x4d, 0x4a, 0x43, 0x44, + 0x19, 0x1e, 0x17, 0x10, 0x05, 0x02, 0x0b, 0x0c, + 0x21, 0x26, 0x2f, 0x28, 0x3d, 0x3a, 0x33, 0x34, + 0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5c, 0x5b, + 0x76, 0x71, 0x78, 0x7f, 0x6a, 0x6d, 0x64, 0x63, + 0x3e, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b, + 0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, + 0xae, 0xa9, 0xa0, 0xa7, 0xb2, 0xb5, 0xbc, 0xbb, + 0x96, 0x91, 0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83, + 0xde, 0xd9, 0xd0, 0xd7, 0xc2, 0xc5, 0xcc, 0xcb, + 0xe6, 0xe1, 0xe8, 0xef, 0xfa, 0xfd, 0xf4, 0xf3 +}; + + +uint16_t crc16tab[256] = { + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, + 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, + 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, + 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, + 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, + 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, + 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, + 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, + 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, + 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, + 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, + 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, + 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, + 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, + 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, + 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, + 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, + 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, + 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, + 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, + 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, + 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, + 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, + 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, + 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, + 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, + 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 +}; + + +uint32_t crc32tab[256] = { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, + 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, + 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, + 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, + 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, + 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, + 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, + 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, + 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, + 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, + 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, + 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, + 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, + 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, + 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, + 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, + 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, + 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, + 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, + 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, + 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, + 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 +}; + +// This function can be used to create a new table with a different polynom +void init_crc8tab(uint8_t l_code, uint8_t l_init) +{ + unsigned i, j, msb; + uint8_t nb; + uint8_t crc; + + for (i = 0; i < 256; ++i) { + crc = l_init; + nb = i ^ 0xff; + for (j = 0; j < 8; ++j) { + msb = (nb & (0x80 >> j)) && 1; + msb ^= (crc >> 7); + crc <<= 1; + if (msb) + crc ^= l_code; + } + crc8tab[i] = crc; + } +} + + +void init_crc16tab(uint16_t l_code, uint16_t l_init) +{ + unsigned i, j, msb; + uint8_t nb; + uint16_t crc; + + for (i = 0; i < 256; ++i) { + crc = l_init; + nb = i ^ 0xff; + for (j = 0; j < 8; ++j) { + msb = (nb & (0x80 >> j)) && 1; + msb ^= (crc >> 15); + crc <<= 1; + if (msb) + crc ^= l_code; + } + crc ^= 0xff00; + crc16tab[i] = crc; + } +} + + +void init_crc32tab(uint32_t l_code, uint32_t l_init) +{ + unsigned i, j, msb; + uint8_t nb; + uint32_t crc; + + for (i = 0; i < 256; ++i) { + crc = l_init; + nb = i ^ 0xff; + for (j = 0; j < 8; ++j) { + msb = (nb & (0x80 >> j)) && 1; + msb ^= (crc >> 31); + crc <<= 1; + if (msb) + crc ^= l_code; + } + crc ^= 0xffffff00; + crc32tab[i] = crc; + } +} + + +uint8_t crc8(uint8_t l_crc, const void *lp_data, unsigned l_nb) +{ + const uint8_t* data = (const uint8_t*)lp_data; + while (l_nb--) { + l_crc = crc8tab[l_crc ^ *(data++)]; + } + return (l_crc); +} + + +uint16_t crc16(uint16_t l_crc, const void *lp_data, unsigned l_nb) +{ + const uint8_t* data = (const uint8_t*)lp_data; + while (l_nb--) { + l_crc = + (l_crc << 8) ^ crc16tab[(l_crc >> 8) ^ *(data++)]; + } + return (l_crc); +} + + +uint32_t crc32(uint32_t l_crc, const void *lp_data, unsigned l_nb) +{ + const uint8_t* data = (const uint8_t*)lp_data; + while (l_nb--) { + l_crc = + (l_crc << 8) ^ crc32tab[((l_crc >> 24) ^ *(data++)) & 0xff]; + } + return (l_crc); +} diff --git a/lib/crc.h b/lib/crc.h new file mode 100644 index 0000000..b1785a1 --- /dev/null +++ b/lib/crc.h @@ -0,0 +1,59 @@ +/* + Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the + Queen in Right of Canada (Communications Research Center Canada) + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#ifndef _CRC +#define _CRC + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#ifndef _WIN32 + #include +#else + #include // For types... + typedef BYTE uint8_t; + typedef WORD uint16_t; + typedef DWORD32 uint32_t; +#endif + + +#ifdef __cplusplus +extern "C" { // } +#endif + +void init_crc8tab(uint8_t l_code, uint8_t l_init); +uint8_t crc8(uint8_t l_crc, const void *lp_data, unsigned l_nb); +extern uint8_t crc8tab[]; + +void init_crc16tab(uint16_t l_code, uint16_t l_init); +uint16_t crc16(uint16_t l_crc, const void *lp_data, unsigned l_nb); +extern uint16_t crc16tab[]; + +void init_crc32tab(uint32_t l_code, uint32_t l_init); +uint32_t crc32(uint32_t l_crc, const void *lp_data, unsigned l_nb); +extern uint32_t crc32tab[]; + +#ifdef __cplusplus +} +#endif + +#endif //_CRC diff --git a/src/ClockTAI.cpp b/src/ClockTAI.cpp deleted file mode 100644 index c376c07..0000000 --- a/src/ClockTAI.cpp +++ /dev/null @@ -1,562 +0,0 @@ -/* - Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, - 2011, 2012 Her Majesty the Queen in Right of Canada (Communications - Research Center Canada) - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . -*/ - -/* This file downloads the TAI-UTC bulletins from the from IETF and parses them - * so that correct time can be communicated in EDI timestamps. - * - * This file contains self-test code that can be executed by running - * g++ -g -Wall -DTEST -DHAVE_CURL -std=c++11 -lcurl -pthread \ - * ClockTAI.cpp Log.cpp RemoteControl.cpp -lboost_system -o taitest && ./taitest - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "ClockTAI.h" -#include "Log.h" - -#include -#include -#include -#if SUPPORT_SETTING_CLOCK_TAI -# include -#endif -#ifdef HAVE_CURL -# include -#endif -#include -#include -#include -#include -#include - -using namespace std; - -#ifdef TEST -static bool wait_longer = true; -#endif - -constexpr int download_retry_interval_hours = 1; - -// Offset between NTP time and POSIX time: -// timestamp_unix = timestamp_ntp - ntp_unix_offset -const int64_t ntp_unix_offset = 2208988800L; - -// leap seconds insertion bulletin is available from the IETF and in the TZ -// distribution -static array default_tai_urls = { - "https://www.ietf.org/timezones/data/leap-seconds.list", - "https://raw.githubusercontent.com/eggert/tz/master/leap-seconds.list", -}; - -// According to the Filesystem Hierarchy Standard, the data in -// /var/tmp "must not be deleted when the system is booted." -static const char *tai_cache_location = "/var/tmp/odr-dabmux-leap-seconds.cache"; - -// read TAI offset from a valid bulletin in IETF format -static int parse_ietf_bulletin(const std::string& bulletin) -{ - // Example Line: - // 3692217600 37 # 1 Jan 2017 - // - // NTP timestampleap seconds# some comment - // The NTP timestamp starts at epoch 1.1.1900. - // The difference between NTP timestamps and unix epoch is 70 - // years i.e. 2208988800 seconds - - std::regex regex_bulletin(R"(([0-9]+)\s+([0-9]+)\s+#.*)"); - - time_t now = time(nullptr); - - int tai_utc_offset = 0; - - int tai_utc_offset_valid = false; - - stringstream ss(bulletin); - - /* We cannot just take the last line, because it might - * be in the future, announcing an upcoming leap second. - * - * So we need to look at the current date, and compare it - * with the date of the leap second. - */ - for (string line; getline(ss, line); ) { - - std::smatch bulletin_entry; - - bool is_match = std::regex_search(line, bulletin_entry, regex_bulletin); - if (is_match) { - if (bulletin_entry.size() != 3) { - throw runtime_error( - "Incorrect number of matched TAI IETF bulletin entries"); - } - const string bulletin_ntp_timestamp(bulletin_entry[1]); - const string bulletin_offset(bulletin_entry[2]); - - const int64_t timestamp_unix = - std::atoll(bulletin_ntp_timestamp.c_str()) - ntp_unix_offset; - - const int offset = std::atoi(bulletin_offset.c_str()); - // Ignore entries announcing leap seconds in the future - if (timestamp_unix < now) { - tai_utc_offset = offset; - tai_utc_offset_valid = true; - } -#if TEST - else { - cerr << "IETF Ignoring offset " << bulletin_offset << - " at TS " << bulletin_ntp_timestamp << - " in the future" << endl; - } -#endif - } - } - - if (not tai_utc_offset_valid) { - throw runtime_error("No data in TAI bulletin"); - } - - return tai_utc_offset; -} - - -struct bulletin_state { - bool valid = false; - int64_t expiry = 0; - int offset = 0; - - bool usable() const { return valid and expiry > 0; } -}; - -static bulletin_state parse_bulletin(const string& bulletin) -{ - // The bulletin contains one line that specifies an expiration date - // in NTP time. If that point in time is in the future, we consider - // the bulletin valid. - // - // The entry looks like this: - //#@ 3707596800 - - bulletin_state ret; - - std::regex regex_expiration(R"(#@\s+([0-9]+))"); - - time_t now = time(nullptr); - - stringstream ss(bulletin); - - for (string line; getline(ss, line); ) { - std::smatch bulletin_entry; - - bool is_match = std::regex_search(line, bulletin_entry, regex_expiration); - if (is_match) { - if (bulletin_entry.size() != 2) { - throw runtime_error( - "Incorrect number of matched TAI IETF bulletin expiration"); - } - const string expiry_data_str(bulletin_entry[1]); - const int64_t expiry_unix = - std::atoll(expiry_data_str.c_str()) - ntp_unix_offset; - -#ifdef TEST - etiLog.level(info) << "Bulletin expires in " << expiry_unix - now; -#endif - ret.expiry = expiry_unix - now; - try { - ret.offset = parse_ietf_bulletin(bulletin); - ret.valid = true; - } - catch (const runtime_error& e) { - etiLog.level(warn) << "Bulletin expiry ok but parse error: " << e.what(); - } - break; - } - } - return ret; -} - - -// callback that receives data from cURL -static size_t fill_bulletin(char *ptr, size_t size, size_t nmemb, void *ctx) -{ - auto *bulletin = reinterpret_cast(ctx); - - size_t len = size * nmemb; - for (size_t i = 0; i < len; i++) { - *bulletin << ptr[i]; - } - return len; -} - -static string download_tai_utc_bulletin(const char* url) -{ - stringstream bulletin; - -#ifdef HAVE_CURL - CURL *curl; - CURLcode res; - - curl = curl_easy_init(); - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, url); - /* Tell libcurl to follow redirection */ - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fill_bulletin); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &bulletin); - - res = curl_easy_perform(curl); - /* always cleanup ! */ - curl_easy_cleanup(curl); - - if (res != CURLE_OK) { - throw runtime_error( "TAI-UTC bulletin download failed: " + - string(curl_easy_strerror(res))); - } - } - return bulletin.str(); -#else - throw runtime_error("Cannot download TAI Clock information without cURL"); -#endif // HAVE_CURL -} - -static string load_bulletin_from_file(const char* cache_filename) -{ - // Clear the bulletin - ifstream f(cache_filename); - if (not f.good()) { - return {}; - } - - stringstream ss; - ss << f.rdbuf(); - f.close(); - - return ss.str(); -} - -ClockTAI::ClockTAI(const std::vector& bulletin_urls) : - RemoteControllable("clocktai") -{ - RC_ADD_PARAMETER(expiry, "Number of seconds until TAI Bulletin expires"); - - if (bulletin_urls.empty()) { - etiLog.level(debug) << "Initialising default TAI Bulletin URLs"; - for (const auto url : default_tai_urls) { - m_bulletin_urls.push_back(url); - } - } - else { - etiLog.level(debug) << "Initialising user-configured TAI Bulletin URLs"; - m_bulletin_urls = bulletin_urls; - } - - for (const auto url : m_bulletin_urls) { - etiLog.level(info) << "TAI Bulletin URL: '" << url << "'"; - } -} - -int ClockTAI::get_valid_offset() -{ - int offset = 0; - bool offset_valid = false; - - std::unique_lock lock(m_data_mutex); - - const auto state = parse_bulletin(m_bulletin); - if (state.usable()) { -#if TEST - etiLog.level(info) << "Bulletin already valid"; -#endif - offset = state.offset; - offset_valid = true; - } - else { - const auto cache_bulletin = load_bulletin_from_file(tai_cache_location); - const auto cache_state = parse_bulletin(cache_bulletin); - - if (cache_state.usable()) { - m_bulletin = cache_bulletin; - offset = cache_state.offset; - offset_valid = true; -#if TEST - etiLog.level(info) << "Bulletin from cache valid with offset=" << offset; -#endif - } - else { - for (const auto url : m_bulletin_urls) { - try { -#if TEST - etiLog.level(info) << "Load bulletin from " << url; -#endif - const auto new_bulletin = download_tai_utc_bulletin(url.c_str()); - const auto new_state = parse_bulletin(new_bulletin); - if (new_state.usable()) { - m_bulletin = new_bulletin; - offset = new_state.offset; - offset_valid = true; - - etiLog.level(debug) << "Loaded valid TAI Bulletin from " << - url << " giving offset=" << offset; - } - else { - etiLog.level(debug) << "Skipping invalid TAI bulletin from " - << url; - } - } - catch (const runtime_error& e) { - etiLog.level(warn) << - "TAI-UTC offset could not be retrieved from " << - url << " : " << e.what(); - } - - if (offset_valid) { - update_cache(tai_cache_location); - break; - } - } - } - } - - if (offset_valid) { - // With the current evolution of the offset, we're probably going - // to reach 500 long after DAB gets replaced by another standard. - if (offset < 0 or offset > 500) { - stringstream ss; - ss << "TAI offset " << offset << " out of range"; - throw range_error(ss.str()); - } - - return offset; - } - else { - // Try again later - throw download_failed(); - } -} - - -int ClockTAI::get_offset() -{ - using namespace std::chrono; - const auto time_now = system_clock::now(); - - std::unique_lock lock(m_data_mutex); - - if (not m_offset_valid) { -#ifdef TEST - // Assume we've downloaded it in the past: - - m_offset = 37; // Valid in early 2017 - m_offset_valid = true; - - // Simulate requiring a new download - m_bulletin_download_time = time_now - hours(24 * 40); -#else - // First time we run we must block until we know - // the offset - lock.unlock(); - try { - m_offset = get_valid_offset(); - } - catch (const download_failed&) { - throw runtime_error("Unable to download TAI bulletin"); - } - lock.lock(); - m_offset_valid = true; - m_bulletin_download_time = time_now; -#endif - etiLog.level(info) << - "Initialised TAI-UTC offset to " << m_offset << "s."; - } - - if (time_now - m_bulletin_download_time > hours(24 * 31)) { - // Refresh if it's older than one month. Leap seconds are - // announced several months in advance - etiLog.level(debug) << "Trying to refresh TAI bulletin"; - - if (m_offset_future.valid()) { - auto state = m_offset_future.wait_for(seconds(0)); - switch (state) { - case future_status::ready: - try { - m_offset = m_offset_future.get(); - m_offset_valid = true; - m_bulletin_download_time = time_now; - - etiLog.level(info) << - "Updated TAI-UTC offset to " << m_offset << "s."; - } - catch (const download_failed&) { - etiLog.level(warn) << - "TAI-UTC download failed, will retry in " << - download_retry_interval_hours << " hour(s)"; - - m_bulletin_download_time += hours(download_retry_interval_hours); - } -#ifdef TEST - wait_longer = false; -#endif - break; - - case future_status::deferred: - case future_status::timeout: - // Not ready yet -#ifdef TEST - etiLog.level(debug) << " async not ready yet"; -#endif - break; - } - } - else { -#ifdef TEST - etiLog.level(debug) << " Launch async"; -#endif - m_offset_future = async(launch::async, &ClockTAI::get_valid_offset, this); - } - } - - return m_offset; -} - -#if SUPPORT_SETTING_CLOCK_TAI -int ClockTAI::update_local_tai_clock(int offset) -{ - struct timex timex_request; - timex_request.modes = ADJ_TAI; - timex_request.constant = offset; - - int err = adjtimex(&timex_request); - if (err == -1) { - perror("adjtimex"); - } - - printf("adjtimex: %d, tai %d\n", err, timex_request.tai); - - return err; -} -#endif - -void ClockTAI::update_cache(const char* cache_filename) -{ - ofstream f(cache_filename); - if (not f.good()) { - throw runtime_error("TAI-UTC bulletin open cache for writing"); - } - - f << m_bulletin; - f.close(); -} - - -void ClockTAI::set_parameter(const string& parameter, const string& value) -{ - if (parameter == "expiry") { - throw ParameterError("Parameter '" + parameter + - "' is not read-only in controllable " + get_rc_name()); - } - else { - throw ParameterError("Parameter '" + parameter + - "' is not exported by controllable " + get_rc_name()); - } -} - -const string ClockTAI::get_parameter(const string& parameter) const -{ - if (parameter == "expiry") { - std::unique_lock lock(m_data_mutex); - const int64_t expiry = parse_bulletin(m_bulletin).expiry; - if (expiry > 0) { - return to_string(expiry); - } - else { - return "Bulletin expired or invalid!"; - } - } - else { - throw ParameterError("Parameter '" + parameter + - "' is not exported by controllable " + get_rc_name()); - } -} - -#if 0 -// Example testing code -void debug_tai_clk() -{ - struct timespec rt_clk; - - int err = clock_gettime(CLOCK_REALTIME, &rt_clk); - if (err) { - perror("REALTIME clock_gettime failed"); - } - - struct timespec tai_clk; - - err = clock_gettime(CLOCK_TAI, &tai_clk); - if (err) { - perror("TAI clock_gettime failed"); - } - - printf("RT - TAI = %ld\n", rt_clk.tv_sec - tai_clk.tv_sec); - - - struct timex timex_request; - timex_request.modes = 0; // Do not set anything - - err = adjtimex(&timex_request); - if (err == -1) { - perror("adjtimex"); - } - - printf("adjtimex: %d, tai %d\n", err, timex_request.tai); -} -#endif - -#if TEST -int main(int argc, char **argv) -{ - using namespace std; - - ClockTAI tai({}); - - while (wait_longer) { - try { - etiLog.level(info) << - "Offset is " << tai.get_offset(); - } - catch (const exception &e) { - etiLog.level(error) << - "Exception " << e.what(); - } - - this_thread::sleep_for(chrono::seconds(2)); - } - - return 0; -} -#endif - diff --git a/src/ClockTAI.h b/src/ClockTAI.h deleted file mode 100644 index 4b3c2ff..0000000 --- a/src/ClockTAI.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, - 2011, 2012 Her Majesty the Queen in Right of Canada (Communications - Research Center Canada) - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . -*/ - -/* The EDI output needs TAI clock, according to ETSI TS 102 693 Annex F - * "EDI Timestamps". This module can set the local CLOCK_TAI clock by - * setting the TAI-UTC offset using adjtimex. - * - * This functionality requires Linux 3.10 (30 Jun 2013) or newer. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include "RemoteControl.h" - -// EDI needs to know UTC-TAI, but doesn't need the CLOCK_TAI to be set. -// We can keep this code, maybe for future use -#define SUPPORT_SETTING_CLOCK_TAI 0 - -/* Loads, parses and represents TAI-UTC offset information from the IETF bulletin */ -class ClockTAI : public RemoteControllable { - public: - ClockTAI(const std::vector& bulletin_urls); - - // Fetch the bulletin from the IETF website and return the current - // TAI-UTC offset. - // Throws runtime_error on failure. - int get_offset(void); - -#if SUPPORT_SETTING_CLOCK_TAI - // Update the local TAI clock according to the TAI-UTC offset - // return 0 on success - int update_local_tai_clock(int offset); -#endif - - private: - class download_failed {}; - - // Either retrieve the bulletin from the cache or if necessarly - // download it, and calculate the TAI-UTC offset. - // Returns the offset or throws download_failed or a range_error - // if the offset is out of bounds. - int get_valid_offset(void); - - // Download of new bulletin is done asynchronously - std::future m_offset_future; - - // Protect all data members, as RC functions are in another thread - mutable std::mutex m_data_mutex; - - // The currently used TAI-UTC offset - int m_offset = 0; - int m_offset_valid = false; - - std::vector m_bulletin_urls; - - std::string m_bulletin; - std::chrono::system_clock::time_point m_bulletin_download_time; - - // Update the cache file with the current m_bulletin - void update_cache(const char* cache_filename); - - - /* Remote control */ - virtual void set_parameter(const std::string& parameter, - const std::string& value); - - /* Getting a parameter always returns a string. */ - virtual const std::string get_parameter(const std::string& parameter) const; -}; - diff --git a/src/Log.cpp b/src/Log.cpp deleted file mode 100644 index 6b78fe0..0000000 --- a/src/Log.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 - Her Majesty the Queen in Right of Canada (Communications Research - Center Canada) - - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include -#include -#include - -#include "Log.h" - -using namespace std; - -/* etiLog is a singleton used in all parts of ODR-DabMod to output log messages. - */ -Logger etiLog; - -void Logger::register_backend(std::shared_ptr backend) -{ - backends.push_back(backend); -} - - -void Logger::log(log_level_t level, const char* fmt, ...) -{ - if (level == discard) { - return; - } - - int size = 100; - std::string str; - va_list ap; - while (1) { - str.resize(size); - va_start(ap, fmt); - int n = vsnprintf((char *)str.c_str(), size, fmt, ap); - va_end(ap); - if (n > -1 && n < size) { - str.resize(n); - break; - } - if (n > -1) - size = n + 1; - else - size *= 2; - } - - logstr(level, move(str)); -} - -void Logger::logstr(log_level_t level, std::string&& message) -{ - if (level == discard) { - return; - } - - /* Remove a potential trailing newline. - * It doesn't look good in syslog - */ - if (message[message.length()-1] == '\n') { - message.resize(message.length()-1); - } - - for (auto &backend : backends) { - backend->log(level, message); - } - - { - std::lock_guard guard(m_cerr_mutex); - std::cerr << levels_as_str[level] << " " << message << std::endl; - } -} - - -LogLine Logger::level(log_level_t level) -{ - return LogLine(this, level); -} - -LogToFile::LogToFile(const std::string& filename) : name("FILE") -{ - FILE* fd = fopen(filename.c_str(), "a"); - if (fd == nullptr) { - fprintf(stderr, "Cannot open log file !"); - throw std::runtime_error("Cannot open log file !"); - } - - log_file.reset(fd); -} - -void LogToFile::log(log_level_t level, const std::string& message) -{ - if (level != log_level_t::discard) { - const char* log_level_text[] = { - "DEBUG", "INFO", "WARN", "ERROR", "ALERT", "EMERG"}; - - // fprintf is thread-safe - fprintf(log_file.get(), SYSLOG_IDENT ": %s: %s\n", - log_level_text[(size_t)level], message.c_str()); - fflush(log_file.get()); - } -} - -void LogToSyslog::log(log_level_t level, const std::string& message) -{ - if (level != log_level_t::discard) { - int syslog_level = LOG_EMERG; - switch (level) { - case debug: syslog_level = LOG_DEBUG; break; - case info: syslog_level = LOG_INFO; break; - /* we don't have the notice level */ - case warn: syslog_level = LOG_WARNING; break; - case error: syslog_level = LOG_ERR; break; - default: syslog_level = LOG_CRIT; break; - case alert: syslog_level = LOG_ALERT; break; - case emerg: syslog_level = LOG_EMERG; break; - } - - syslog(syslog_level, SYSLOG_IDENT " %s", message.c_str()); - } -} diff --git a/src/Log.h b/src/Log.h deleted file mode 100644 index 18f8c99..0000000 --- a/src/Log.h +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 - Her Majesty the Queen in Right of Canada (Communications Research - Center Canada) - - Copyright (C) 2018 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SYSLOG_IDENT "ODR-DabMux" -#define SYSLOG_FACILITY LOG_LOCAL0 - -enum log_level_t {debug = 0, info, warn, error, alert, emerg, discard}; - -static const std::string levels_as_str[] = - { " ", " ", "WARN ", "ERROR", "ALERT", "EMERG", "-----"} ; - -/** Abstract class all backends must inherit from */ -class LogBackend { - public: - virtual ~LogBackend() {}; - virtual void log(log_level_t level, const std::string& message) = 0; - virtual std::string get_name() const = 0; -}; - -/** A Logging backend for Syslog */ -class LogToSyslog : public LogBackend { - public: - LogToSyslog() : name("SYSLOG") { - openlog(SYSLOG_IDENT, LOG_PID, SYSLOG_FACILITY); - } - - virtual ~LogToSyslog() { - closelog(); - } - - void log(log_level_t level, const std::string& message); - - std::string get_name() const { return name; } - - private: - const std::string name; - - LogToSyslog(const LogToSyslog& other) = delete; - const LogToSyslog& operator=(const LogToSyslog& other) = delete; -}; - -class LogToFile : public LogBackend { - public: - LogToFile(const std::string& filename); - void log(log_level_t level, const std::string& message); - std::string get_name() const { return name; } - - private: - const std::string name; - - struct FILEDeleter{ void operator()(FILE* fd){ if(fd) fclose(fd);}}; - std::unique_ptr log_file; - - LogToFile(const LogToFile& other) = delete; - const LogToFile& operator=(const LogToFile& other) = delete; -}; - -class LogLine; - -class Logger { - public: - void register_backend(std::shared_ptr backend); - - /* Log the message to all backends */ - void log(log_level_t level, const char* fmt, ...); - - void logstr(log_level_t level, std::string&& message); - - /* Return a LogLine for the given level - * so that you can write etiLog.level(info) << "stuff = " << 21 */ - LogLine level(log_level_t level); - - private: - std::list > backends; - - std::mutex m_cerr_mutex; -}; - -extern Logger etiLog; - -// Accumulate a line of logs, using same syntax as stringstream -// The line is logged when the LogLine gets destroyed -class LogLine { - public: - LogLine(const LogLine& logline); - const LogLine& operator=(const LogLine& other) = delete; - LogLine(Logger* logger, log_level_t level) : - logger_(logger) - { - level_ = level; - } - - // Push the new element into the stringstream - template - LogLine& operator<<(T s) { - if (level_ != discard) { - os << s; - } - return *this; - } - - ~LogLine() - { - if (level_ != discard) { - logger_->logstr(level_, os.str()); - } - } - - private: - std::ostringstream os; - log_level_t level_; - Logger* logger_; -}; - - diff --git a/src/RemoteControl.cpp b/src/RemoteControl.cpp deleted file mode 100644 index b32c21a..0000000 --- a/src/RemoteControl.cpp +++ /dev/null @@ -1,595 +0,0 @@ -/* - Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 - Her Majesty the Queen in Right of Canada (Communications Research - Center Canada) - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ -#include -#include -#include -#include -#include -#include - -#include "RemoteControl.h" - -using boost::asio::ip::tcp; -using namespace std; - -RemoteControllers rcs; - -RemoteControllerTelnet::~RemoteControllerTelnet() -{ - m_active = false; - m_io_service.stop(); - - if (m_restarter_thread.joinable()) { - m_restarter_thread.join(); - } - - if (m_child_thread.joinable()) { - m_child_thread.join(); - } -} - -void RemoteControllerTelnet::restart() -{ - if (m_restarter_thread.joinable()) { - m_restarter_thread.join(); - } - - m_restarter_thread = std::thread( - &RemoteControllerTelnet::restart_thread, - this, 0); -} - -RemoteControllable::~RemoteControllable() { - rcs.remove_controllable(this); -} - -std::list RemoteControllable::get_supported_parameters() const { - std::list parameterlist; - for (const auto& param : m_parameters) { - parameterlist.push_back(param[0]); - } - return parameterlist; -} - -void RemoteControllers::add_controller(std::shared_ptr rc) { - m_controllers.push_back(rc); -} - -void RemoteControllers::enrol(RemoteControllable *rc) { - controllables.push_back(rc); -} - -void RemoteControllers::remove_controllable(RemoteControllable *rc) { - controllables.remove(rc); -} - -std::list< std::vector > RemoteControllers::get_param_list_values(const std::string& name) { - RemoteControllable* controllable = get_controllable_(name); - - std::list< std::vector > allparams; - for (auto ¶m : controllable->get_supported_parameters()) { - std::vector item; - item.push_back(param); - try { - item.push_back(controllable->get_parameter(param)); - } - catch (const ParameterError &e) { - item.push_back(std::string("error: ") + e.what()); - } - - allparams.push_back(item); - } - return allparams; -} - -std::string RemoteControllers::get_param(const std::string& name, const std::string& param) { - RemoteControllable* controllable = get_controllable_(name); - return controllable->get_parameter(param); -} - -void RemoteControllers::check_faults() { - for (auto &controller : m_controllers) { - if (controller->fault_detected()) { - etiLog.level(warn) << - "Detected Remote Control fault, restarting it"; - controller->restart(); - } - } -} - -RemoteControllable* RemoteControllers::get_controllable_(const std::string& name) -{ - auto rc = std::find_if(controllables.begin(), controllables.end(), - [&](RemoteControllable* r) { return r->get_rc_name() == name; }); - - if (rc == controllables.end()) { - throw ParameterError("Module name unknown"); - } - else { - return *rc; - } -} - -void RemoteControllers::set_param( - const std::string& name, - const std::string& param, - const std::string& value) -{ - etiLog.level(info) << "RC: Setting " << name << " " << param - << " to " << value; - RemoteControllable* controllable = get_controllable_(name); - try { - return controllable->set_parameter(param, value); - } - catch (const ios_base::failure& e) { - etiLog.level(info) << "RC: Failed to set " << name << " " << param - << " to " << value << ": " << e.what(); - throw ParameterError("Cannot understand value"); - } -} - -// This runs in a separate thread, because -// it would take too long to be done in the main loop -// thread. -void RemoteControllerTelnet::restart_thread(long) -{ - m_active = false; - m_io_service.stop(); - - if (m_child_thread.joinable()) { - m_child_thread.join(); - } - - m_child_thread = std::thread(&RemoteControllerTelnet::process, this, 0); -} - -void RemoteControllerTelnet::handle_accept( - const boost::system::error_code& boost_error, - boost::shared_ptr< boost::asio::ip::tcp::socket > socket, - boost::asio::ip::tcp::acceptor& acceptor) -{ - - const std::string welcome = "ODR-DabMux Remote Control CLI\n" - "Write 'help' for help.\n" - "**********\n"; - const std::string prompt = "> "; - - std::string in_message; - size_t length; - - if (boost_error) { - etiLog.level(error) << "RC: Error accepting connection"; - return; - } - - try { - etiLog.level(info) << "RC: Accepted"; - - boost::system::error_code ignored_error; - - boost::asio::write(*socket, boost::asio::buffer(welcome), - boost::asio::transfer_all(), - ignored_error); - - while (m_active && in_message != "quit") { - boost::asio::write(*socket, boost::asio::buffer(prompt), - boost::asio::transfer_all(), - ignored_error); - - in_message = ""; - - boost::asio::streambuf buffer; - length = boost::asio::read_until(*socket, buffer, "\n", ignored_error); - - std::istream str(&buffer); - std::getline(str, in_message); - - if (length == 0) { - etiLog.level(info) << "RC: Connection terminated"; - break; - } - - while (in_message.length() > 0 && - (in_message[in_message.length()-1] == '\r' || - in_message[in_message.length()-1] == '\n')) { - in_message.erase(in_message.length()-1, 1); - } - - if (in_message.length() == 0) { - continue; - } - - etiLog.level(info) << "RC: Got message '" << in_message << "'"; - - dispatch_command(*socket, in_message); - } - etiLog.level(info) << "RC: Closing socket"; - socket->close(); - } - catch (const std::exception& e) - { - etiLog.level(error) << "Remote control caught exception: " << e.what(); - } -} - -void RemoteControllerTelnet::process(long) -{ - m_active = true; - - while (m_active) { - m_io_service.reset(); - - tcp::acceptor acceptor(m_io_service, tcp::endpoint( - boost::asio::ip::address::from_string("127.0.0.1"), m_port) ); - - - // Add a job to start accepting connections. - boost::shared_ptr socket( - new tcp::socket(acceptor.get_io_service())); - - // Add an accept call to the service. This will prevent io_service::run() - // from returning. - etiLog.level(info) << "RC: Waiting for connection on port " << m_port; - acceptor.async_accept(*socket, - boost::bind(&RemoteControllerTelnet::handle_accept, - this, - boost::asio::placeholders::error, - socket, - boost::ref(acceptor))); - - // Process event loop. - m_io_service.run(); - } - - etiLog.level(info) << "RC: Leaving"; - m_fault = true; -} - -static std::vector tokenise(const std::string& message) { - stringstream ss(message); - std::vector all_tokens; - std::string item; - - while (std::getline(ss, item, ' ')) { - all_tokens.push_back(move(item)); - } - return all_tokens; -} - - -void RemoteControllerTelnet::dispatch_command(tcp::socket& socket, string command) -{ - vector cmd = tokenise(command); - - if (cmd[0] == "help") { - reply(socket, - "The following commands are supported:\n" - " list\n" - " * Lists the modules that are loaded and their parameters\n" - " show MODULE\n" - " * Lists all parameters and their values from module MODULE\n" - " get MODULE PARAMETER\n" - " * Gets the value for the specified PARAMETER from module MODULE\n" - " set MODULE PARAMETER VALUE\n" - " * Sets the value for the PARAMETER ofr module MODULE\n" - " quit\n" - " * Terminate this session\n" - "\n"); - } - else if (cmd[0] == "list") { - stringstream ss; - - if (cmd.size() == 1) { - for (auto &controllable : rcs.controllables) { - ss << controllable->get_rc_name() << endl; - - list< vector > params = controllable->get_parameter_descriptions(); - for (auto ¶m : params) { - ss << "\t" << param[0] << " : " << param[1] << endl; - } - } - } - else { - reply(socket, "Too many arguments for command 'list'"); - } - - reply(socket, ss.str()); - } - else if (cmd[0] == "show") { - if (cmd.size() == 2) { - try { - stringstream ss; - list< vector > r = rcs.get_param_list_values(cmd[1]); - for (auto ¶m_val : r) { - ss << param_val[0] << ": " << param_val[1] << endl; - } - reply(socket, ss.str()); - - } - catch (const ParameterError &e) { - reply(socket, e.what()); - } - } - else { - reply(socket, "Incorrect parameters for command 'show'"); - } - } - else if (cmd[0] == "get") { - if (cmd.size() == 3) { - try { - string r = rcs.get_param(cmd[1], cmd[2]); - reply(socket, r); - } - catch (const ParameterError &e) { - reply(socket, e.what()); - } - } - else { - reply(socket, "Incorrect parameters for command 'get'"); - } - } - else if (cmd[0] == "set") { - if (cmd.size() >= 4) { - try { - stringstream new_param_value; - for (size_t i = 3; i < cmd.size(); i++) { - new_param_value << cmd[i]; - - if (i+1 < cmd.size()) { - new_param_value << " "; - } - } - - rcs.set_param(cmd[1], cmd[2], new_param_value.str()); - reply(socket, "ok"); - } - catch (const ParameterError &e) { - reply(socket, e.what()); - } - catch (const exception &e) { - reply(socket, "Error: Invalid parameter value. "); - } - } - else { - reply(socket, "Incorrect parameters for command 'set'"); - } - } - else if (cmd[0] == "quit") { - reply(socket, "Goodbye"); - } - else { - reply(socket, "Message not understood"); - } -} - -void RemoteControllerTelnet::reply(tcp::socket& socket, string message) -{ - boost::system::error_code ignored_error; - stringstream ss; - ss << message << "\r\n"; - boost::asio::write(socket, boost::asio::buffer(ss.str()), - boost::asio::transfer_all(), - ignored_error); -} - - -#if defined(HAVE_RC_ZEROMQ) - -RemoteControllerZmq::~RemoteControllerZmq() { - m_active = false; - m_fault = false; - - if (m_restarter_thread.joinable()) { - m_restarter_thread.join(); - } - - if (m_child_thread.joinable()) { - m_child_thread.join(); - } -} - -void RemoteControllerZmq::restart() -{ - if (m_restarter_thread.joinable()) { - m_restarter_thread.join(); - } - - m_restarter_thread = std::thread(&RemoteControllerZmq::restart_thread, this); -} - -// This runs in a separate thread, because -// it would take too long to be done in the main loop -// thread. -void RemoteControllerZmq::restart_thread() -{ - m_active = false; - - if (m_child_thread.joinable()) { - m_child_thread.join(); - } - - m_child_thread = std::thread(&RemoteControllerZmq::process, this); -} - -void RemoteControllerZmq::recv_all(zmq::socket_t& pSocket, std::vector &message) -{ - bool more = true; - do { - zmq::message_t msg; - pSocket.recv(&msg); - std::string incoming((char*)msg.data(), msg.size()); - message.push_back(incoming); - more = msg.more(); - } while (more); -} - -void RemoteControllerZmq::send_ok_reply(zmq::socket_t &pSocket) -{ - zmq::message_t msg(2); - char repCode[2] = {'o', 'k'}; - memcpy ((void*) msg.data(), repCode, 2); - pSocket.send(msg, 0); -} - -void RemoteControllerZmq::send_fail_reply(zmq::socket_t &pSocket, const std::string &error) -{ - zmq::message_t msg1(4); - char repCode[4] = {'f', 'a', 'i', 'l'}; - memcpy ((void*) msg1.data(), repCode, 4); - pSocket.send(msg1, ZMQ_SNDMORE); - - zmq::message_t msg2(error.length()); - memcpy ((void*) msg2.data(), error.c_str(), error.length()); - pSocket.send(msg2, 0); -} - -void RemoteControllerZmq::process() -{ - m_fault = false; - - // create zmq reply socket for receiving ctrl parameters - try { - zmq::socket_t repSocket(m_zmqContext, ZMQ_REP); - - // connect the socket - int hwm = 100; - int linger = 0; - repSocket.setsockopt(ZMQ_RCVHWM, &hwm, sizeof(hwm)); - repSocket.setsockopt(ZMQ_SNDHWM, &hwm, sizeof(hwm)); - repSocket.setsockopt(ZMQ_LINGER, &linger, sizeof(linger)); - repSocket.bind(m_endpoint.c_str()); - - // create pollitem that polls the ZMQ sockets - zmq::pollitem_t pollItems[] = { {repSocket, 0, ZMQ_POLLIN, 0} }; - while (m_active) { - zmq::poll(pollItems, 1, 100); - std::vector msg; - - if (pollItems[0].revents & ZMQ_POLLIN) { - recv_all(repSocket, msg); - - std::string command((char*)msg[0].data(), msg[0].size()); - - if (msg.size() == 1 && command == "ping") { - send_ok_reply(repSocket); - } - else if (msg.size() == 1 && command == "list") { - size_t cohort_size = rcs.controllables.size(); - for (auto &controllable : rcs.controllables) { - std::stringstream ss; - ss << "{ \"name\": \"" << controllable->get_rc_name() << "\"," << - " \"params\": { "; - - list< vector > params = controllable->get_parameter_descriptions(); - size_t i = 0; - for (auto ¶m : params) { - if (i > 0) { - ss << ", "; - } - - ss << "\"" << param[0] << "\": " << - "\"" << param[1] << "\""; - - i++; - } - - ss << " } }"; - - std::string msg_s = ss.str(); - - zmq::message_t zmsg(ss.str().size()); - memcpy ((void*) zmsg.data(), msg_s.data(), msg_s.size()); - - int flag = (--cohort_size > 0) ? ZMQ_SNDMORE : 0; - repSocket.send(zmsg, flag); - } - } - else if (msg.size() == 2 && command == "show") { - std::string module((char*) msg[1].data(), msg[1].size()); - try { - list< vector > r = rcs.get_param_list_values(module); - size_t r_size = r.size(); - for (auto ¶m_val : r) { - std::stringstream ss; - ss << param_val[0] << ": " << param_val[1] << endl; - zmq::message_t zmsg(ss.str().size()); - memcpy(zmsg.data(), ss.str().data(), ss.str().size()); - - int flag = (--r_size > 0) ? ZMQ_SNDMORE : 0; - repSocket.send(zmsg, flag); - } - } - catch (const ParameterError &err) { - send_fail_reply(repSocket, err.what()); - } - } - else if (msg.size() == 3 && command == "get") { - std::string module((char*) msg[1].data(), msg[1].size()); - std::string parameter((char*) msg[2].data(), msg[2].size()); - - try { - std::string value = rcs.get_param(module, parameter); - zmq::message_t zmsg(value.size()); - memcpy ((void*) zmsg.data(), value.data(), value.size()); - repSocket.send(zmsg, 0); - } - catch (const ParameterError &err) { - send_fail_reply(repSocket, err.what()); - } - } - else if (msg.size() == 4 && command == "set") { - std::string module((char*) msg[1].data(), msg[1].size()); - std::string parameter((char*) msg[2].data(), msg[2].size()); - std::string value((char*) msg[3].data(), msg[3].size()); - - try { - rcs.set_param(module, parameter, value); - send_ok_reply(repSocket); - } - catch (const ParameterError &err) { - send_fail_reply(repSocket, err.what()); - } - } - else { - send_fail_reply(repSocket, - "Unsupported command. commands: list, show, get, set"); - } - } - } - repSocket.close(); - } - catch (const zmq::error_t &e) { - etiLog.level(error) << "ZMQ RC error: " << std::string(e.what()); - } - catch (const std::exception& e) { - etiLog.level(error) << "ZMQ RC caught exception: " << e.what(); - m_fault = true; - } -} - -#endif - diff --git a/src/RemoteControl.h b/src/RemoteControl.h deleted file mode 100644 index 0726b28..0000000 --- a/src/RemoteControl.h +++ /dev/null @@ -1,263 +0,0 @@ -/* - Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 - Her Majesty the Queen in Right of Canada (Communications Research - Center Canada) - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - This module adds remote-control capability to some of the dabmux modules. - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#if defined(HAVE_RC_ZEROMQ) -# include "zmq.hpp" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Log.h" - -#define RC_ADD_PARAMETER(p, desc) { \ - std::vector p; \ - p.push_back(#p); \ - p.push_back(desc); \ - m_parameters.push_back(p); \ -} - -class ParameterError : public std::exception -{ - public: - ParameterError(std::string message) : m_message(message) {} - ~ParameterError() throw() {} - const char* what() const throw() { return m_message.c_str(); } - - private: - std::string m_message; -}; - -class RemoteControllable; - -/* Remote controllers (that recieve orders from the user) - * must implement BaseRemoteController - */ -class BaseRemoteController { - public: - /* When this returns one, the remote controller cannot be - * used anymore, and must be restarted by dabmux - */ - virtual bool fault_detected() = 0; - - /* In case of a fault, the remote controller can be - * restarted. - */ - virtual void restart() = 0; - - virtual ~BaseRemoteController() {} -}; - -/* Objects that support remote control must implement the following class */ -class RemoteControllable { - public: - RemoteControllable(const std::string& name) : - m_rc_name(name) {} - - RemoteControllable(const RemoteControllable& other) = delete; - RemoteControllable& operator=(const RemoteControllable& other) = delete; - - virtual ~RemoteControllable(); - - /* return a short name used to identify the controllable. - * It might be used in the commands the user has to type, so keep - * it short - */ - virtual std::string get_rc_name() const { return m_rc_name; } - - /* Return a list of possible parameters that can be set */ - virtual std::list get_supported_parameters() const; - - /* Return a mapping of the descriptions of all parameters */ - virtual std::list< std::vector > - get_parameter_descriptions() const - { - return m_parameters; - } - - /* Base function to set parameters. */ - virtual void set_parameter( - const std::string& parameter, - const std::string& value) = 0; - - /* Getting a parameter always returns a string. */ - virtual const std::string get_parameter(const std::string& parameter) const = 0; - - protected: - std::string m_rc_name; - std::list< std::vector > m_parameters; -}; - -/* Holds all our remote controllers and controlled object. - */ -class RemoteControllers { - public: - void add_controller(std::shared_ptr rc); - void enrol(RemoteControllable *rc); - void remove_controllable(RemoteControllable *rc); - void check_faults(); - std::list< std::vector > get_param_list_values(const std::string& name); - std::string get_param(const std::string& name, const std::string& param); - - void set_param( - const std::string& name, - const std::string& param, - const std::string& value); - - std::list controllables; - - private: - RemoteControllable* get_controllable_(const std::string& name); - - std::list > m_controllers; -}; - -extern RemoteControllers rcs; - -/* Implements a Remote controller based on a simple telnet CLI - * that listens on localhost - */ -class RemoteControllerTelnet : public BaseRemoteController { - public: - RemoteControllerTelnet() - : m_active(false), - m_io_service(), - m_fault(false), - m_port(0) { } - - RemoteControllerTelnet(int port) - : m_active(port > 0), - m_io_service(), - m_fault(false), - m_port(port) - { - restart(); - } - - - RemoteControllerTelnet& operator=(const RemoteControllerTelnet& other) = delete; - RemoteControllerTelnet(const RemoteControllerTelnet& other) = delete; - - ~RemoteControllerTelnet(); - - virtual bool fault_detected() { return m_fault; } - - virtual void restart(); - - private: - void restart_thread(long); - - void process(long); - - void dispatch_command(boost::asio::ip::tcp::socket& socket, - std::string command); - - void reply(boost::asio::ip::tcp::socket& socket, std::string message); - - void handle_accept( - const boost::system::error_code& boost_error, - boost::shared_ptr< boost::asio::ip::tcp::socket > socket, - boost::asio::ip::tcp::acceptor& acceptor); - - std::atomic m_active; - - boost::asio::io_service m_io_service; - - /* This is set to true if a fault occurred */ - std::atomic m_fault; - std::thread m_restarter_thread; - - std::thread m_child_thread; - - int m_port; -}; - -#if defined(HAVE_RC_ZEROMQ) -/* Implements a Remote controller using zmq transportlayer - * that listens on localhost - */ -class RemoteControllerZmq : public BaseRemoteController { - public: - RemoteControllerZmq() - : m_active(false), m_fault(false), - m_zmqContext(1), - m_endpoint("") { } - - RemoteControllerZmq(const std::string& endpoint) - : m_active(not endpoint.empty()), m_fault(false), - m_zmqContext(1), - m_endpoint(endpoint), - m_child_thread(&RemoteControllerZmq::process, this) { } - - RemoteControllerZmq& operator=(const RemoteControllerZmq& other) = delete; - RemoteControllerZmq(const RemoteControllerZmq& other) = delete; - - ~RemoteControllerZmq(); - - virtual bool fault_detected() { return m_fault; } - - virtual void restart(); - - private: - void restart_thread(); - - void recv_all(zmq::socket_t &pSocket, std::vector &message); - void send_ok_reply(zmq::socket_t &pSocket); - void send_fail_reply(zmq::socket_t &pSocket, const std::string &error); - void process(); - - std::atomic m_active; - - /* This is set to true if a fault occurred */ - std::atomic m_fault; - std::thread m_restarter_thread; - - zmq::context_t m_zmqContext; - - std::string m_endpoint; - std::thread m_child_thread; -}; -#endif - diff --git a/src/crc.c b/src/crc.c deleted file mode 100644 index cc02473..0000000 --- a/src/crc.c +++ /dev/null @@ -1,266 +0,0 @@ -/* - Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the - Queen in Right of Canada (Communications Research Center Canada) - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "crc.h" -#ifndef _WIN32 -# include -# include -#endif -#include -#include - -//#define CCITT 0x1021 - -uint8_t crc8tab[256] = { - 0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, - 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d, - 0x70, 0x77, 0x7e, 0x79, 0x6c, 0x6b, 0x62, 0x65, - 0x48, 0x4f, 0x46, 0x41, 0x54, 0x53, 0x5a, 0x5d, - 0xe0, 0xe7, 0xee, 0xe9, 0xfc, 0xfb, 0xf2, 0xf5, - 0xd8, 0xdf, 0xd6, 0xd1, 0xc4, 0xc3, 0xca, 0xcd, - 0x90, 0x97, 0x9e, 0x99, 0x8c, 0x8b, 0x82, 0x85, - 0xa8, 0xaf, 0xa6, 0xa1, 0xb4, 0xb3, 0xba, 0xbd, - 0xc7, 0xc0, 0xc9, 0xce, 0xdb, 0xdc, 0xd5, 0xd2, - 0xff, 0xf8, 0xf1, 0xf6, 0xe3, 0xe4, 0xed, 0xea, - 0xb7, 0xb0, 0xb9, 0xbe, 0xab, 0xac, 0xa5, 0xa2, - 0x8f, 0x88, 0x81, 0x86, 0x93, 0x94, 0x9d, 0x9a, - 0x27, 0x20, 0x29, 0x2e, 0x3b, 0x3c, 0x35, 0x32, - 0x1f, 0x18, 0x11, 0x16, 0x03, 0x04, 0x0d, 0x0a, - 0x57, 0x50, 0x59, 0x5e, 0x4b, 0x4c, 0x45, 0x42, - 0x6f, 0x68, 0x61, 0x66, 0x73, 0x74, 0x7d, 0x7a, - 0x89, 0x8e, 0x87, 0x80, 0x95, 0x92, 0x9b, 0x9c, - 0xb1, 0xb6, 0xbf, 0xb8, 0xad, 0xaa, 0xa3, 0xa4, - 0xf9, 0xfe, 0xf7, 0xf0, 0xe5, 0xe2, 0xeb, 0xec, - 0xc1, 0xc6, 0xcf, 0xc8, 0xdd, 0xda, 0xd3, 0xd4, - 0x69, 0x6e, 0x67, 0x60, 0x75, 0x72, 0x7b, 0x7c, - 0x51, 0x56, 0x5f, 0x58, 0x4d, 0x4a, 0x43, 0x44, - 0x19, 0x1e, 0x17, 0x10, 0x05, 0x02, 0x0b, 0x0c, - 0x21, 0x26, 0x2f, 0x28, 0x3d, 0x3a, 0x33, 0x34, - 0x4e, 0x49, 0x40, 0x47, 0x52, 0x55, 0x5c, 0x5b, - 0x76, 0x71, 0x78, 0x7f, 0x6a, 0x6d, 0x64, 0x63, - 0x3e, 0x39, 0x30, 0x37, 0x22, 0x25, 0x2c, 0x2b, - 0x06, 0x01, 0x08, 0x0f, 0x1a, 0x1d, 0x14, 0x13, - 0xae, 0xa9, 0xa0, 0xa7, 0xb2, 0xb5, 0xbc, 0xbb, - 0x96, 0x91, 0x98, 0x9f, 0x8a, 0x8d, 0x84, 0x83, - 0xde, 0xd9, 0xd0, 0xd7, 0xc2, 0xc5, 0xcc, 0xcb, - 0xe6, 0xe1, 0xe8, 0xef, 0xfa, 0xfd, 0xf4, 0xf3 -}; - - -uint16_t crc16tab[256] = { - 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, - 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, - 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, - 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, - 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, - 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, - 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, - 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, - 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, - 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, - 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, - 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, - 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, - 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, - 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, - 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, - 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, - 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, - 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, - 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, - 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, - 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, - 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, - 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, - 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, - 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, - 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, - 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, - 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, - 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, - 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, - 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 -}; - - -uint32_t crc32tab[256] = { - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, - 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, - 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, - 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, - 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, - 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, - 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, - 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, - 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, - 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, - 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, - 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, - 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, - 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, - 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, - 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, - 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, - 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, - 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, - 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, - 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, - 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, - 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, - 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, - 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, - 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, - 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, - 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, - 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, - 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, - 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, - 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, - 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, - 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, - 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, - 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, - 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, - 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, - 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, - 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, - 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, - 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, - 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, - 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, - 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, - 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, - 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, - 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, - 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, - 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, - 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, - 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, - 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, - 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 -}; - -// This function can be used to create a new table with a different polynom -void init_crc8tab(uint8_t l_code, uint8_t l_init) -{ - unsigned i, j, msb; - uint8_t nb; - uint8_t crc; - - for (i = 0; i < 256; ++i) { - crc = l_init; - nb = i ^ 0xff; - for (j = 0; j < 8; ++j) { - msb = (nb & (0x80 >> j)) && 1; - msb ^= (crc >> 7); - crc <<= 1; - if (msb) - crc ^= l_code; - } - crc8tab[i] = crc; - } -} - - -void init_crc16tab(uint16_t l_code, uint16_t l_init) -{ - unsigned i, j, msb; - uint8_t nb; - uint16_t crc; - - for (i = 0; i < 256; ++i) { - crc = l_init; - nb = i ^ 0xff; - for (j = 0; j < 8; ++j) { - msb = (nb & (0x80 >> j)) && 1; - msb ^= (crc >> 15); - crc <<= 1; - if (msb) - crc ^= l_code; - } - crc ^= 0xff00; - crc16tab[i] = crc; - } -} - - -void init_crc32tab(uint32_t l_code, uint32_t l_init) -{ - unsigned i, j, msb; - uint8_t nb; - uint32_t crc; - - for (i = 0; i < 256; ++i) { - crc = l_init; - nb = i ^ 0xff; - for (j = 0; j < 8; ++j) { - msb = (nb & (0x80 >> j)) && 1; - msb ^= (crc >> 31); - crc <<= 1; - if (msb) - crc ^= l_code; - } - crc ^= 0xffffff00; - crc32tab[i] = crc; - } -} - - -uint8_t crc8(uint8_t l_crc, const void *lp_data, unsigned l_nb) -{ - const uint8_t* data = (const uint8_t*)lp_data; - while (l_nb--) { - l_crc = crc8tab[l_crc ^ *(data++)]; - } - return (l_crc); -} - - -uint16_t crc16(uint16_t l_crc, const void *lp_data, unsigned l_nb) -{ - const uint8_t* data = (const uint8_t*)lp_data; - while (l_nb--) { - l_crc = - (l_crc << 8) ^ crc16tab[(l_crc >> 8) ^ *(data++)]; - } - return (l_crc); -} - - -uint32_t crc32(uint32_t l_crc, const void *lp_data, unsigned l_nb) -{ - const uint8_t* data = (const uint8_t*)lp_data; - while (l_nb--) { - l_crc = - (l_crc << 8) ^ crc32tab[((l_crc >> 24) ^ *(data++)) & 0xff]; - } - return (l_crc); -} diff --git a/src/crc.h b/src/crc.h deleted file mode 100644 index b1785a1..0000000 --- a/src/crc.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Her Majesty the - Queen in Right of Canada (Communications Research Center Canada) - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#ifndef _CRC -#define _CRC - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#ifndef _WIN32 - #include -#else - #include // For types... - typedef BYTE uint8_t; - typedef WORD uint16_t; - typedef DWORD32 uint32_t; -#endif - - -#ifdef __cplusplus -extern "C" { // } -#endif - -void init_crc8tab(uint8_t l_code, uint8_t l_init); -uint8_t crc8(uint8_t l_crc, const void *lp_data, unsigned l_nb); -extern uint8_t crc8tab[]; - -void init_crc16tab(uint16_t l_code, uint16_t l_init); -uint16_t crc16(uint16_t l_crc, const void *lp_data, unsigned l_nb); -extern uint16_t crc16tab[]; - -void init_crc32tab(uint32_t l_code, uint32_t l_init); -uint32_t crc32(uint32_t l_crc, const void *lp_data, unsigned l_nb); -extern uint32_t crc32tab[]; - -#ifdef __cplusplus -} -#endif - -#endif //_CRC -- cgit v1.2.3 From 5ee85c4ac41337e383eb1a735bc05f1e5d46a98f Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Mon, 19 Aug 2019 17:18:51 +0200 Subject: Use EDI output from odr-mmbtools-common --- Makefile.am | 52 +++--- lib/ClockTAI.cpp | 159 ++++++++++------ lib/ClockTAI.h | 10 +- lib/edioutput/AFPacket.cpp | 96 ++++++++++ lib/edioutput/AFPacket.h | 61 ++++++ lib/edioutput/Config.h | 84 +++++++++ lib/edioutput/Interleaver.cpp | 122 ++++++++++++ lib/edioutput/Interleaver.h | 75 ++++++++ lib/edioutput/PFT.cpp | 327 ++++++++++++++++++++++++++++++++ lib/edioutput/PFT.h | 78 ++++++++ lib/edioutput/TagItems.cpp | 383 ++++++++++++++++++++++++++++++++++++++ lib/edioutput/TagItems.h | 229 +++++++++++++++++++++++ lib/edioutput/TagPacket.cpp | 78 ++++++++ lib/edioutput/TagPacket.h | 56 ++++++ lib/edioutput/Transport.cpp | 188 +++++++++++++++++++ lib/edioutput/Transport.h | 71 +++++++ src/DabMultiplexer.cpp | 2 +- src/DabMultiplexer.h | 8 +- src/dabOutput/edi/AFPacket.cpp | 96 ---------- src/dabOutput/edi/AFPacket.h | 61 ------ src/dabOutput/edi/Config.h | 84 --------- src/dabOutput/edi/Interleaver.cpp | 122 ------------ src/dabOutput/edi/Interleaver.h | 75 -------- src/dabOutput/edi/PFT.cpp | 327 -------------------------------- src/dabOutput/edi/PFT.h | 78 -------- src/dabOutput/edi/TagItems.cpp | 216 --------------------- src/dabOutput/edi/TagItems.h | 149 --------------- src/dabOutput/edi/TagPacket.cpp | 79 -------- src/dabOutput/edi/TagPacket.h | 56 ------ src/dabOutput/edi/Transport.cpp | 189 ------------------- src/dabOutput/edi/Transport.h | 71 ------- src/zmq2edi/EDISender.cpp | 2 +- src/zmq2edi/EDISender.h | 6 +- 33 files changed, 1990 insertions(+), 1700 deletions(-) create mode 100644 lib/edioutput/AFPacket.cpp create mode 100644 lib/edioutput/AFPacket.h create mode 100644 lib/edioutput/Config.h create mode 100644 lib/edioutput/Interleaver.cpp create mode 100644 lib/edioutput/Interleaver.h create mode 100644 lib/edioutput/PFT.cpp create mode 100644 lib/edioutput/PFT.h create mode 100644 lib/edioutput/TagItems.cpp create mode 100644 lib/edioutput/TagItems.h create mode 100644 lib/edioutput/TagPacket.cpp create mode 100644 lib/edioutput/TagPacket.h create mode 100644 lib/edioutput/Transport.cpp create mode 100644 lib/edioutput/Transport.h delete mode 100644 src/dabOutput/edi/AFPacket.cpp delete mode 100644 src/dabOutput/edi/AFPacket.h delete mode 100644 src/dabOutput/edi/Config.h delete mode 100644 src/dabOutput/edi/Interleaver.cpp delete mode 100644 src/dabOutput/edi/Interleaver.h delete mode 100644 src/dabOutput/edi/PFT.cpp delete mode 100644 src/dabOutput/edi/PFT.h delete mode 100644 src/dabOutput/edi/TagItems.cpp delete mode 100644 src/dabOutput/edi/TagItems.h delete mode 100644 src/dabOutput/edi/TagPacket.cpp delete mode 100644 src/dabOutput/edi/TagPacket.h delete mode 100644 src/dabOutput/edi/Transport.cpp delete mode 100644 src/dabOutput/edi/Transport.h (limited to 'lib') diff --git a/Makefile.am b/Makefile.am index 216f7c0..e426f74 100644 --- a/Makefile.am +++ b/Makefile.am @@ -81,19 +81,6 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ src/dabOutput/dabOutputZMQ.cpp \ src/dabOutput/metadata.h \ src/dabOutput/metadata.cpp \ - src/dabOutput/edi/AFPacket.cpp \ - src/dabOutput/edi/AFPacket.h \ - src/dabOutput/edi/Config.h \ - src/dabOutput/edi/Interleaver.cpp \ - src/dabOutput/edi/Interleaver.h \ - src/dabOutput/edi/PFT.cpp \ - src/dabOutput/edi/PFT.h \ - src/dabOutput/edi/TagItems.cpp \ - src/dabOutput/edi/TagItems.h \ - src/dabOutput/edi/TagPacket.cpp \ - src/dabOutput/edi/TagPacket.h \ - src/dabOutput/edi/Transport.cpp \ - src/dabOutput/edi/Transport.h \ src/ConfigParser.cpp \ src/ConfigParser.h \ src/Eti.h \ @@ -169,6 +156,19 @@ odr_dabmux_SOURCES =src/DabMux.cpp \ lib/edi/common.cpp \ lib/edi/common.h \ lib/edi/buffer_unpack.hpp \ + lib/edioutput/AFPacket.cpp \ + lib/edioutput/AFPacket.h \ + lib/edioutput/Config.h \ + lib/edioutput/Interleaver.cpp \ + lib/edioutput/Interleaver.h \ + lib/edioutput/PFT.cpp \ + lib/edioutput/PFT.h \ + lib/edioutput/TagItems.cpp \ + lib/edioutput/TagItems.h \ + lib/edioutput/TagPacket.cpp \ + lib/edioutput/TagPacket.h \ + lib/edioutput/Transport.cpp \ + lib/edioutput/Transport.h \ lib/ReedSolomon.h \ lib/ReedSolomon.cpp \ lib/Socket.h \ @@ -199,19 +199,19 @@ odr_zmq2edi_SOURCES = src/zmq2edi/zmq2edi.cpp \ src/dabOutput/dabOutput.h \ src/dabOutput/metadata.h \ src/dabOutput/metadata.cpp \ - src/dabOutput/edi/AFPacket.cpp \ - src/dabOutput/edi/AFPacket.h \ - src/dabOutput/edi/Config.h \ - src/dabOutput/edi/Interleaver.cpp \ - src/dabOutput/edi/Interleaver.h \ - src/dabOutput/edi/PFT.cpp \ - src/dabOutput/edi/PFT.h \ - src/dabOutput/edi/TagItems.cpp \ - src/dabOutput/edi/TagItems.h \ - src/dabOutput/edi/TagPacket.cpp \ - src/dabOutput/edi/TagPacket.h \ - src/dabOutput/edi/Transport.cpp \ - src/dabOutput/edi/Transport.h \ + lib/edioutput/AFPacket.cpp \ + lib/edioutput/AFPacket.h \ + lib/edioutput/Config.h \ + lib/edioutput/Interleaver.cpp \ + lib/edioutput/Interleaver.h \ + lib/edioutput/PFT.cpp \ + lib/edioutput/PFT.h \ + lib/edioutput/TagItems.cpp \ + lib/edioutput/TagItems.h \ + lib/edioutput/TagPacket.cpp \ + lib/edioutput/TagPacket.h \ + lib/edioutput/Transport.cpp \ + lib/edioutput/Transport.h \ lib/Log.h \ lib/Log.cpp \ lib/crc.h \ diff --git a/lib/ClockTAI.cpp b/lib/ClockTAI.cpp index 42497f4..2656345 100644 --- a/lib/ClockTAI.cpp +++ b/lib/ClockTAI.cpp @@ -9,27 +9,27 @@ http://www.opendigitalradio.org */ /* - This file is part of ODR-DabMux. + This file is part of the ODR-mmbTools. - ODR-DabMux is free software: you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - ODR-DabMux is distributed in the hope that it will be useful, + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . -*/ + along with this program. If not, see . + */ /* This file downloads the TAI-UTC bulletins from the from IETF and parses them * so that correct time can be communicated in EDI timestamps. * * This file contains self-test code that can be executed by running - * g++ -g -Wall -DTEST -DHAVE_CURL -std=c++11 -lcurl -pthread \ + * g++ -g -Wall -DTAI_TEST -DHAVE_CURL -std=c++11 -lcurl -pthread \ * ClockTAI.cpp Log.cpp RemoteControl.cpp -lboost_system -o taitest && ./taitest */ @@ -40,9 +40,9 @@ #include "ClockTAI.h" #include "Log.h" -#include -#include -#include +#include +#include +#include #if SUPPORT_SETTING_CLOCK_TAI # include #endif @@ -54,10 +54,13 @@ #include #include #include +#include +#include +#include using namespace std; -#ifdef TEST +#ifdef DOWNLOADED_IN_THE_PAST_TEST static bool wait_longer = true; #endif @@ -76,7 +79,7 @@ static array default_tai_urls = { // According to the Filesystem Hierarchy Standard, the data in // /var/tmp "must not be deleted when the system is booted." -static const char *tai_cache_location = "/var/tmp/odr-dabmux-leap-seconds.cache"; +static const char *tai_cache_location = "/var/tmp/odr-leap-seconds.cache"; // read TAI offset from a valid bulletin in IETF format static int parse_ietf_bulletin(const std::string& bulletin) @@ -127,7 +130,7 @@ static int parse_ietf_bulletin(const std::string& bulletin) tai_utc_offset = offset; tai_utc_offset_valid = true; } -#if TEST +#if TAI_TEST else { cerr << "IETF Ignoring offset " << bulletin_offset << " at TS " << bulletin_ntp_timestamp << @@ -183,7 +186,7 @@ static bulletin_state parse_bulletin(const string& bulletin) const int64_t expiry_unix = std::atoll(expiry_data_str.c_str()) - ntp_unix_offset; -#ifdef TEST +#ifdef TAI_TEST etiLog.level(info) << "Bulletin expires in " << expiry_unix - now; #endif ret.expiry = expiry_unix - now; @@ -246,17 +249,46 @@ static string download_tai_utc_bulletin(const char* url) static string load_bulletin_from_file(const char* cache_filename) { - // Clear the bulletin - ifstream f(cache_filename); - if (not f.good()) { - return {}; + int fd = open(cache_filename, O_RDWR); // lockf requires O_RDWR + if (fd == -1) { + etiLog.level(error) << "TAI-UTC bulletin open cache for reading: " << + strerror(errno); + return ""; } - stringstream ss; - ss << f.rdbuf(); - f.close(); + lseek(fd, 0, SEEK_SET); + + vector buf(1024); + vector new_bulletin_data; + + ssize_t ret = lockf(fd, F_LOCK, 0); + if (ret == 0) { + // exclusive lock acquired + + do { + ret = read(fd, buf.data(), buf.size()); + + if (ret == -1) { + close(fd); + etiLog.level(error) << "TAI-UTC bulletin read cache: " << + strerror(errno); + return ""; + } + + copy(buf.data(), buf.data() + ret, back_inserter(new_bulletin_data)); + } while (ret > 0); + + close(fd); - return ss.str(); + return string{new_bulletin_data.data(), new_bulletin_data.size()}; + } + else { + etiLog.level(error) << + "TAI-UTC bulletin acquire cache lock for reading: " << + strerror(errno); + close(fd); + } + return ""; } ClockTAI::ClockTAI(const std::vector& bulletin_urls) : @@ -289,7 +321,7 @@ int ClockTAI::get_valid_offset() const auto state = parse_bulletin(m_bulletin); if (state.usable()) { -#if TEST +#if TAI_TEST etiLog.level(info) << "Bulletin already valid"; #endif offset = state.offset; @@ -297,20 +329,25 @@ int ClockTAI::get_valid_offset() } else { const auto cache_bulletin = load_bulletin_from_file(tai_cache_location); +#if TAI_TEST + etiLog.level(info) << "Loaded cache bulletin with " << + std::count_if(cache_bulletin.cbegin(), cache_bulletin.cend(), + [](const char c){ return c == '\n'; }) << " lines"; +#endif const auto cache_state = parse_bulletin(cache_bulletin); if (cache_state.usable()) { m_bulletin = cache_bulletin; offset = cache_state.offset; offset_valid = true; -#if TEST +#if TAI_TEST etiLog.level(info) << "Bulletin from cache valid with offset=" << offset; #endif } else { for (const auto url : m_bulletin_urls) { try { -#if TEST +#if TAI_TEST etiLog.level(info) << "Load bulletin from " << url; #endif const auto new_bulletin = download_tai_utc_bulletin(url.c_str()); @@ -368,7 +405,7 @@ int ClockTAI::get_offset() std::unique_lock lock(m_data_mutex); if (not m_offset_valid) { -#ifdef TEST +#ifdef DOWNLOADED_IN_THE_PAST_TEST // Assume we've downloaded it in the past: m_offset = 37; // Valid in early 2017 @@ -418,7 +455,7 @@ int ClockTAI::get_offset() m_bulletin_download_time += hours(download_retry_interval_hours); } -#ifdef TEST +#ifdef DOWNLOADED_IN_THE_PAST_TEST wait_longer = false; #endif break; @@ -426,14 +463,14 @@ int ClockTAI::get_offset() case future_status::deferred: case future_status::timeout: // Not ready yet -#ifdef TEST +#ifdef TAI_TEST etiLog.level(debug) << " async not ready yet"; #endif break; } } else { -#ifdef TEST +#ifdef TAI_TEST etiLog.level(debug) << " Launch async"; #endif m_offset_future = async(launch::async, &ClockTAI::get_valid_offset, this); @@ -463,13 +500,45 @@ int ClockTAI::update_local_tai_clock(int offset) void ClockTAI::update_cache(const char* cache_filename) { - ofstream f(cache_filename); - if (not f.good()) { - throw runtime_error("TAI-UTC bulletin open cache for writing"); + int fd = open(cache_filename, O_RDWR | O_CREAT, 00664); + if (fd == -1) { + etiLog.level(error) << + "TAI-UTC bulletin open cache for writing: " << + strerror(errno); + return; } - f << m_bulletin; - f.close(); + lseek(fd, 0, SEEK_SET); + + ssize_t ret = lockf(fd, F_LOCK, 0); + if (ret == 0) { + // exclusive lock acquired + const char *data = m_bulletin.data(); + size_t remaining = m_bulletin.size(); + + while (remaining > 0) { + ret = write(fd, data, remaining); + if (ret == -1) { + close(fd); + etiLog.level(error) << + "TAI-UTC bulletin write cache: " << + strerror(errno); + return; + } + + remaining -= ret; + data += ret; + } + etiLog.level(debug) << "TAI-UTC bulletin cache updated"; + close(fd); + } + else { + close(fd); + etiLog.level(error) << + "TAI-UTC bulletin acquire cache lock for writing: " << + strerror(errno); + return; + } } @@ -536,27 +605,3 @@ void debug_tai_clk() } #endif -#if TEST -int main(int argc, char **argv) -{ - using namespace std; - - ClockTAI tai({}); - - while (wait_longer) { - try { - etiLog.level(info) << - "Offset is " << tai.get_offset(); - } - catch (const exception &e) { - etiLog.level(error) << - "Exception " << e.what(); - } - - this_thread::sleep_for(chrono::seconds(2)); - } - - return 0; -} -#endif - diff --git a/lib/ClockTAI.h b/lib/ClockTAI.h index bb85815..50a6323 100644 --- a/lib/ClockTAI.h +++ b/lib/ClockTAI.h @@ -9,21 +9,21 @@ http://www.opendigitalradio.org */ /* - This file is part of ODR-DabMux. + This file is part of the ODR-mmbTools. - ODR-DabMux is free software: you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - ODR-DabMux is distributed in the hope that it will be useful, + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . -*/ + along with this program. If not, see . + */ /* The EDI output needs TAI clock, according to ETSI TS 102 693 Annex F * "EDI Timestamps". This module can set the local CLOCK_TAI clock by diff --git a/lib/edioutput/AFPacket.cpp b/lib/edioutput/AFPacket.cpp new file mode 100644 index 0000000..b38c38b --- /dev/null +++ b/lib/edioutput/AFPacket.cpp @@ -0,0 +1,96 @@ +/* + Copyright (C) 2014 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output. + This implements an AF Packet as defined ETSI TS 102 821. + Also see ETSI TS 102 693 + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "config.h" +#include "crc.h" +#include "AFPacket.h" +#include "TagItems.h" +#include "TagPacket.h" +#include +#include +#include +#include +#include +#include + +namespace edi { + +// Header PT field. AF packet contains TAG payload +const uint8_t AFHEADER_PT_TAG = 'T'; + +// AF Packet Major (3 bits) and Minor (4 bits) version +const uint8_t AFHEADER_VERSION = 0x10; // MAJ=1, MIN=0 + +AFPacket AFPacketiser::Assemble(TagPacket tag_packet) +{ + std::vector payload = tag_packet.Assemble(); + + if (m_verbose) + std::cerr << "Assemble AFPacket " << seq << std::endl; + + std::string pack_data("AF"); // SYNC + std::vector packet(pack_data.begin(), pack_data.end()); + + uint32_t taglength = payload.size(); + + if (m_verbose) + std::cerr << " AFPacket payload size " << payload.size() << std::endl; + + // write length into packet + packet.push_back((taglength >> 24) & 0xFF); + packet.push_back((taglength >> 16) & 0xFF); + packet.push_back((taglength >> 8) & 0xFF); + packet.push_back(taglength & 0xFF); + + // fill rest of header + packet.push_back(seq >> 8); + packet.push_back(seq & 0xFF); + seq++; + packet.push_back((have_crc ? 0x80 : 0) | AFHEADER_VERSION); // ar_cf: CRC=1 + packet.push_back(AFHEADER_PT_TAG); + + // insert payload, must have a length multiple of 8 bytes + packet.insert(packet.end(), payload.begin(), payload.end()); + + // calculate CRC over AF Header and payload + uint16_t crc = 0xffff; + crc = crc16(crc, &(packet.front()), packet.size()); + crc ^= 0xffff; + + if (m_verbose) + fprintf(stderr, " AFPacket crc %x\n", crc); + + packet.push_back((crc >> 8) & 0xFF); + packet.push_back(crc & 0xFF); + + if (m_verbose) + std::cerr << " AFPacket length " << packet.size() << std::endl; + + return packet; +} + +} diff --git a/lib/edioutput/AFPacket.h b/lib/edioutput/AFPacket.h new file mode 100644 index 0000000..f2c4e35 --- /dev/null +++ b/lib/edioutput/AFPacket.h @@ -0,0 +1,61 @@ +/* + Copyright (C) 2014 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output. + This implements an AF Packet as defined ETSI TS 102 821. + Also see ETSI TS 102 693 + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include +#include +#include "TagItems.h" +#include "TagPacket.h" + +namespace edi { + +typedef std::vector AFPacket; + +// ETSI TS 102 821, 6.1 AF packet structure +class AFPacketiser +{ + public: + AFPacketiser() : + m_verbose(false) {}; + AFPacketiser(bool verbose) : + m_verbose(verbose) {}; + + AFPacket Assemble(TagPacket tag_packet); + + private: + static const bool have_crc = true; + + uint16_t seq = 0; //counter that overflows at 0xFFFF + + bool m_verbose; +}; + +} + diff --git a/lib/edioutput/Config.h b/lib/edioutput/Config.h new file mode 100644 index 0000000..ca76322 --- /dev/null +++ b/lib/edioutput/Config.h @@ -0,0 +1,84 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + UDP and TCP transports and their configuration + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include +#include +#include +#include + +namespace edi { + +/** Configuration for EDI output */ + +struct destination_t { + virtual ~destination_t() {}; +}; + +// Can represent both unicast and multicast destinations +struct udp_destination_t : public destination_t { + std::string dest_addr; + std::string source_addr; + unsigned int source_port = 0; + unsigned int ttl = 10; +}; + +// TCP server that can accept multiple connections +struct tcp_server_t : public destination_t { + unsigned int listen_port = 0; + size_t max_frames_queued = 1024; +}; + +// TCP client that connects to one endpoint +struct tcp_client_t : public destination_t { + std::string dest_addr; + unsigned int dest_port = 0; + size_t max_frames_queued = 1024; +}; + +struct configuration_t { + unsigned chunk_len = 207; // RSk, data length of each chunk + unsigned fec = 0; // number of fragments that can be recovered + bool dump = false; // dump a file with the EDI packets + bool verbose = false; + bool enable_pft = false; // Enable protection and fragmentation + unsigned int tagpacket_alignment = 0; + std::vector > destinations; + unsigned int dest_port = 0; // common destination port, because it's encoded in the transport layer + unsigned int latency_frames = 0; // if nonzero, enable interleaver with a latency of latency_frames * 24ms + + bool enabled() const { return destinations.size() > 0; } + bool interleaver_enabled() const { return latency_frames > 0; } + + void print() const; +}; + +} + + diff --git a/lib/edioutput/Interleaver.cpp b/lib/edioutput/Interleaver.cpp new file mode 100644 index 0000000..f26a50e --- /dev/null +++ b/lib/edioutput/Interleaver.cpp @@ -0,0 +1,122 @@ +/* + Copyright (C) 2017 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + Interleaving of PFT fragments to increase robustness against + burst packet loss. + + This is possible because EDI has to assume that fragments may reach + the receiver out of order. + + */ +/* + This file is part of ODR-DabMux. + + ODR-DabMux is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + ODR-DabMux is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ODR-DabMux. If not, see . + */ + +#include "Interleaver.h" +#include + +namespace edi { + +void Interleaver::SetLatency(size_t latency_frames) +{ + m_latency = latency_frames; +} + +Interleaver::fragment_vec Interleaver::Interleave(fragment_vec &fragments) +{ + m_fragment_count = fragments.size(); + + // Create vectors containing Fcount*latency fragments in total + // and store them into the deque + if (m_buffer.empty()) { + m_buffer.emplace_back(); + } + + auto& last_buffer = m_buffer.back(); + + for (auto& fragment : fragments) { + const bool last_buffer_is_complete = + (last_buffer.size() >= m_fragment_count * m_latency); + + if (last_buffer_is_complete) { + m_buffer.emplace_back(); + last_buffer = m_buffer.back(); + } + + last_buffer.push_back(std::move(fragment)); + } + + fragments.clear(); + + while ( not m_buffer.empty() and + (m_buffer.front().size() >= m_fragment_count * m_latency)) { + + auto& first_buffer = m_buffer.front(); + + assert(first_buffer.size() == m_fragment_count * m_latency); + + /* Assume we have 5 fragments per AF frame, and latency of 3. + * This will give the following strides: + * 0 1 2 + * +-------+-------+---+ + * | 0 1 | 2 3 | 4 | + * | | +---+ | + * | 5 6 | 7 | 8 9 | + * | +---+ | | + * |10 |11 12 |13 14 | + * +---+-------+-------+ + * + * ix will be 0, 5, 10, 1, 6 in the first loop + */ + + for (size_t i = 0; i < m_fragment_count; i++) { + const size_t ix = m_interleave_offset + m_fragment_count * m_stride; + m_interleaved_fragments.push_back(first_buffer.at(ix)); + + m_stride += 1; + if (m_stride >= m_latency) { + m_interleave_offset++; + m_stride = 0; + } + } + + if (m_interleave_offset >= m_fragment_count) { + m_interleave_offset = 0; + m_stride = 0; + m_buffer.pop_front(); + } + } + + std::vector interleaved_frags; + + const size_t n = std::min(m_fragment_count, m_interleaved_fragments.size()); + std::move(m_interleaved_fragments.begin(), + m_interleaved_fragments.begin() + n, + std::back_inserter(interleaved_frags)); + m_interleaved_fragments.erase( + m_interleaved_fragments.begin(), + m_interleaved_fragments.begin() + n); + + return interleaved_frags; +} + +} + + diff --git a/lib/edioutput/Interleaver.h b/lib/edioutput/Interleaver.h new file mode 100644 index 0000000..3029d5d --- /dev/null +++ b/lib/edioutput/Interleaver.h @@ -0,0 +1,75 @@ +/* + Copyright (C) 2017 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + Interleaving of PFT fragments to increase robustness against + burst packet loss. + + This is possible because EDI has to assume that fragments may reach + the receiver out of order. + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include +#include +#include +#include +#include "Log.h" +#include "PFT.h" + +namespace edi { + +class Interleaver { + public: + using fragment_vec = std::vector; + + /* Configure the interleaver to use latency_frames number of AF + * packets for interleaving. Total delay through the interleaver + * will be latency_frames * 24ms + */ + void SetLatency(size_t latency_frames); + + /* Move the fragments for an AF Packet into the interleaver and + * return interleaved fragments to be transmitted. + */ + fragment_vec Interleave(fragment_vec &fragments); + + private: + size_t m_latency = 0; + size_t m_fragment_count = 0; + size_t m_interleave_offset = 0; + size_t m_stride = 0; + + /* Buffer that accumulates enough fragments to interleave */ + std::deque m_buffer; + + /* Buffer that contains fragments that have been interleaved, + * to avoid that the interleaver output is too bursty + */ + std::deque m_interleaved_fragments; +}; + +} + diff --git a/lib/edioutput/PFT.cpp b/lib/edioutput/PFT.cpp new file mode 100644 index 0000000..371d36f --- /dev/null +++ b/lib/edioutput/PFT.cpp @@ -0,0 +1,327 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + Protection, Fragmentation and Transport. (PFT) + + Are supported: + Reed-Solomon and Fragmentation + + This implements part of PFT as defined ETSI TS 102 821. + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "PFT.h" +#include "crc.h" +#include "ReedSolomon.h" + +namespace edi { + +using namespace std; + +// An integer division that rounds up, i.e. ceil(a/b) +#define CEIL_DIV(a, b) (a % b == 0 ? a / b : a / b + 1) + +PFT::PFT() { } + +PFT::PFT(const configuration_t &conf) : + m_k(conf.chunk_len), + m_m(conf.fec), + m_dest_port(conf.dest_port), + m_pseq(0), + m_num_chunks(0), + m_verbose(conf.verbose) + { + if (m_k > 207) { + etiLog.level(warn) << + "EDI PFT: maximum chunk size is 207."; + throw std::out_of_range("EDI PFT Chunk size too large."); + } + + if (m_m > 5) { + etiLog.level(warn) << + "EDI PFT: high number of recoverable fragments" + " may lead to large overhead"; + // See TS 102 821, 7.2.1 Known values, list entry for 'm' + } + } + +RSBlock PFT::Protect(AFPacket af_packet) +{ + RSBlock rs_block; + + // number of chunks is ceil(afpacketsize / m_k) + // TS 102 821 7.2.2: c = ceil(l / k_max) + m_num_chunks = CEIL_DIV(af_packet.size(), m_k); + + if (m_verbose) { + fprintf(stderr, "Protect %zu chunks of size %zu\n", + m_num_chunks, af_packet.size()); + } + + // calculate size of chunk: + // TS 102 821 7.2.2: k = ceil(l / c) + // chunk_len does not include the 48 bytes of protection. + const size_t chunk_len = CEIL_DIV(af_packet.size(), m_num_chunks); + if (chunk_len > 207) { + std::stringstream ss; + ss << "Chunk length " << chunk_len << " too large (>207)"; + throw std::runtime_error(ss.str()); + } + + // The last RS chunk is zero padded + // TS 102 821 7.2.2: z = c*k - l + const size_t zero_pad = m_num_chunks * chunk_len - af_packet.size(); + + // Create the RS(k+p,k) encoder + const int firstRoot = 1; // Discovered by analysing EDI dump + const int gfPoly = 0x11d; + const bool reverse = false; + // The encoding has to be 255, 207 always, because the chunk has to + // be padded at the end, and not at the beginning as libfec would + // do + ReedSolomon rs_encoder(255, 207, reverse, gfPoly, firstRoot); + + // add zero padding to last chunk + for (size_t i = 0; i < zero_pad; i++) { + af_packet.push_back(0); + } + + if (m_verbose) { + fprintf(stderr, " add %zu zero padding\n", zero_pad); + } + + // Calculate RS for each chunk and assemble RS block + for (size_t i = 0; i < af_packet.size(); i+= chunk_len) { + vector chunk(207); + vector protection(PARITYBYTES); + + // copy chunk_len bytes into new chunk + memcpy(&chunk.front(), &af_packet[i], chunk_len); + + // calculate RS for chunk with padding + rs_encoder.encode(&chunk.front(), &protection.front(), 207); + + // Drop the padding + chunk.resize(chunk_len); + + // append new chunk and protection to the RS Packet + rs_block.insert(rs_block.end(), chunk.begin(), chunk.end()); + rs_block.insert(rs_block.end(), protection.begin(), protection.end()); + } + + return rs_block; +} + +vector< vector > PFT::ProtectAndFragment(AFPacket af_packet) +{ + const bool enable_RS = (m_m > 0); + + if (enable_RS) { + RSBlock rs_block = Protect(af_packet); + +#if 0 + fprintf(stderr, " af_packet (%zu):", af_packet.size()); + for (size_t i = 0; i < af_packet.size(); i++) { + fprintf(stderr, "%02x ", af_packet[i]); + } + fprintf(stderr, "\n"); + + fprintf(stderr, " rs_block (%zu):", rs_block.size()); + for (size_t i = 0; i < rs_block.size(); i++) { + fprintf(stderr, "%02x ", rs_block[i]); + } + fprintf(stderr, "\n"); +#endif + + // TS 102 821 7.2.2: s_max = MIN(floor(c*p/(m+1)), MTU - h)) + const size_t max_payload_size = ( m_num_chunks * PARITYBYTES ) / (m_m + 1); + + // Calculate fragment count and size + // TS 102 821 7.2.2: ceil((l + c*p + z) / s_max) + // l + c*p + z = length of RS block + const size_t num_fragments = CEIL_DIV(rs_block.size(), max_payload_size); + + // TS 102 821 7.2.2: ceil((l + c*p + z) / f) + const size_t fragment_size = CEIL_DIV(rs_block.size(), num_fragments); + + if (m_verbose) + fprintf(stderr, " PnF fragment_size %zu, num frag %zu\n", + fragment_size, num_fragments); + + vector< vector > fragments(num_fragments); + + for (size_t i = 0; i < num_fragments; i++) { + fragments[i].resize(fragment_size); + for (size_t j = 0; j < fragment_size; j++) { + const size_t ix = j*num_fragments + i; + if (ix < rs_block.size()) { + fragments[i][j] = rs_block[ix]; + } + else { + fragments[i][j] = 0; + } + } + } + + return fragments; + } + else { // No RS, only fragmentation + // TS 102 821 7.2.2: s_max = MTU - h + // Ethernet MTU is 1500, but maybe you are routing over a network which + // has some sort of packet encapsulation. Add some margin. + const size_t max_payload_size = 1400; + + // Calculate fragment count and size + // TS 102 821 7.2.2: ceil((l + c*p + z) / s_max) + // l + c*p + z = length of AF packet + const size_t num_fragments = CEIL_DIV(af_packet.size(), max_payload_size); + + // TS 102 821 7.2.2: ceil((l + c*p + z) / f) + const size_t fragment_size = CEIL_DIV(af_packet.size(), num_fragments); + vector< vector > fragments(num_fragments); + + for (size_t i = 0; i < num_fragments; i++) { + fragments[i].reserve(fragment_size); + + for (size_t j = 0; j < fragment_size; j++) { + const size_t ix = i*fragment_size + j; + if (ix < af_packet.size()) { + fragments[i].push_back(af_packet.at(ix)); + } + else { + break; + } + } + } + + return fragments; + } +} + +std::vector< PFTFragment > PFT::Assemble(AFPacket af_packet) +{ + vector< vector > fragments = ProtectAndFragment(af_packet); + vector< vector > pft_fragments; // These contain PF headers + + const bool enable_RS = (m_m > 0); + const bool enable_transport = true; + + unsigned int findex = 0; + + unsigned fcount = fragments.size(); + + // calculate size of chunk: + // TS 102 821 7.2.2: k = ceil(l / c) + // chunk_len does not include the 48 bytes of protection. + const size_t chunk_len = enable_RS ? + CEIL_DIV(af_packet.size(), m_num_chunks) : 0; + + // The last RS chunk is zero padded + // TS 102 821 7.2.2: z = c*k - l + const size_t zero_pad = enable_RS ? + m_num_chunks * chunk_len - af_packet.size() : 0; + + for (const auto &fragment : fragments) { + // Psync + std::string psync("PF"); + std::vector packet(psync.begin(), psync.end()); + + // Pseq + packet.push_back(m_pseq >> 8); + packet.push_back(m_pseq & 0xFF); + + // Findex + packet.push_back(findex >> 16); + packet.push_back(findex >> 8); + packet.push_back(findex & 0xFF); + findex++; + + // Fcount + packet.push_back(fcount >> 16); + packet.push_back(fcount >> 8); + packet.push_back(fcount & 0xFF); + + // RS (1 bit), transport (1 bit) and Plen (14 bits) + unsigned int plen = fragment.size(); + if (enable_RS) { + plen |= 0x8000; // Set FEC bit + } + + if (enable_transport) { + plen |= 0x4000; // Set ADDR bit + } + + packet.push_back(plen >> 8); + packet.push_back(plen & 0xFF); + + if (enable_RS) { + packet.push_back(chunk_len); // RSk + packet.push_back(zero_pad); // RSz + } + + if (enable_transport) { + // Source (16 bits) + uint16_t addr_source = 0; + packet.push_back(addr_source >> 8); + packet.push_back(addr_source & 0xFF); + + // Dest (16 bits) + packet.push_back(m_dest_port >> 8); + packet.push_back(m_dest_port & 0xFF); + } + + // calculate CRC over AF Header and payload + uint16_t crc = 0xffff; + crc = crc16(crc, &(packet.front()), packet.size()); + crc ^= 0xffff; + + packet.push_back((crc >> 8) & 0xFF); + packet.push_back(crc & 0xFF); + + // insert payload, must have a length multiple of 8 bytes + packet.insert(packet.end(), fragment.begin(), fragment.end()); + + pft_fragments.push_back(packet); + +#if 0 + fprintf(stderr, "* PFT pseq %d, findex %d, fcount %d, plen %d\n", + m_pseq, findex, fcount, plen & ~0xC000); +#endif + } + + m_pseq++; + + return pft_fragments; +} + +} + diff --git a/lib/edioutput/PFT.h b/lib/edioutput/PFT.h new file mode 100644 index 0000000..0ff4839 --- /dev/null +++ b/lib/edioutput/PFT.h @@ -0,0 +1,78 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + Protection, Fragmentation and Transport. (PFT) + + Are supported: + Reed-Solomon and Fragmentation + + This implements part of PFT as defined ETSI TS 102 821. + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include +#include +#include +#include +#include "AFPacket.h" +#include "Log.h" +#include "ReedSolomon.h" +#include "Config.h" + +namespace edi { + +typedef std::vector RSBlock; +typedef std::vector PFTFragment; + +class PFT +{ + public: + static constexpr int PARITYBYTES = 48; + + PFT(); + PFT(const configuration_t& conf); + + // return a list of PFT fragments with the correct + // PFT headers + std::vector< PFTFragment > Assemble(AFPacket af_packet); + + // Apply Reed-Solomon FEC to the AF Packet + RSBlock Protect(AFPacket af_packet); + + // Cut a RSBlock into several fragments that can be transmitted + std::vector< std::vector > ProtectAndFragment(AFPacket af_packet); + + private: + unsigned int m_k = 207; // length of RS data word + unsigned int m_m = 3; // number of fragments that can be recovered if lost + unsigned int m_dest_port = 12000; // Destination port for transport header + uint16_t m_pseq = 0; + size_t m_num_chunks = 0; + bool m_verbose = 0; +}; + +} + diff --git a/lib/edioutput/TagItems.cpp b/lib/edioutput/TagItems.cpp new file mode 100644 index 0000000..35a6852 --- /dev/null +++ b/lib/edioutput/TagItems.cpp @@ -0,0 +1,383 @@ +/* + EDI output. + This defines a few TAG items as defined ETSI TS 102 821 and + ETSI TS 102 693 + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#include "config.h" +#include "TagItems.h" +#include +#include +#include +#include +#include + +namespace edi { + +TagStarPTR::TagStarPTR(const std::string& protocol) + : m_protocol(protocol) +{ + if (m_protocol.size() != 4) { + throw std::runtime_error("TagStarPTR protocol invalid length"); + } +} + +std::vector TagStarPTR::Assemble() +{ + //std::cerr << "TagItem *ptr" << std::endl; + std::string pack_data("*ptr"); + std::vector packet(pack_data.begin(), pack_data.end()); + + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + packet.push_back(0x40); + + packet.insert(packet.end(), m_protocol.begin(), m_protocol.end()); + + // Major + packet.push_back(0); + packet.push_back(0); + + // Minor + packet.push_back(0); + packet.push_back(0); + return packet; +} + +std::vector TagDETI::Assemble() +{ + std::string pack_data("deti"); + std::vector packet(pack_data.begin(), pack_data.end()); + packet.reserve(256); + + // Placeholder for length + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + + uint8_t fct = dlfc % 250; + uint8_t fcth = dlfc / 250; + + + uint16_t detiHeader = fct | (fcth << 8) | (rfudf << 13) | (ficf << 14) | (atstf << 15); + packet.push_back(detiHeader >> 8); + packet.push_back(detiHeader & 0xFF); + + uint32_t etiHeader = mnsc | (rfu << 16) | (rfa << 17) | + (fp << 19) | (mid << 22) | (stat << 24); + packet.push_back((etiHeader >> 24) & 0xFF); + packet.push_back((etiHeader >> 16) & 0xFF); + packet.push_back((etiHeader >> 8) & 0xFF); + packet.push_back(etiHeader & 0xFF); + + if (atstf) { + packet.push_back(utco); + + packet.push_back((seconds >> 24) & 0xFF); + packet.push_back((seconds >> 16) & 0xFF); + packet.push_back((seconds >> 8) & 0xFF); + packet.push_back(seconds & 0xFF); + + packet.push_back((tsta >> 16) & 0xFF); + packet.push_back((tsta >> 8) & 0xFF); + packet.push_back(tsta & 0xFF); + } + + if (ficf) { + for (size_t i = 0; i < fic_length; i++) { + packet.push_back(fic_data[i]); + } + } + + if (rfudf) { + packet.push_back((rfud >> 16) & 0xFF); + packet.push_back((rfud >> 8) & 0xFF); + packet.push_back(rfud & 0xFF); + } + + // calculate and update size + // remove TAG name and TAG length fields and convert to bits + uint32_t taglength = (packet.size() - 8) * 8; + + // write length into packet + packet[4] = (taglength >> 24) & 0xFF; + packet[5] = (taglength >> 16) & 0xFF; + packet[6] = (taglength >> 8) & 0xFF; + packet[7] = taglength & 0xFF; + + dlfc = (dlfc+1) % 5000; + + /* + std::cerr << "TagItem deti, packet.size " << packet.size() << std::endl; + std::cerr << " fic length " << fic_length << std::endl; + std::cerr << " length " << taglength / 8 << std::endl; + */ + return packet; +} + +void TagDETI::set_edi_time(const std::time_t t, int tai_utc_offset) +{ + utco = tai_utc_offset - 32; + + const std::time_t posix_timestamp_1_jan_2000 = 946684800; + + seconds = t - posix_timestamp_1_jan_2000 + utco; +} + +std::vector TagESTn::Assemble() +{ + std::string pack_data("est"); + std::vector packet(pack_data.begin(), pack_data.end()); + packet.reserve(mst_length*8 + 16); + + packet.push_back(id); + + // Placeholder for length + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + + if (tpl > 0x3F) { + throw std::runtime_error("TagESTn: invalid TPL value"); + } + + if (sad > 0x3FF) { + throw std::runtime_error("TagESTn: invalid SAD value"); + } + + if (scid > 0x3F) { + throw std::runtime_error("TagESTn: invalid SCID value"); + } + + uint32_t sstc = (scid << 18) | (sad << 8) | (tpl << 2) | rfa; + packet.push_back((sstc >> 16) & 0xFF); + packet.push_back((sstc >> 8) & 0xFF); + packet.push_back(sstc & 0xFF); + + for (size_t i = 0; i < mst_length * 8; i++) { + packet.push_back(mst_data[i]); + } + + // calculate and update size + // remove TAG name and TAG length fields and convert to bits + uint32_t taglength = (packet.size() - 8) * 8; + + // write length into packet + packet[4] = (taglength >> 24) & 0xFF; + packet[5] = (taglength >> 16) & 0xFF; + packet[6] = (taglength >> 8) & 0xFF; + packet[7] = taglength & 0xFF; + + /* + std::cerr << "TagItem ESTn, length " << packet.size() << std::endl; + std::cerr << " mst_length " << mst_length << std::endl; + */ + return packet; +} + +std::vector TagDSTI::Assemble() +{ + std::string pack_data("dsti"); + std::vector packet(pack_data.begin(), pack_data.end()); + packet.reserve(256); + + // Placeholder for length + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + + uint8_t dfctl = dflc % 250; + uint8_t dfcth = dflc / 250; + + + uint16_t dstiHeader = dfctl | (dfcth << 8) | (rfadf << 13) | (atstf << 14) | (stihf << 15); + packet.push_back(dstiHeader >> 8); + packet.push_back(dstiHeader & 0xFF); + + if (stihf) { + packet.push_back(stat); + packet.push_back((spid >> 8) & 0xFF); + packet.push_back(spid & 0xFF); + } + + if (atstf) { + packet.push_back(utco); + + packet.push_back((seconds >> 24) & 0xFF); + packet.push_back((seconds >> 16) & 0xFF); + packet.push_back((seconds >> 8) & 0xFF); + packet.push_back(seconds & 0xFF); + + packet.push_back((tsta >> 16) & 0xFF); + packet.push_back((tsta >> 8) & 0xFF); + packet.push_back(tsta & 0xFF); + } + + if (rfadf) { + for (size_t i = 0; i < rfad.size(); i++) { + packet.push_back(rfad[i]); + } + } + // calculate and update size + // remove TAG name and TAG length fields and convert to bits + uint32_t taglength = (packet.size() - 8) * 8; + + // write length into packet + packet[4] = (taglength >> 24) & 0xFF; + packet[5] = (taglength >> 16) & 0xFF; + packet[6] = (taglength >> 8) & 0xFF; + packet[7] = taglength & 0xFF; + + dflc = (dflc+1) % 5000; + + /* + std::cerr << "TagItem dsti, packet.size " << packet.size() << std::endl; + std::cerr << " length " << taglength / 8 << std::endl; + */ + return packet; +} + +void TagDSTI::set_edi_time(const std::time_t t, int tai_utc_offset) +{ + utco = tai_utc_offset - 32; + + const std::time_t posix_timestamp_1_jan_2000 = 946684800; + + seconds = t - posix_timestamp_1_jan_2000 + utco; +} + +#if 0 +/* Update the EDI time. t is in UTC, TAI offset is requested from adjtimex */ +void TagDSTI::set_edi_time(const std::time_t t) +{ + if (tai_offset_cache_updated_at == 0 or tai_offset_cache_updated_at + 3600 < t) { + struct timex timex_request; + timex_request.modes = 0; + + int err = adjtimex(&timex_request); + if (err == -1) { + throw std::runtime_error("adjtimex failed"); + } + + if (timex_request.tai == 0) { + throw std::runtime_error("CLOCK_TAI is not properly set up"); + } + tai_offset_cache = timex_request.tai; + tai_offset_cache_updated_at = t; + + fprintf(stderr, "adjtimex: %d, tai %d\n", err, timex_request.tai); + } + + utco = tai_offset_cache - 32; + + const std::time_t posix_timestamp_1_jan_2000 = 946684800; + + seconds = t - posix_timestamp_1_jan_2000 + utco; +} +#endif + +std::vector TagSSm::Assemble() +{ + std::string pack_data("ss"); + std::vector packet(pack_data.begin(), pack_data.end()); + packet.reserve(istd_length + 16); + + packet.push_back((id >> 8) & 0xFF); + packet.push_back(id & 0xFF); + + // Placeholder for length + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + packet.push_back(0); + + if (rfa > 0x1F) { + throw std::runtime_error("TagSSm: invalid RFA value"); + } + + if (tid > 0x7) { + throw std::runtime_error("TagSSm: invalid tid value"); + } + + if (tidext > 0x7) { + throw std::runtime_error("TagSSm: invalid tidext value"); + } + + if (stid > 0x0FFF) { + throw std::runtime_error("TagSSm: invalid stid value"); + } + + uint32_t istc = (rfa << 19) | (tid << 16) | (tidext << 13) | ((crcstf ? 1 : 0) << 12) | stid; + packet.push_back((istc >> 16) & 0xFF); + packet.push_back((istc >> 8) & 0xFF); + packet.push_back(istc & 0xFF); + + for (size_t i = 0; i < istd_length; i++) { + packet.push_back(istd_data[i]); + } + + // calculate and update size + // remove TAG name and TAG length fields and convert to bits + uint32_t taglength = (packet.size() - 8) * 8; + + // write length into packet + packet[4] = (taglength >> 24) & 0xFF; + packet[5] = (taglength >> 16) & 0xFF; + packet[6] = (taglength >> 8) & 0xFF; + packet[7] = taglength & 0xFF; + + /* + std::cerr << "TagItem SSm, length " << packet.size() << std::endl; + std::cerr << " istd_length " << istd_length << std::endl; + */ + return packet; +} + + +std::vector TagStarDMY::Assemble() +{ + std::string pack_data("*dmy"); + std::vector packet(pack_data.begin(), pack_data.end()); + + packet.resize(4 + 4 + length_); + + const uint32_t length_bits = length_ * 8; + + packet[4] = (length_bits >> 24) & 0xFF; + packet[5] = (length_bits >> 16) & 0xFF; + packet[6] = (length_bits >> 8) & 0xFF; + packet[7] = length_bits & 0xFF; + + // The remaining bytes in the packet are "undefined data" + + return packet; +} + +} + diff --git a/lib/edioutput/TagItems.h b/lib/edioutput/TagItems.h new file mode 100644 index 0000000..25daa14 --- /dev/null +++ b/lib/edioutput/TagItems.h @@ -0,0 +1,229 @@ +/* + EDI output. + This defines a few TAG items as defined ETSI TS 102 821 and + ETSI TS 102 693 + + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include +#include +#include +#include +#include + +namespace edi { + +class TagItem +{ + public: + virtual std::vector Assemble() = 0; +}; + +// ETSI TS 102 693, 5.1.1 Protocol type and revision +class TagStarPTR : public TagItem +{ + public: + TagStarPTR(const std::string& protocol); + std::vector Assemble(); + + private: + std::string m_protocol = ""; +}; + +// ETSI TS 102 693, 5.1.3 DAB ETI(LI) Management (deti) +class TagDETI : public TagItem +{ + public: + std::vector Assemble(); + + /***** DATA in intermediary format ****/ + // For the ETI Header: must be defined ! + uint8_t stat = 0; + uint8_t mid = 0; + uint8_t fp = 0; + uint8_t rfa = 0; + uint8_t rfu = 0; // MNSC is valid + uint16_t mnsc = 0; + uint16_t dlfc = 0; // modulo 5000 frame counter + + // ATST (optional) + bool atstf = false; // presence of atst data + + /* UTCO: Offset (in seconds) between UTC and the Seconds value. The + * value is expressed as an unsigned 8-bit quantity. As of February + * 2009, the value shall be 2 and shall change as a result of each + * modification of the number of leap seconds, as proscribed by + * International Earth Rotation and Reference Systems Service (IERS). + * + * According to Annex F + * EDI = TAI - 32s (constant) + * EDI = UTC + UTCO + * we derive + * UTCO = TAI-UTC - 32 + * where the TAI-UTC offset is given by the USNO bulletin using + * the ClockTAI module. + */ + uint8_t utco = 0; + + /* Update the EDI time. t is in UTC */ + void set_edi_time(const std::time_t t, int tai_utc_offset); + + /* The number of SI seconds since 2000-01-01 T 00:00:00 UTC as an + * unsigned 32-bit quantity. Contrary to POSIX, this value also + * counts leap seconds. + */ + uint32_t seconds = 0; + + /* TSTA: Shall be the 24 least significant bits of the Time Stamp + * (TIST) field from the STI-D(LI) Frame. The full definition for the + * STI TIST can be found in annex B of EN 300 797 [4]. The most + * significant 8 bits of the TIST field of the incoming STI-D(LI) + * frame, if required, may be carried in the RFAD field. + */ + uint32_t tsta = 0xFFFFFF; + + // the FIC (optional) + bool ficf = false; + const unsigned char* fic_data; + size_t fic_length; + + // rfu + bool rfudf = false; + uint32_t rfud = 0; + + +}; + +// ETSI TS 102 693, 5.1.5 ETI Sub-Channel Stream +class TagESTn : public TagItem +{ + public: + std::vector Assemble(); + + // SSTCn + uint8_t scid; + uint16_t sad; + uint8_t tpl; + uint8_t rfa; + + // Pointer to MSTn data + uint8_t* mst_data; + size_t mst_length; // STLn * 8 bytes + + uint8_t id; +}; + +// ETSI TS 102 693, 5.1.2 DAB STI-D(LI) Management +class TagDSTI : public TagItem +{ + public: + std::vector Assemble(); + + // dsti Header + bool stihf = false; + bool atstf = false; // presence of atst data + bool rfadf = false; + uint16_t dflc = 0; // modulo 5000 frame counter + + // STI Header (optional) + uint8_t stat = 0; + uint16_t spid = 0; + + /* UTCO: Offset (in seconds) between UTC and the Seconds value. The + * value is expressed as an unsigned 8-bit quantity. As of February + * 2009, the value shall be 2 and shall change as a result of each + * modification of the number of leap seconds, as proscribed by + * International Earth Rotation and Reference Systems Service (IERS). + * + * According to Annex F + * EDI = TAI - 32s (constant) + * EDI = UTC + UTCO + * we derive + * UTCO = TAI-UTC - 32 + * where the TAI-UTC offset is given by the USNO bulletin using + * the ClockTAI module. + */ + uint8_t utco = 0; + + /* Update the EDI time. t is in UTC */ + void set_edi_time(const std::time_t t, int tai_utc_offset); + + /* The number of SI seconds since 2000-01-01 T 00:00:00 UTC as an + * unsigned 32-bit quantity. Contrary to POSIX, this value also + * counts leap seconds. + */ + uint32_t seconds = 0; + + /* TSTA: Shall be the 24 least significant bits of the Time Stamp + * (TIST) field from the STI-D(LI) Frame. The full definition for the + * STI TIST can be found in annex B of EN 300 797 [4]. The most + * significant 8 bits of the TIST field of the incoming STI-D(LI) + * frame, if required, may be carried in the RFAD field. + */ + uint32_t tsta = 0xFFFFFF; + + std::array rfad; + + private: + int tai_offset_cache = 0; + std::time_t tai_offset_cache_updated_at = 0; +}; + +// ETSI TS 102 693, 5.1.4 STI-D Payload Stream +class TagSSm : public TagItem +{ + public: + std::vector Assemble(); + + // SSTCn + uint8_t rfa = 0; + uint8_t tid = 0; // See EN 300 797, 5.4.1.1. Value 0 means "MSC sub-channel" + uint8_t tidext = 0; // EN 300 797, 5.4.1.3, Value 0 means "MSC audio stream" + bool crcstf = false; + uint16_t stid = 0; + + // Pointer to ISTDm data + const uint8_t *istd_data; + size_t istd_length; // bytes + + uint16_t id = 0; +}; + +// ETSI TS 102 821, 5.2.2.2 Dummy padding +class TagStarDMY : public TagItem +{ + public: + /* length is the TAG value length in bytes */ + TagStarDMY(uint32_t length) : length_(length) {} + std::vector Assemble(); + + private: + uint32_t length_; +}; + +} + diff --git a/lib/edioutput/TagPacket.cpp b/lib/edioutput/TagPacket.cpp new file mode 100644 index 0000000..b0bf9a1 --- /dev/null +++ b/lib/edioutput/TagPacket.cpp @@ -0,0 +1,78 @@ +/* + Copyright (C) 2014 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output. + This defines a TAG Packet. + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#include "config.h" +#include "TagPacket.h" +#include "TagItems.h" +#include +#include +#include +#include +#include +#include + +namespace edi { + +TagPacket::TagPacket(unsigned int alignment) : m_alignment(alignment) +{ } + +std::vector TagPacket::Assemble() +{ + std::list::iterator tag; + + std::vector packet; + + //std::cerr << "Assemble TAGPacket" << std::endl; + + for (tag = tag_items.begin(); tag != tag_items.end(); ++tag) { + std::vector tag_data = (*tag)->Assemble(); + packet.insert(packet.end(), tag_data.begin(), tag_data.end()); + + //std::cerr << " Add TAGItem of length " << tag_data.size() << std::endl; + } + + if (m_alignment == 0) { /* no padding */ } + else if (m_alignment == 8) { + // Add padding inside TAG packet + while (packet.size() % 8 > 0) { + packet.push_back(0); // TS 102 821, 5.1, "padding shall be undefined" + } + } + else if (m_alignment > 8) { + TagStarDMY dmy(m_alignment - 8); + auto dmy_data = dmy.Assemble(); + packet.insert(packet.end(), dmy_data.begin(), dmy_data.end()); + } + else { + std::cerr << "Invalid alignment requirement " << m_alignment << + " defined in TagPacket" << std::endl; + } + + return packet; +} + +} + diff --git a/lib/edioutput/TagPacket.h b/lib/edioutput/TagPacket.h new file mode 100644 index 0000000..1e40ce7 --- /dev/null +++ b/lib/edioutput/TagPacket.h @@ -0,0 +1,56 @@ +/* + Copyright (C) 2014 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output. + This defines a TAG Packet. + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include "TagItems.h" +#include +#include +#include +#include + +namespace edi { + +// A TagPacket is nothing else than a list of tag items, with an +// Assemble function that puts the bytestream together and adds +// padding such that the total length is a multiple of 8 Bytes. +// +// ETSI TS 102 821, 5.1 Tag Packet +class TagPacket +{ + public: + TagPacket(unsigned int alignment); + std::vector Assemble(); + + std::list tag_items; + + private: + unsigned int m_alignment; +}; + +} + diff --git a/lib/edioutput/Transport.cpp b/lib/edioutput/Transport.cpp new file mode 100644 index 0000000..0d5c237 --- /dev/null +++ b/lib/edioutput/Transport.cpp @@ -0,0 +1,188 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + UDP and TCP transports and their configuration + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ +#include "Transport.h" +#include + +using namespace std; + +namespace edi { + +void configuration_t::print() const +{ + etiLog.level(info) << "EDI"; + etiLog.level(info) << " verbose " << verbose; + for (auto edi_dest : destinations) { + if (auto udp_dest = dynamic_pointer_cast(edi_dest)) { + etiLog.level(info) << " UDP to " << udp_dest->dest_addr << ":" << dest_port; + if (not udp_dest->source_addr.empty()) { + etiLog.level(info) << " source " << udp_dest->source_addr; + etiLog.level(info) << " ttl " << udp_dest->ttl; + } + etiLog.level(info) << " source port " << udp_dest->source_port; + } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + etiLog.level(info) << " TCP listening on port " << tcp_dest->listen_port; + etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; + } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + etiLog.level(info) << " TCP client connecting to " << tcp_dest->dest_addr << ":" << tcp_dest->dest_port; + etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; + } + else { + throw logic_error("EDI destination not implemented"); + } + } + if (interleaver_enabled()) { + etiLog.level(info) << " interleave " << latency_frames * 24 << " ms"; + } +} + + +Sender::Sender(const configuration_t& conf) : + m_conf(conf), + edi_pft(m_conf) +{ + if (m_conf.verbose) { + etiLog.log(info, "Setup EDI"); + } + + for (const auto& edi_dest : m_conf.destinations) { + if (const auto udp_dest = dynamic_pointer_cast(edi_dest)) { + auto udp_socket = std::make_shared(udp_dest->source_port); + + if (not udp_dest->source_addr.empty()) { + udp_socket->setMulticastSource(udp_dest->source_addr.c_str()); + udp_socket->setMulticastTTL(udp_dest->ttl); + } + + udp_sockets.emplace(udp_dest.get(), udp_socket); + } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + auto dispatcher = make_shared(tcp_dest->max_frames_queued); + dispatcher->start(tcp_dest->listen_port, "0.0.0.0"); + tcp_dispatchers.emplace(tcp_dest.get(), dispatcher); + } + else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { + auto tcp_socket = make_shared(); + tcp_socket->connect(tcp_dest->dest_addr, tcp_dest->dest_port); + tcp_senders.emplace(tcp_dest.get(), tcp_socket); + } + else { + throw logic_error("EDI destination not implemented"); + } + } + + if (m_conf.interleaver_enabled()) { + edi_interleaver.SetLatency(m_conf.latency_frames); + } + + if (m_conf.dump) { + edi_debug_file.open("./edi.debug"); + } + + if (m_conf.verbose) { + etiLog.log(info, "EDI set up"); + } +} + +void Sender::write(const TagPacket& tagpacket) +{ + // Assemble into one AF Packet + edi::AFPacket af_packet = edi_afPacketiser.Assemble(tagpacket); + + if (m_conf.enable_pft) { + // Apply PFT layer to AF Packet (Reed Solomon FEC and Fragmentation) + vector edi_fragments = edi_pft.Assemble(af_packet); + + if (m_conf.verbose) { + fprintf(stderr, "EDI number of PFT fragment before interleaver %zu\n", + edi_fragments.size()); + } + + if (m_conf.interleaver_enabled()) { + edi_fragments = edi_interleaver.Interleave(edi_fragments); + } + + // Send over ethernet + for (const auto& edi_frag : edi_fragments) { + for (auto& dest : m_conf.destinations) { + if (const auto& udp_dest = dynamic_pointer_cast(dest)) { + Socket::InetAddress addr; + addr.resolveUdpDestination(udp_dest->dest_addr, m_conf.dest_port); + + udp_sockets.at(udp_dest.get())->send(edi_frag, addr); + } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_dispatchers.at(tcp_dest.get())->write(edi_frag); + } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_senders.at(tcp_dest.get())->sendall(edi_frag.data(), edi_frag.size()); + } + else { + throw logic_error("EDI destination not implemented"); + } + } + + if (m_conf.dump) { + ostream_iterator debug_iterator(edi_debug_file); + copy(edi_frag.begin(), edi_frag.end(), debug_iterator); + } + } + + if (m_conf.verbose) { + fprintf(stderr, "EDI number of PFT fragments %zu\n", + edi_fragments.size()); + } + } + else { + // Send over ethernet + for (auto& dest : m_conf.destinations) { + if (const auto& udp_dest = dynamic_pointer_cast(dest)) { + Socket::InetAddress addr; + addr.resolveUdpDestination(udp_dest->dest_addr, m_conf.dest_port); + + udp_sockets.at(udp_dest.get())->send(af_packet, addr); + } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_dispatchers.at(tcp_dest.get())->write(af_packet); + } + else if (auto tcp_dest = dynamic_pointer_cast(dest)) { + tcp_senders.at(tcp_dest.get())->sendall(af_packet.data(), af_packet.size()); + } + else { + throw logic_error("EDI destination not implemented"); + } + } + + if (m_conf.dump) { + ostream_iterator debug_iterator(edi_debug_file); + copy(af_packet.begin(), af_packet.end(), debug_iterator); + } + } +} + +} diff --git a/lib/edioutput/Transport.h b/lib/edioutput/Transport.h new file mode 100644 index 0000000..325acf8 --- /dev/null +++ b/lib/edioutput/Transport.h @@ -0,0 +1,71 @@ +/* + Copyright (C) 2019 + Matthias P. Braendli, matthias.braendli@mpb.li + + http://www.opendigitalradio.org + + EDI output, + UDP and TCP transports and their configuration + + */ +/* + This file is part of the ODR-mmbTools. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +#pragma once + +#include "config.h" +#include "Config.h" +#include "AFPacket.h" +#include "PFT.h" +#include "Interleaver.h" +#include "Socket.h" +#include +#include +#include +#include +#include + +namespace edi { + +/** Configuration for EDI output */ + +class Sender { + public: + Sender(const configuration_t& conf); + + void write(const TagPacket& tagpacket); + + private: + configuration_t m_conf; + std::ofstream edi_debug_file; + + // The TagPacket will then be placed into an AFPacket + edi::AFPacketiser edi_afPacketiser; + + // The AF Packet will be protected with reed-solomon and split in fragments + edi::PFT edi_pft; + + // To mitigate for burst packet loss, PFT fragments can be sent out-of-order + edi::Interleaver edi_interleaver; + + std::unordered_map> udp_sockets; + std::unordered_map> tcp_dispatchers; + std::unordered_map> tcp_senders; +}; + +} + diff --git a/src/DabMultiplexer.cpp b/src/DabMultiplexer.cpp index 489787f..0d68ac2 100644 --- a/src/DabMultiplexer.cpp +++ b/src/DabMultiplexer.cpp @@ -376,7 +376,7 @@ void DabMultiplexer::mux_frame(std::vector >& outputs // For EDI, save ETI(LI) Management data into a TAG Item DETI edi::TagDETI edi_tagDETI; - edi::TagStarPTR edi_tagStarPtr; + edi::TagStarPTR edi_tagStarPtr("DETI"); map edi_subchannelToTag; // The above Tag Items will be assembled into a TAG Packet diff --git a/src/DabMultiplexer.h b/src/DabMultiplexer.h index d1075a6..56a8dde 100644 --- a/src/DabMultiplexer.h +++ b/src/DabMultiplexer.h @@ -30,10 +30,10 @@ #endif #include "dabOutput/dabOutput.h" -#include "dabOutput/edi/TagItems.h" -#include "dabOutput/edi/TagPacket.h" -#include "dabOutput/edi/AFPacket.h" -#include "dabOutput/edi/Transport.h" +#include "edioutput/TagItems.h" +#include "edioutput/TagPacket.h" +#include "edioutput/AFPacket.h" +#include "edioutput/Transport.h" #include "fig/FIGCarousel.h" #include "crc.h" #include "utils.h" diff --git a/src/dabOutput/edi/AFPacket.cpp b/src/dabOutput/edi/AFPacket.cpp deleted file mode 100644 index a58a980..0000000 --- a/src/dabOutput/edi/AFPacket.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - Copyright (C) 2014 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output. - This implements an AF Packet as defined ETSI TS 102 821. - Also see ETSI TS 102 693 - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ -#include "config.h" -#include "crc.h" -#include "AFPacket.h" -#include "TagItems.h" -#include "TagPacket.h" -#include -#include -#include -#include -#include -#include - -namespace edi { - -// Header PT field. AF packet contains TAG payload -const uint8_t AFHEADER_PT_TAG = 'T'; - -// AF Packet Major (3 bits) and Minor (4 bits) version -const uint8_t AFHEADER_VERSION = 0x10; // MAJ=1, MIN=0 - -AFPacket AFPacketiser::Assemble(TagPacket tag_packet) -{ - std::vector payload = tag_packet.Assemble(); - - if (m_verbose) - std::cerr << "Assemble AFPacket " << seq << std::endl; - - std::string pack_data("AF"); // SYNC - std::vector packet(pack_data.begin(), pack_data.end()); - - uint32_t taglength = payload.size(); - - if (m_verbose) - std::cerr << " AFPacket payload size " << payload.size() << std::endl; - - // write length into packet - packet.push_back((taglength >> 24) & 0xFF); - packet.push_back((taglength >> 16) & 0xFF); - packet.push_back((taglength >> 8) & 0xFF); - packet.push_back(taglength & 0xFF); - - // fill rest of header - packet.push_back(seq >> 8); - packet.push_back(seq & 0xFF); - seq++; - packet.push_back((have_crc ? 0x80 : 0) | AFHEADER_VERSION); // ar_cf: CRC=1 - packet.push_back(AFHEADER_PT_TAG); - - // insert payload, must have a length multiple of 8 bytes - packet.insert(packet.end(), payload.begin(), payload.end()); - - // calculate CRC over AF Header and payload - uint16_t crc = 0xffff; - crc = crc16(crc, &(packet.front()), packet.size()); - crc ^= 0xffff; - - if (m_verbose) - fprintf(stderr, " AFPacket crc %x\n", crc); - - packet.push_back((crc >> 8) & 0xFF); - packet.push_back(crc & 0xFF); - - if (m_verbose) - std::cerr << " AFPacket length " << packet.size() << std::endl; - - return packet; -} - -} diff --git a/src/dabOutput/edi/AFPacket.h b/src/dabOutput/edi/AFPacket.h deleted file mode 100644 index b4ccef1..0000000 --- a/src/dabOutput/edi/AFPacket.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - Copyright (C) 2014 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output. - This implements an AF Packet as defined ETSI TS 102 821. - Also see ETSI TS 102 693 - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include -#include -#include "TagItems.h" -#include "TagPacket.h" - -namespace edi { - -typedef std::vector AFPacket; - -// ETSI TS 102 821, 6.1 AF packet structure -class AFPacketiser -{ - public: - AFPacketiser() : - m_verbose(false) {}; - AFPacketiser(bool verbose) : - m_verbose(verbose) {}; - - AFPacket Assemble(TagPacket tag_packet); - - private: - static const bool have_crc = true; - - uint16_t seq = 0; //counter that overflows at 0xFFFF - - bool m_verbose; -}; - -} - diff --git a/src/dabOutput/edi/Config.h b/src/dabOutput/edi/Config.h deleted file mode 100644 index 0c7dce8..0000000 --- a/src/dabOutput/edi/Config.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - UDP and TCP transports and their configuration - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include -#include -#include -#include - -namespace edi { - -/** Configuration for EDI output */ - -struct destination_t { - virtual ~destination_t() {}; -}; - -// Can represent both unicast and multicast destinations -struct udp_destination_t : public destination_t { - std::string dest_addr; - std::string source_addr; - unsigned int source_port = 0; - unsigned int ttl = 10; -}; - -// TCP server that can accept multiple connections -struct tcp_server_t : public destination_t { - unsigned int listen_port = 0; - size_t max_frames_queued = 1024; -}; - -// TCP client that connects to one endpoint -struct tcp_client_t : public destination_t { - std::string dest_addr; - unsigned int dest_port = 0; - size_t max_frames_queued = 1024; -}; - -struct configuration_t { - unsigned chunk_len = 207; // RSk, data length of each chunk - unsigned fec = 0; // number of fragments that can be recovered - bool dump = false; // dump a file with the EDI packets - bool verbose = false; - bool enable_pft = false; // Enable protection and fragmentation - unsigned int tagpacket_alignment = 0; - std::vector > destinations; - unsigned int dest_port = 0; // common destination port, because it's encoded in the transport layer - unsigned int latency_frames = 0; // if nonzero, enable interleaver with a latency of latency_frames * 24ms - - bool enabled() const { return destinations.size() > 0; } - bool interleaver_enabled() const { return latency_frames > 0; } - - void print() const; -}; - -} - - diff --git a/src/dabOutput/edi/Interleaver.cpp b/src/dabOutput/edi/Interleaver.cpp deleted file mode 100644 index f26a50e..0000000 --- a/src/dabOutput/edi/Interleaver.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - Copyright (C) 2017 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - Interleaving of PFT fragments to increase robustness against - burst packet loss. - - This is possible because EDI has to assume that fragments may reach - the receiver out of order. - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "Interleaver.h" -#include - -namespace edi { - -void Interleaver::SetLatency(size_t latency_frames) -{ - m_latency = latency_frames; -} - -Interleaver::fragment_vec Interleaver::Interleave(fragment_vec &fragments) -{ - m_fragment_count = fragments.size(); - - // Create vectors containing Fcount*latency fragments in total - // and store them into the deque - if (m_buffer.empty()) { - m_buffer.emplace_back(); - } - - auto& last_buffer = m_buffer.back(); - - for (auto& fragment : fragments) { - const bool last_buffer_is_complete = - (last_buffer.size() >= m_fragment_count * m_latency); - - if (last_buffer_is_complete) { - m_buffer.emplace_back(); - last_buffer = m_buffer.back(); - } - - last_buffer.push_back(std::move(fragment)); - } - - fragments.clear(); - - while ( not m_buffer.empty() and - (m_buffer.front().size() >= m_fragment_count * m_latency)) { - - auto& first_buffer = m_buffer.front(); - - assert(first_buffer.size() == m_fragment_count * m_latency); - - /* Assume we have 5 fragments per AF frame, and latency of 3. - * This will give the following strides: - * 0 1 2 - * +-------+-------+---+ - * | 0 1 | 2 3 | 4 | - * | | +---+ | - * | 5 6 | 7 | 8 9 | - * | +---+ | | - * |10 |11 12 |13 14 | - * +---+-------+-------+ - * - * ix will be 0, 5, 10, 1, 6 in the first loop - */ - - for (size_t i = 0; i < m_fragment_count; i++) { - const size_t ix = m_interleave_offset + m_fragment_count * m_stride; - m_interleaved_fragments.push_back(first_buffer.at(ix)); - - m_stride += 1; - if (m_stride >= m_latency) { - m_interleave_offset++; - m_stride = 0; - } - } - - if (m_interleave_offset >= m_fragment_count) { - m_interleave_offset = 0; - m_stride = 0; - m_buffer.pop_front(); - } - } - - std::vector interleaved_frags; - - const size_t n = std::min(m_fragment_count, m_interleaved_fragments.size()); - std::move(m_interleaved_fragments.begin(), - m_interleaved_fragments.begin() + n, - std::back_inserter(interleaved_frags)); - m_interleaved_fragments.erase( - m_interleaved_fragments.begin(), - m_interleaved_fragments.begin() + n); - - return interleaved_frags; -} - -} - - diff --git a/src/dabOutput/edi/Interleaver.h b/src/dabOutput/edi/Interleaver.h deleted file mode 100644 index f1cff30..0000000 --- a/src/dabOutput/edi/Interleaver.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - Copyright (C) 2017 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - Interleaving of PFT fragments to increase robustness against - burst packet loss. - - This is possible because EDI has to assume that fragments may reach - the receiver out of order. - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include -#include -#include -#include -#include "Log.h" -#include "PFT.h" - -namespace edi { - -class Interleaver { - public: - using fragment_vec = std::vector; - - /* Configure the interleaver to use latency_frames number of AF - * packets for interleaving. Total delay through the interleaver - * will be latency_frames * 24ms - */ - void SetLatency(size_t latency_frames); - - /* Move the fragments for an AF Packet into the interleaver and - * return interleaved fragments to be transmitted. - */ - fragment_vec Interleave(fragment_vec &fragments); - - private: - size_t m_latency = 0; - size_t m_fragment_count = 0; - size_t m_interleave_offset = 0; - size_t m_stride = 0; - - /* Buffer that accumulates enough fragments to interleave */ - std::deque m_buffer; - - /* Buffer that contains fragments that have been interleaved, - * to avoid that the interleaver output is too bursty - */ - std::deque m_interleaved_fragments; -}; - -} - diff --git a/src/dabOutput/edi/PFT.cpp b/src/dabOutput/edi/PFT.cpp deleted file mode 100644 index 63dfa34..0000000 --- a/src/dabOutput/edi/PFT.cpp +++ /dev/null @@ -1,327 +0,0 @@ -/* - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - Protection, Fragmentation and Transport. (PFT) - - Are supported: - Reed-Solomon and Fragmentation - - This implements part of PFT as defined ETSI TS 102 821. - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "PFT.h" -#include "crc.h" -#include "ReedSolomon.h" - -namespace edi { - -using namespace std; - -// An integer division that rounds up, i.e. ceil(a/b) -#define CEIL_DIV(a, b) (a % b == 0 ? a / b : a / b + 1) - -PFT::PFT() { } - -PFT::PFT(const configuration_t &conf) : - m_k(conf.chunk_len), - m_m(conf.fec), - m_dest_port(conf.dest_port), - m_pseq(0), - m_num_chunks(0), - m_verbose(conf.verbose) - { - if (m_k > 207) { - etiLog.level(warn) << - "EDI PFT: maximum chunk size is 207."; - throw std::out_of_range("EDI PFT Chunk size too large."); - } - - if (m_m > 5) { - etiLog.level(warn) << - "EDI PFT: high number of recoverable fragments" - " may lead to large overhead"; - // See TS 102 821, 7.2.1 Known values, list entry for 'm' - } - } - -RSBlock PFT::Protect(AFPacket af_packet) -{ - RSBlock rs_block; - - // number of chunks is ceil(afpacketsize / m_k) - // TS 102 821 7.2.2: c = ceil(l / k_max) - m_num_chunks = CEIL_DIV(af_packet.size(), m_k); - - if (m_verbose) { - fprintf(stderr, "Protect %zu chunks of size %zu\n", - m_num_chunks, af_packet.size()); - } - - // calculate size of chunk: - // TS 102 821 7.2.2: k = ceil(l / c) - // chunk_len does not include the 48 bytes of protection. - const size_t chunk_len = CEIL_DIV(af_packet.size(), m_num_chunks); - if (chunk_len > 207) { - std::stringstream ss; - ss << "Chunk length " << chunk_len << " too large (>207)"; - throw std::runtime_error(ss.str()); - } - - // The last RS chunk is zero padded - // TS 102 821 7.2.2: z = c*k - l - const size_t zero_pad = m_num_chunks * chunk_len - af_packet.size(); - - // Create the RS(k+p,k) encoder - const int firstRoot = 1; // Discovered by analysing EDI dump - const int gfPoly = 0x11d; - const bool reverse = false; - // The encoding has to be 255, 207 always, because the chunk has to - // be padded at the end, and not at the beginning as libfec would - // do - ReedSolomon rs_encoder(255, 207, reverse, gfPoly, firstRoot); - - // add zero padding to last chunk - for (size_t i = 0; i < zero_pad; i++) { - af_packet.push_back(0); - } - - if (m_verbose) { - fprintf(stderr, " add %zu zero padding\n", zero_pad); - } - - // Calculate RS for each chunk and assemble RS block - for (size_t i = 0; i < af_packet.size(); i+= chunk_len) { - vector chunk(207); - vector protection(PARITYBYTES); - - // copy chunk_len bytes into new chunk - memcpy(&chunk.front(), &af_packet[i], chunk_len); - - // calculate RS for chunk with padding - rs_encoder.encode(&chunk.front(), &protection.front(), 207); - - // Drop the padding - chunk.resize(chunk_len); - - // append new chunk and protection to the RS Packet - rs_block.insert(rs_block.end(), chunk.begin(), chunk.end()); - rs_block.insert(rs_block.end(), protection.begin(), protection.end()); - } - - return rs_block; -} - -vector< vector > PFT::ProtectAndFragment(AFPacket af_packet) -{ - const bool enable_RS = (m_m > 0); - - if (enable_RS) { - RSBlock rs_block = Protect(af_packet); - -#if 0 - fprintf(stderr, " af_packet (%zu):", af_packet.size()); - for (size_t i = 0; i < af_packet.size(); i++) { - fprintf(stderr, "%02x ", af_packet[i]); - } - fprintf(stderr, "\n"); - - fprintf(stderr, " rs_block (%zu):", rs_block.size()); - for (size_t i = 0; i < rs_block.size(); i++) { - fprintf(stderr, "%02x ", rs_block[i]); - } - fprintf(stderr, "\n"); -#endif - - // TS 102 821 7.2.2: s_max = MIN(floor(c*p/(m+1)), MTU - h)) - const size_t max_payload_size = ( m_num_chunks * PARITYBYTES ) / (m_m + 1); - - // Calculate fragment count and size - // TS 102 821 7.2.2: ceil((l + c*p + z) / s_max) - // l + c*p + z = length of RS block - const size_t num_fragments = CEIL_DIV(rs_block.size(), max_payload_size); - - // TS 102 821 7.2.2: ceil((l + c*p + z) / f) - const size_t fragment_size = CEIL_DIV(rs_block.size(), num_fragments); - - if (m_verbose) - fprintf(stderr, " PnF fragment_size %zu, num frag %zu\n", - fragment_size, num_fragments); - - vector< vector > fragments(num_fragments); - - for (size_t i = 0; i < num_fragments; i++) { - fragments[i].resize(fragment_size); - for (size_t j = 0; j < fragment_size; j++) { - const size_t ix = j*num_fragments + i; - if (ix < rs_block.size()) { - fragments[i][j] = rs_block[ix]; - } - else { - fragments[i][j] = 0; - } - } - } - - return fragments; - } - else { // No RS, only fragmentation - // TS 102 821 7.2.2: s_max = MTU - h - // Ethernet MTU is 1500, but maybe you are routing over a network which - // has some sort of packet encapsulation. Add some margin. - const size_t max_payload_size = 1400; - - // Calculate fragment count and size - // TS 102 821 7.2.2: ceil((l + c*p + z) / s_max) - // l + c*p + z = length of AF packet - const size_t num_fragments = CEIL_DIV(af_packet.size(), max_payload_size); - - // TS 102 821 7.2.2: ceil((l + c*p + z) / f) - const size_t fragment_size = CEIL_DIV(af_packet.size(), num_fragments); - vector< vector > fragments(num_fragments); - - for (size_t i = 0; i < num_fragments; i++) { - fragments[i].reserve(fragment_size); - - for (size_t j = 0; j < fragment_size; j++) { - const size_t ix = i*fragment_size + j; - if (ix < af_packet.size()) { - fragments[i].push_back(af_packet.at(ix)); - } - else { - break; - } - } - } - - return fragments; - } -} - -std::vector< PFTFragment > PFT::Assemble(AFPacket af_packet) -{ - vector< vector > fragments = ProtectAndFragment(af_packet); - vector< vector > pft_fragments; // These contain PF headers - - const bool enable_RS = (m_m > 0); - const bool enable_transport = true; - - unsigned int findex = 0; - - unsigned fcount = fragments.size(); - - // calculate size of chunk: - // TS 102 821 7.2.2: k = ceil(l / c) - // chunk_len does not include the 48 bytes of protection. - const size_t chunk_len = enable_RS ? - CEIL_DIV(af_packet.size(), m_num_chunks) : 0; - - // The last RS chunk is zero padded - // TS 102 821 7.2.2: z = c*k - l - const size_t zero_pad = enable_RS ? - m_num_chunks * chunk_len - af_packet.size() : 0; - - for (const auto &fragment : fragments) { - // Psync - std::string psync("PF"); - std::vector packet(psync.begin(), psync.end()); - - // Pseq - packet.push_back(m_pseq >> 8); - packet.push_back(m_pseq & 0xFF); - - // Findex - packet.push_back(findex >> 16); - packet.push_back(findex >> 8); - packet.push_back(findex & 0xFF); - findex++; - - // Fcount - packet.push_back(fcount >> 16); - packet.push_back(fcount >> 8); - packet.push_back(fcount & 0xFF); - - // RS (1 bit), transport (1 bit) and Plen (14 bits) - unsigned int plen = fragment.size(); - if (enable_RS) { - plen |= 0x8000; // Set FEC bit - } - - if (enable_transport) { - plen |= 0x4000; // Set ADDR bit - } - - packet.push_back(plen >> 8); - packet.push_back(plen & 0xFF); - - if (enable_RS) { - packet.push_back(chunk_len); // RSk - packet.push_back(zero_pad); // RSz - } - - if (enable_transport) { - // Source (16 bits) - uint16_t addr_source = 0; - packet.push_back(addr_source >> 8); - packet.push_back(addr_source & 0xFF); - - // Dest (16 bits) - packet.push_back(m_dest_port >> 8); - packet.push_back(m_dest_port & 0xFF); - } - - // calculate CRC over AF Header and payload - uint16_t crc = 0xffff; - crc = crc16(crc, &(packet.front()), packet.size()); - crc ^= 0xffff; - - packet.push_back((crc >> 8) & 0xFF); - packet.push_back(crc & 0xFF); - - // insert payload, must have a length multiple of 8 bytes - packet.insert(packet.end(), fragment.begin(), fragment.end()); - - pft_fragments.push_back(packet); - -#if 0 - fprintf(stderr, "* PFT pseq %d, findex %d, fcount %d, plen %d\n", - m_pseq, findex, fcount, plen & ~0xC000); -#endif - } - - m_pseq++; - - return pft_fragments; -} - -} - diff --git a/src/dabOutput/edi/PFT.h b/src/dabOutput/edi/PFT.h deleted file mode 100644 index 4076bf3..0000000 --- a/src/dabOutput/edi/PFT.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - Protection, Fragmentation and Transport. (PFT) - - Are supported: - Reed-Solomon and Fragmentation - - This implements part of PFT as defined ETSI TS 102 821. - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include -#include -#include -#include -#include "AFPacket.h" -#include "Log.h" -#include "ReedSolomon.h" -#include "dabOutput/edi/Config.h" - -namespace edi { - -typedef std::vector RSBlock; -typedef std::vector PFTFragment; - -class PFT -{ - public: - static constexpr int PARITYBYTES = 48; - - PFT(); - PFT(const configuration_t& conf); - - // return a list of PFT fragments with the correct - // PFT headers - std::vector< PFTFragment > Assemble(AFPacket af_packet); - - // Apply Reed-Solomon FEC to the AF Packet - RSBlock Protect(AFPacket af_packet); - - // Cut a RSBlock into several fragments that can be transmitted - std::vector< std::vector > ProtectAndFragment(AFPacket af_packet); - - private: - unsigned int m_k = 207; // length of RS data word - unsigned int m_m = 3; // number of fragments that can be recovered if lost - unsigned int m_dest_port = 12000; // Destination port for transport header - uint16_t m_pseq = 0; - size_t m_num_chunks = 0; - bool m_verbose = 0; -}; - -} - diff --git a/src/dabOutput/edi/TagItems.cpp b/src/dabOutput/edi/TagItems.cpp deleted file mode 100644 index dfb4934..0000000 --- a/src/dabOutput/edi/TagItems.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/* - EDI output. - This defines a few TAG items as defined ETSI TS 102 821 and - ETSI TS 102 693 - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "config.h" -#include "TagItems.h" -#include -#include -#include -#include -#include - -namespace edi { - -std::vector TagStarPTR::Assemble() -{ - //std::cerr << "TagItem *ptr" << std::endl; - std::string pack_data("*ptr"); - std::vector packet(pack_data.begin(), pack_data.end()); - - packet.push_back(0); - packet.push_back(0); - packet.push_back(0); - packet.push_back(0x40); - - std::string protocol("DETI"); - packet.insert(packet.end(), protocol.begin(), protocol.end()); - - // Major - packet.push_back(0); - packet.push_back(0); - - // Minor - packet.push_back(0); - packet.push_back(0); - return packet; -} - -std::vector TagDETI::Assemble() -{ - std::string pack_data("deti"); - std::vector packet(pack_data.begin(), pack_data.end()); - packet.reserve(256); - - // Placeholder for length - packet.push_back(0); - packet.push_back(0); - packet.push_back(0); - packet.push_back(0); - - uint8_t fct = dlfc % 250; - uint8_t fcth = dlfc / 250; - - - uint16_t detiHeader = fct | (fcth << 8) | (rfudf << 13) | (ficf << 14) | (atstf << 15); - packet.push_back(detiHeader >> 8); - packet.push_back(detiHeader & 0xFF); - - uint32_t etiHeader = mnsc | (rfu << 16) | (rfa << 17) | - (fp << 19) | (mid << 22) | (stat << 24); - packet.push_back((etiHeader >> 24) & 0xFF); - packet.push_back((etiHeader >> 16) & 0xFF); - packet.push_back((etiHeader >> 8) & 0xFF); - packet.push_back(etiHeader & 0xFF); - - if (atstf) { - packet.push_back(utco); - - packet.push_back((seconds >> 24) & 0xFF); - packet.push_back((seconds >> 16) & 0xFF); - packet.push_back((seconds >> 8) & 0xFF); - packet.push_back(seconds & 0xFF); - - packet.push_back((tsta >> 16) & 0xFF); - packet.push_back((tsta >> 8) & 0xFF); - packet.push_back(tsta & 0xFF); - } - - if (ficf) { - for (size_t i = 0; i < fic_length; i++) { - packet.push_back(fic_data[i]); - } - } - - if (rfudf) { - packet.push_back((rfud >> 16) & 0xFF); - packet.push_back((rfud >> 8) & 0xFF); - packet.push_back(rfud & 0xFF); - } - - // calculate and update size - // remove TAG name and TAG length fields and convert to bits - uint32_t taglength = (packet.size() - 8) * 8; - - // write length into packet - packet[4] = (taglength >> 24) & 0xFF; - packet[5] = (taglength >> 16) & 0xFF; - packet[6] = (taglength >> 8) & 0xFF; - packet[7] = taglength & 0xFF; - - dlfc = (dlfc+1) % 5000; - - /* - std::cerr << "TagItem deti, packet.size " << packet.size() << std::endl; - std::cerr << " fic length " << fic_length << std::endl; - std::cerr << " length " << taglength / 8 << std::endl; - */ - return packet; -} - -void TagDETI::set_edi_time(const std::time_t t, int tai_utc_offset) -{ - utco = tai_utc_offset - 32; - - const std::time_t posix_timestamp_1_jan_2000 = 946684800; - - seconds = t - posix_timestamp_1_jan_2000 + utco; -} - -std::vector TagESTn::Assemble() -{ - std::string pack_data("est"); - std::vector packet(pack_data.begin(), pack_data.end()); - packet.reserve(mst_length*8 + 16); - - packet.push_back(id); - - // Placeholder for length - packet.push_back(0); - packet.push_back(0); - packet.push_back(0); - packet.push_back(0); - - if (tpl > 0x3F) { - throw std::runtime_error("TagESTn: invalid TPL value"); - } - - if (sad > 0x3FF) { - throw std::runtime_error("TagESTn: invalid SAD value"); - } - - if (scid > 0x3F) { - throw std::runtime_error("TagESTn: invalid SCID value"); - } - - uint32_t sstc = (scid << 18) | (sad << 8) | (tpl << 2) | rfa; - packet.push_back((sstc >> 16) & 0xFF); - packet.push_back((sstc >> 8) & 0xFF); - packet.push_back(sstc & 0xFF); - - for (size_t i = 0; i < mst_length * 8; i++) { - packet.push_back(mst_data[i]); - } - - // calculate and update size - // remove TAG name and TAG length fields and convert to bits - uint32_t taglength = (packet.size() - 8) * 8; - - // write length into packet - packet[4] = (taglength >> 24) & 0xFF; - packet[5] = (taglength >> 16) & 0xFF; - packet[6] = (taglength >> 8) & 0xFF; - packet[7] = taglength & 0xFF; - - /* - std::cerr << "TagItem ESTn, length " << packet.size() << std::endl; - std::cerr << " mst_length " << mst_length << std::endl; - */ - return packet; -} - -std::vector TagStarDMY::Assemble() -{ - std::string pack_data("*dmy"); - std::vector packet(pack_data.begin(), pack_data.end()); - - packet.resize(4 + 4 + length_); - - const uint32_t length_bits = length_ * 8; - - packet[4] = (length_bits >> 24) & 0xFF; - packet[5] = (length_bits >> 16) & 0xFF; - packet[6] = (length_bits >> 8) & 0xFF; - packet[7] = length_bits & 0xFF; - - // The remaining bytes in the packet are "undefined data" - - return packet; -} - -} - diff --git a/src/dabOutput/edi/TagItems.h b/src/dabOutput/edi/TagItems.h deleted file mode 100644 index b29a142..0000000 --- a/src/dabOutput/edi/TagItems.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - EDI output. - This defines a few TAG items as defined ETSI TS 102 821 and - ETSI TS 102 693 - - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include "Eti.h" -#include -#include -#include -#include - -namespace edi { - -class TagItem -{ - public: - virtual std::vector Assemble() = 0; -}; - -// ETSI TS 102 693, 5.1.1 Protocol type and revision -class TagStarPTR : public TagItem -{ - public: - std::vector Assemble(); -}; - -// ETSI TS 102 693, 5.1.3 DAB ETI(LI) Management (deti) -class TagDETI : public TagItem -{ - public: - std::vector Assemble(); - - /***** DATA in intermediary format ****/ - // For the ETI Header: must be defined ! - uint8_t stat = 0; - uint8_t mid = 0; - uint8_t fp = 0; - uint8_t rfa = 0; - uint8_t rfu = 0; // MNSC is valid - uint16_t mnsc = 0; - uint16_t dlfc = 0; // modulo 5000 frame counter - - // ATST (optional) - bool atstf = false; // presence of atst data - - /* UTCO: Offset (in seconds) between UTC and the Seconds value. The - * value is expressed as an unsigned 8-bit quantity. As of February - * 2009, the value shall be 2 and shall change as a result of each - * modification of the number of leap seconds, as proscribed by - * International Earth Rotation and Reference Systems Service (IERS). - * - * According to Annex F - * EDI = TAI - 32s (constant) - * EDI = UTC + UTCO - * we derive - * UTCO = TAI-UTC - 32 - * where the TAI-UTC offset is given by the USNO bulletin using - * the ClockTAI module. - */ - uint8_t utco = 0; - - /* Update the EDI time. t is in UTC */ - void set_edi_time(const std::time_t t, int tai_utc_offset); - - /* The number of SI seconds since 2000-01-01 T 00:00:00 UTC as an - * unsigned 32-bit quantity. Contrary to POSIX, this value also - * counts leap seconds. - */ - uint32_t seconds = 0; - - /* TSTA: Shall be the 24 least significant bits of the Time Stamp - * (TIST) field from the STI-D(LI) Frame. The full definition for the - * STI TIST can be found in annex B of EN 300 797 [4]. The most - * significant 8 bits of the TIST field of the incoming STI-D(LI) - * frame, if required, may be carried in the RFAD field. - */ - uint32_t tsta = 0xFFFFFF; - - // the FIC (optional) - bool ficf = false; - const unsigned char* fic_data; - size_t fic_length; - - // rfu - bool rfudf = false; - uint32_t rfud = 0; - - -}; - -// ETSI TS 102 693, 5.1.5 ETI Sub-Channel Stream -class TagESTn : public TagItem -{ - public: - std::vector Assemble(); - - // SSTCn - uint8_t scid; - uint16_t sad; - uint8_t tpl; - uint8_t rfa; - - // Pointer to MSTn data - uint8_t* mst_data; - size_t mst_length; // STLn * 8 bytes - - uint8_t id; -}; - -// ETSI TS 102 821, 5.2.2.2 Dummy padding -class TagStarDMY : public TagItem -{ - public: - /* length is the TAG value length in bytes */ - TagStarDMY(uint32_t length) : length_(length) {} - std::vector Assemble(); - - private: - uint32_t length_; -}; - -} - diff --git a/src/dabOutput/edi/TagPacket.cpp b/src/dabOutput/edi/TagPacket.cpp deleted file mode 100644 index b16dc33..0000000 --- a/src/dabOutput/edi/TagPacket.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - Copyright (C) 2014 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output. - This defines a TAG Packet. - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "config.h" -#include "Eti.h" -#include "TagPacket.h" -#include "TagItems.h" -#include -#include -#include -#include -#include -#include - -namespace edi { - -TagPacket::TagPacket(unsigned int alignment) : m_alignment(alignment) -{ } - -std::vector TagPacket::Assemble() -{ - std::list::iterator tag; - - std::vector packet; - - //std::cerr << "Assemble TAGPacket" << std::endl; - - for (tag = tag_items.begin(); tag != tag_items.end(); ++tag) { - std::vector tag_data = (*tag)->Assemble(); - packet.insert(packet.end(), tag_data.begin(), tag_data.end()); - - //std::cerr << " Add TAGItem of length " << tag_data.size() << std::endl; - } - - if (m_alignment == 0) { /* no padding */ } - else if (m_alignment == 8) { - // Add padding inside TAG packet - while (packet.size() % 8 > 0) { - packet.push_back(0); // TS 102 821, 5.1, "padding shall be undefined" - } - } - else if (m_alignment > 8) { - TagStarDMY dmy(m_alignment - 8); - auto dmy_data = dmy.Assemble(); - packet.insert(packet.end(), dmy_data.begin(), dmy_data.end()); - } - else { - std::cerr << "Invalid alignment requirement " << m_alignment << - " defined in TagPacket" << std::endl; - } - - return packet; -} - -} - diff --git a/src/dabOutput/edi/TagPacket.h b/src/dabOutput/edi/TagPacket.h deleted file mode 100644 index a861cbb..0000000 --- a/src/dabOutput/edi/TagPacket.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - Copyright (C) 2014 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output. - This defines a TAG Packet. - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include "TagItems.h" -#include -#include -#include -#include - -namespace edi { - -// A TagPacket is nothing else than a list of tag items, with an -// Assemble function that puts the bytestream together and adds -// padding such that the total length is a multiple of 8 Bytes. -// -// ETSI TS 102 821, 5.1 Tag Packet -class TagPacket -{ - public: - TagPacket(unsigned int alignment); - std::vector Assemble(); - - std::list tag_items; - - private: - unsigned int m_alignment; -}; - -} - diff --git a/src/dabOutput/edi/Transport.cpp b/src/dabOutput/edi/Transport.cpp deleted file mode 100644 index 187aabe..0000000 --- a/src/dabOutput/edi/Transport.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/* - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - UDP and TCP transports and their configuration - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#include "Transport.h" -#include - -using namespace std; - -namespace edi { - -void configuration_t::print() const -{ - etiLog.level(info) << "EDI"; - etiLog.level(info) << " verbose " << verbose; - for (auto edi_dest : destinations) { - if (auto udp_dest = dynamic_pointer_cast(edi_dest)) { - etiLog.level(info) << " UDP to " << udp_dest->dest_addr << ":" << dest_port; - if (not udp_dest->source_addr.empty()) { - etiLog.level(info) << " source " << udp_dest->source_addr; - etiLog.level(info) << " ttl " << udp_dest->ttl; - } - etiLog.level(info) << " source port " << udp_dest->source_port; - } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { - etiLog.level(info) << " TCP listening on port " << tcp_dest->listen_port; - etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; - } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { - etiLog.level(info) << " TCP client connecting to " << tcp_dest->dest_addr << ":" << tcp_dest->dest_port; - etiLog.level(info) << " max frames queued " << tcp_dest->max_frames_queued; - } - else { - throw logic_error("EDI destination not implemented"); - } - } - if (interleaver_enabled()) { - etiLog.level(info) << " interleave " << latency_frames * 24 << " ms"; - } -} - - -Sender::Sender(const configuration_t& conf) : - m_conf(conf), - edi_pft(m_conf) -{ - if (m_conf.verbose) { - etiLog.log(info, "Setup EDI"); - } - - for (const auto& edi_dest : m_conf.destinations) { - if (const auto udp_dest = dynamic_pointer_cast(edi_dest)) { - auto udp_socket = std::make_shared(udp_dest->source_port); - - if (not udp_dest->source_addr.empty()) { - udp_socket->setMulticastSource(udp_dest->source_addr.c_str()); - udp_socket->setMulticastTTL(udp_dest->ttl); - } - - udp_sockets.emplace(udp_dest.get(), udp_socket); - } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { - auto dispatcher = make_shared(tcp_dest->max_frames_queued); - dispatcher->start(tcp_dest->listen_port, "0.0.0.0"); - tcp_dispatchers.emplace(tcp_dest.get(), dispatcher); - } - else if (auto tcp_dest = dynamic_pointer_cast(edi_dest)) { - auto tcp_socket = make_shared(); - tcp_socket->connect(tcp_dest->dest_addr, tcp_dest->dest_port); - tcp_senders.emplace(tcp_dest.get(), tcp_socket); - } - else { - throw logic_error("EDI destination not implemented"); - } - } - - if (m_conf.interleaver_enabled()) { - edi_interleaver.SetLatency(m_conf.latency_frames); - } - - if (m_conf.dump) { - edi_debug_file.open("./edi.debug"); - } - - if (m_conf.verbose) { - etiLog.log(info, "EDI set up"); - } -} - -void Sender::write(const TagPacket& tagpacket) -{ - // Assemble into one AF Packet - edi::AFPacket af_packet = edi_afPacketiser.Assemble(tagpacket); - - if (m_conf.enable_pft) { - // Apply PFT layer to AF Packet (Reed Solomon FEC and Fragmentation) - vector edi_fragments = edi_pft.Assemble(af_packet); - - if (m_conf.verbose) { - fprintf(stderr, "EDI number of PFT fragment before interleaver %zu\n", - edi_fragments.size()); - } - - if (m_conf.interleaver_enabled()) { - edi_fragments = edi_interleaver.Interleave(edi_fragments); - } - - // Send over ethernet - for (const auto& edi_frag : edi_fragments) { - for (auto& dest : m_conf.destinations) { - if (const auto& udp_dest = dynamic_pointer_cast(dest)) { - Socket::InetAddress addr; - addr.resolveUdpDestination(udp_dest->dest_addr, m_conf.dest_port); - - udp_sockets.at(udp_dest.get())->send(edi_frag, addr); - } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { - tcp_dispatchers.at(tcp_dest.get())->write(edi_frag); - } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { - tcp_senders.at(tcp_dest.get())->sendall(edi_frag.data(), edi_frag.size()); - } - else { - throw logic_error("EDI destination not implemented"); - } - } - - if (m_conf.dump) { - ostream_iterator debug_iterator(edi_debug_file); - copy(edi_frag.begin(), edi_frag.end(), debug_iterator); - } - } - - if (m_conf.verbose) { - fprintf(stderr, "EDI number of PFT fragments %zu\n", - edi_fragments.size()); - } - } - else { - // Send over ethernet - for (auto& dest : m_conf.destinations) { - if (const auto& udp_dest = dynamic_pointer_cast(dest)) { - Socket::InetAddress addr; - addr.resolveUdpDestination(udp_dest->dest_addr, m_conf.dest_port); - - udp_sockets.at(udp_dest.get())->send(af_packet, addr); - } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { - tcp_dispatchers.at(tcp_dest.get())->write(af_packet); - } - else if (auto tcp_dest = dynamic_pointer_cast(dest)) { - tcp_senders.at(tcp_dest.get())->sendall(af_packet.data(), af_packet.size()); - } - else { - throw logic_error("EDI destination not implemented"); - } - } - - if (m_conf.dump) { - ostream_iterator debug_iterator(edi_debug_file); - copy(af_packet.begin(), af_packet.end(), debug_iterator); - } - } -} - -} diff --git a/src/dabOutput/edi/Transport.h b/src/dabOutput/edi/Transport.h deleted file mode 100644 index 9633275..0000000 --- a/src/dabOutput/edi/Transport.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright (C) 2019 - Matthias P. Braendli, matthias.braendli@mpb.li - - http://www.opendigitalradio.org - - EDI output, - UDP and TCP transports and their configuration - - */ -/* - This file is part of ODR-DabMux. - - ODR-DabMux is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - ODR-DabMux is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ODR-DabMux. If not, see . - */ - -#pragma once - -#include "config.h" -#include "dabOutput/edi/Config.h" -#include "AFPacket.h" -#include "PFT.h" -#include "Interleaver.h" -#include "Socket.h" -#include -#include -#include -#include -#include - -namespace edi { - -/** Configuration for EDI output */ - -class Sender { - public: - Sender(const configuration_t& conf); - - void write(const TagPacket& tagpacket); - - private: - configuration_t m_conf; - std::ofstream edi_debug_file; - - // The TagPacket will then be placed into an AFPacket - edi::AFPacketiser edi_afPacketiser; - - // The AF Packet will be protected with reed-solomon and split in fragments - edi::PFT edi_pft; - - // To mitigate for burst packet loss, PFT fragments can be sent out-of-order - edi::Interleaver edi_interleaver; - - std::unordered_map> udp_sockets; - std::unordered_map> tcp_dispatchers; - std::unordered_map> tcp_senders; -}; - -} - diff --git a/src/zmq2edi/EDISender.cpp b/src/zmq2edi/EDISender.cpp index 2128abf..2188f8a 100644 --- a/src/zmq2edi/EDISender.cpp +++ b/src/zmq2edi/EDISender.cpp @@ -79,7 +79,7 @@ void EDISender::print_configuration() void EDISender::send_eti_frame(uint8_t* p, metadata_t metadata) { edi::TagDETI edi_tagDETI; - edi::TagStarPTR edi_tagStarPtr; + edi::TagStarPTR edi_tagStarPtr("DETI"); map edi_subchannelToTag; // The above Tag Items will be assembled into a TAG Packet edi::TagPacket edi_tagpacket(edi_conf.tagpacket_alignment); diff --git a/src/zmq2edi/EDISender.h b/src/zmq2edi/EDISender.h index bb9c8bc..3525b4b 100644 --- a/src/zmq2edi/EDISender.h +++ b/src/zmq2edi/EDISender.h @@ -34,9 +34,9 @@ #include #include "ThreadsafeQueue.h" #include "dabOutput/dabOutput.h" -#include "dabOutput/edi/TagItems.h" -#include "dabOutput/edi/TagPacket.h" -#include "dabOutput/edi/Transport.h" +#include "edioutput/TagItems.h" +#include "edioutput/TagPacket.h" +#include "edioutput/Transport.h" // This metadata gets transmitted in the zmq stream struct metadata_t { -- cgit v1.2.3