Rework timestamping

 * Ensure MNSC and EDI carry the same timestamp
 * Rename `edi_tistoffset` to `tist_offset`
 * Remove conditional compilation of EDI output
 * Reset PPS so as to align ETI frames across mux restarts

10 files changed, 79 insertions, 103 deletions

diff --git a/INSTALL.md b/INSTALL.md
index 0f1439a..b2f0ae6 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -4,7 +4,7 @@ Required dependencies:
 * A C++11 compiler
 * Boost 1.48 or later
 * ZeroMQ 4 or later
-* (optional) cURL to download the TAI-UTC bulletin, needed for the EDI output.
+* (optional) cURL to download the TAI-UTC bulletin, needed for timestamps in EDI and ZMQ output.
 
 Dependencies on Debian
 ----------------------
diff --git a/Makefile.am b/Makefile.am
index fb6bae3..3dbb918 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -25,19 +25,12 @@ else
 GITVERSION_FLAGS =
 endif
 
-bin_PROGRAMS=odr-dabmux zmqinput-keygen
+bin_PROGRAMS=odr-dabmux zmqinput-keygen odr-zmq2edi
 
 if HAVE_OUTPUT_RAW_TEST
 bin_PROGRAMS+=odr-zmq2farsync
 endif
 
-if HAVE_EDI_TEST
-bin_PROGRAMS+=odr-zmq2edi
-CURL_LIBS    =-lcurl
-else
-CURL_LIBS    =
-endif
-
 FARSYNC_DIR=lib/farsync/linux
 INCLUDE=-I$(FARSYNC_DIR) -Ilib/charset -Ilib -Isrc
 
@@ -60,7 +53,7 @@ lib_charset_sources = lib/charset/charset.cpp \
 
 odr_dabmux_CFLAGS   =-Wall $(INCLUDE) $(GITVERSION_FLAGS)
 odr_dabmux_CXXFLAGS =-Wall -std=c++11 $(INCLUDE) $(GITVERSION_FLAGS) $(BOOST_CPPFLAGS) $(ZMQ_CPPFLAGS)
-odr_dabmux_LDADD    =$(ZMQ_LIBS) $(CURL_LIBS) $(BOOST_LDFLAGS) \
+odr_dabmux_LDADD    =$(ZMQ_LIBS) $(BOOST_LDFLAGS) \
 					 -lpthread $(BOOST_SYSTEM_LIB) $(BOOST_THREAD_LIB) $(BOOST_ASIO_LIB)
 
 odr_dabmux_SOURCES  =src/DabMux.cpp \
diff --git a/configure.ac b/configure.ac
index 25dec53..d65f51c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -85,16 +85,15 @@ AC_ARG_ENABLE([output_simul],
 AS_IF([test "x$enable_output_simul" = "xyes"],
         [AC_DEFINE(HAVE_OUTPUT_SIMUL, [1], [Define if SIMUL output is enabled])])
 
-# EDI
-AC_ARG_ENABLE([output_edi],
-        [AS_HELP_STRING([--disable-output-edi], [Disable EDI output])],
-        [], [enable_output_edi=yes])
-AS_IF([test "x$enable_output_edi" = "xyes"],
-       AC_CHECK_LIB(curl, curl_easy_init, [true], [AC_MSG_ERROR([cURL is required for EDI output])]))
-AS_IF([test "x$enable_output_edi" = "xyes"],
-        [AC_DEFINE(HAVE_OUTPUT_EDI, [1], [Define if EDI output is enabled])])
-AS_IF([test "x$enable_output_edi" = "xyes"],
-        [AC_DEFINE(HAVE_CURL, [1], [Define if cURL is available])])
+# EDI and ZMQ output metadata require TAI-UTC offset, which requires downloading the IETF TAI bulletin
+AC_CHECK_LIB(curl, curl_easy_init)
+have_curl=$ac_cv_lib_curl_curl_easy_init
+
+AS_IF([test "x$have_curl" = "xyes"],
+             [AC_DEFINE(HAVE_CURL, [1], [Define if cURL is available])])
+
+AS_IF([test "x$have_curl" = "xno"],
+             [AC_MSG_WARN([cURL not found, timestamps will not work])])
 
 AC_LANG_PUSH([C++])
 AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [CXXFLAGS="$CXXFLAGS -Wduplicated-cond"], [], ["-Werror"])
@@ -132,9 +131,6 @@ AC_DEFINE([HAVE_INPUT_ZEROMQ], [1], [Define if ZeroMQ input is enabled])
 AC_DEFINE([HAVE_OUTPUT_ZEROMQ], [1], [Define if ZeroMQ output is enabled])
 AC_DEFINE([HAVE_RC_ZEROMQ], [1], [Define if ZeroMQ enabled for rc])
 
-# Link against cURL
-AM_CONDITIONAL([HAVE_EDI_TEST],
-			   [test "x$enable_output_edi" = "xyes"])
 # Do not build odr-zmq2farsync if no RAW output
 AM_CONDITIONAL([HAVE_OUTPUT_RAW_TEST],
 			   [test "x$enable_output_raw" = "xyes"])
@@ -157,7 +153,7 @@ echo
 echo "Outputs:"
 enabled=""
 disabled=""
-for output in file fifo udp tcp raw simul edi
+for output in file fifo udp tcp raw simul
 do
     eval var=\$enable_output_$output
     AS_IF([test "x$var" = "xyes"],
@@ -167,6 +163,13 @@ done
 echo "  Enabled: $enabled zmq"
 echo "  Disabled: $disabled"
 
+if test "$have_curl" = "no" ; then
+echo
+echo "WARNING! cURL not found: ODR-DabMux will not support timestamps"
+echo
+fi
+
+
 echo
 echo "***********************************************"
 echo
diff --git a/doc/advanced.mux b/doc/advanced.mux
index ab145fa..c2a4411 100644
--- a/doc/advanced.mux
+++ b/doc/advanced.mux
@@ -28,13 +28,13 @@ general {
     writescca false
 
     ; Enable timestamp definition necessary for SFN
-    ; This also enables time encoding using the MNSC.
+    ; This also enables time encoding using the MNSC and in EDI.
     tist false
 
-    ; On startup, EDI time is initialised to system time. If you want
+    ; On startup, the timestamp is initialised to system time. If you want
     ; to add an offset, uncomment the following line and give a number
     ; in seconds.
-    ; tist_edioffset 0
+    ; tist_offset 0
 
     ; The management server is a simple TCP server that can present
     ; statistics data (buffers, overruns, underruns, etc)
diff --git a/doc/example.mux b/doc/example.mux
index 76d6828..6c2bc18 100644
--- a/doc/example.mux
+++ b/doc/example.mux
@@ -236,7 +236,7 @@ outputs {
         ; If TIST is enabled, requires leap-second information (see example.mux)
         ;
         ; WARNING! requires ODR-DabMux to be compiled with
-        ; EDI output, and this will enable leap second download
+        ; cURL support, and this will enable leap second download
         ; as for the EDI output!
         allowmetadata true
     }
diff --git a/src/DabMultiplexer.cpp b/src/DabMultiplexer.cpp
index e157523..3a7f31f 100644
--- a/src/DabMultiplexer.cpp
+++ b/src/DabMultiplexer.cpp
@@ -3,7 +3,7 @@
    2011, 2012 Her Majesty the Queen in Right of Canada (Communications
    Research Center Canada)
 
-   Copyright (C) 2017
+   Copyright (C) 2019
    Matthias P. Braendli, matthias.braendli@mpb.li
    */
 /*
@@ -83,19 +83,12 @@ DabMultiplexer::DabMultiplexer(
         boost::property_tree::ptree pt) :
     RemoteControllable("mux"),
     m_pt(pt),
-    timestamp(0),
-    MNSC_increment_time(false),
-    sync(0x49C5F8),
-    currentFrame(0),
     ensemble(std::make_shared<dabEnsemble>()),
-    m_tai_clock_required(false),
     m_clock_tai(split_pipe_separated_string(pt.get("general.tai_clock_bulletins", ""))),
     fig_carousel(ensemble)
 {
-    gettimeofday(&mnsc_time, nullptr);
-
     RC_ADD_PARAMETER(frames, "Show number of frames generated [read-only]");
-    RC_ADD_PARAMETER(tist_edioffset, "EDI Time offset in seconds");
+    RC_ADD_PARAMETER(tist_offset, "Timestamp offset in integral number of seconds");
 
     rcs.enrol(&m_clock_tai);
 }
@@ -109,7 +102,6 @@ void DabMultiplexer::set_edi_config(const edi_configuration_t& new_edi_conf)
 {
     edi_conf = new_edi_conf;
 
-#if HAVE_OUTPUT_EDI
     if (edi_conf.verbose) {
         etiLog.log(info, "Setup EDI");
     }
@@ -146,7 +138,6 @@ void DabMultiplexer::set_edi_config(const edi_configuration_t& new_edi_conf)
     // The AF Packet will be protected with reed-solomon and split in fragments
     edi::PFT pft(edi_conf);
     edi_pft = pft;
-#endif
 }
 
 
@@ -185,15 +176,24 @@ void DabMultiplexer::prepare(bool require_tai_clock)
      * Ideally, we must be able to restart transmission s.t. the receiver
      * synchronisation is preserved.
      */
-    gettimeofday(&mnsc_time, nullptr);
+    using Sec = chrono::seconds;
+    const auto now = chrono::time_point_cast<Sec>(chrono::system_clock::now());
 
-#if HAVE_OUTPUT_EDI
-    edi_time = chrono::system_clock::now();
+    edi_time = chrono::system_clock::to_time_t(now);
+
+    // We define that when the time is multiple of six seconds, the timestamp
+    // (PPS offset) is 0. This ensures consistency of TIST even across a
+    // mux restart
+    timestamp = 0;
+    edi_time -= (edi_time % 6);
+    while (edi_time < chrono::system_clock::to_time_t(now)) {
+        increment_timestamp();
+    }
 
     // Try to load offset once
 
     bool tist_enabled = m_pt.get("general.tist", false);
-    m_tist_edioffset = m_pt.get<int>("general.tist_edioffset", 0);
+    m_tist_offset = m_pt.get<int>("general.tist_offset", 0);
 
     m_tai_clock_required = (tist_enabled and edi_conf.enabled()) or require_tai_clock;
 
@@ -204,7 +204,9 @@ void DabMultiplexer::prepare(bool require_tai_clock)
         catch (const std::runtime_error& e) {
             etiLog.level(error) <<
                 "Could not initialise TAI clock properly. "
-                "Do you have a working internet connection?";
+                "TAI clock is required when TIST is enabled with an EDI output, "
+                "or when a ZMQ output with metadata is used. "
+                "Error: " << e.what();
             throw;
         }
     }
@@ -212,10 +214,7 @@ void DabMultiplexer::prepare(bool require_tai_clock)
     if (edi_conf.interleaver_enabled()) {
         edi_interleaver.SetLatency(edi_conf.latency_frames);
     }
-#endif
 
-    // Shift ms by 14 to Timestamp level 2, see below in Section TIST
-    timestamp = (mnsc_time.tv_usec / 1000) << 14;
 }
 
 
@@ -396,6 +395,15 @@ void DabMultiplexer::prepare_data_inputs()
     }
 }
 
+void DabMultiplexer::increment_timestamp()
+{
+    timestamp += 24 << 14; // Shift 24ms by 14 to Timestamp level 2
+    if (timestamp > 0xf9FFff) {
+        timestamp -= 0xfa0000; // Substract 16384000, corresponding to one second
+        edi_time += 1;
+    }
+}
+
 /*  Each call creates one ETI frame */
 void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs)
 {
@@ -545,18 +553,17 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
 
     eoh->MNSC = 0;
 
+    if (fc->FP == 0) {
+        // update the latched time only when FP==0 to ensure MNSC encodes
+        // a consistent time
+        edi_time_latched_for_mnsc = edi_time + m_tist_offset;
+    }
+
     struct tm time_tm;
-    gmtime_r(&mnsc_time.tv_sec, &time_tm);
-    switch (fc->FP & 0x3)
-    {
+    gmtime_r(&edi_time_latched_for_mnsc, &time_tm);
+    switch (fc->FP & 0x3) {
         case 0:
-            if (MNSC_increment_time)
             {
-                MNSC_increment_time = false;
-                mnsc_time.tv_sec += 1;
-            }
-            {
-
                 eti_MNSC_TIME_0 *mnsc = (eti_MNSC_TIME_0 *) &eoh->MNSC;
                 // Set fields according to ETS 300 799 -- 5.5.1 and A.2.2
                 mnsc->type = 0;
@@ -670,17 +677,12 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
         edi_tagDETI.tsta = 0xffffff;
     }
 
-#if HAVE_OUTPUT_EDI
     const bool tist_enabled = m_pt.get("general.tist", false);
 
     if (tist_enabled and m_tai_clock_required) {
         try {
             const auto tai_utc_offset = m_clock_tai.get_offset();
-
-            edi_tagDETI.set_edi_time(edi_time +
-                    std::chrono::seconds(m_tist_edioffset),
-                    tai_utc_offset);
-
+            edi_tagDETI.set_edi_time(edi_time + m_tist_offset, tai_utc_offset);
             edi_tagDETI.atstf = true;
 
             for (auto output : outputs) {
@@ -701,7 +703,6 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
             etiLog.level(error) << "Could not get UTC-TAI offset for EDI timestamp";
         }
     }
-#endif
 
 
     /* Coding of the TIST, according to ETS 300 799 Annex C
@@ -716,17 +717,7 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
     time resolution
     */
 
-    timestamp += 24 << 14; // Shift 24ms by 14 to Timestamp level 2
-    if (timestamp > 0xf9FFff)
-    {
-        timestamp -= 0xfa0000; // Substract 16384000, corresponding to one second
-
-        // Also update MNSC time for next time FP==0
-        MNSC_increment_time = true;
-
-        // Immediately update edi time
-        edi_time += chrono::seconds(1);
-    }
+    increment_timestamp();
 
     /**********************************************************************
      ***********   Section FRPD   *****************************************
@@ -755,11 +746,9 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
         }
     }
 
-#if HAVE_OUTPUT_EDI
     /**********************************************************************
      ***********   Finalise and send EDI   ********************************
      **********************************************************************/
-
     if (edi_conf.enabled()) {
         // put tags *ptr, DETI and all subchannels into one TagPacket
         edi_tagpacket.tag_items.push_back(&edi_tagStarPtr);
@@ -822,7 +811,6 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
             }
         }
     }
-#endif // HAVE_OUTPUT_EDI
 
 #if _DEBUG
     /**********************************************************************
@@ -831,12 +819,12 @@ void DabMultiplexer::mux_frame(std::vector<std::shared_ptr<DabOutput> >& outputs
     if (currentFrame % 100 == 0) {
         if (enableTist) {
             etiLog.log(info, "ETI frame number %i Timestamp: %d + %f",
-                    currentFrame, mnsc_time.tv_sec,
+                    currentFrame, edi_time,
                     (timestamp & 0xFFFFFF) / 16384000.0);
         }
         else {
             etiLog.log(info, "ETI frame number %i Time: %d, no TIST",
-                    currentFrame, mnsc_time.tv_sec);
+                    currentFrame, edi_time);
         }
     }
 #endif
@@ -871,8 +859,8 @@ void DabMultiplexer::set_parameter(const std::string& parameter,
             " is read-only";
         throw ParameterError(ss.str());
     }
-    else if (parameter == "tist_edioffset") {
-        m_tist_edioffset = std::stoi(value);
+    else if (parameter == "tist_offset") {
+        m_tist_offset = std::stoi(value);
     }
     else {
         stringstream ss;
@@ -889,8 +877,8 @@ const std::string DabMultiplexer::get_parameter(const std::string& parameter) co
     if (parameter == "frames") {
         ss << currentFrame;
     }
-    else if (parameter == "tist_edioffset") {
-        ss << m_tist_edioffset;
+    else if (parameter == "tist_offset") {
+        ss << m_tist_offset;
     }
     else {
         ss << "Parameter '" << parameter <<
diff --git a/src/DabMultiplexer.h b/src/DabMultiplexer.h
index 127ecfb..7090be7 100644
--- a/src/DabMultiplexer.h
+++ b/src/DabMultiplexer.h
@@ -3,7 +3,7 @@
    2011, 2012 Her Majesty the Queen in Right of Canada (Communications
    Research Center Canada)
 
-   Copyright (C) 2016
+   Copyright (C) 2019
    Matthias P. Braendli, matthias.braendli@mpb.li
    */
 /*
@@ -80,26 +80,25 @@ class DabMultiplexer : public RemoteControllable {
         void prepare_subchannels(void);
         void prepare_services_components(void);
         void prepare_data_inputs(void);
+        void increment_timestamp(void);
 
         boost::property_tree::ptree m_pt;
 
-        unsigned timestamp;
-        bool MNSC_increment_time;
-        struct timeval mnsc_time;
-        std::chrono::system_clock::time_point edi_time;
+        unsigned timestamp = 0;
+        std::time_t edi_time;
+        std::time_t edi_time_latched_for_mnsc;
 
         edi_configuration_t edi_conf;
 
-        uint32_t sync;
-        unsigned long currentFrame;
+        uint32_t sync = 0x49C5F8;
+        unsigned long currentFrame = 0;
 
         std::shared_ptr<dabEnsemble> ensemble;
 
-        int m_tist_edioffset = 0;
-        bool m_tai_clock_required;
+        int m_tist_offset = 0;
+        bool m_tai_clock_required = false;
         ClockTAI m_clock_tai;
 
-#if HAVE_OUTPUT_EDI
         std::ofstream edi_debug_file;
 
         // The TagPacket will then be placed into an AFPacket
@@ -110,7 +109,6 @@ class DabMultiplexer : public RemoteControllable {
 
         // To mitigate for burst packet loss, PFT fragments can be sent out-of-order
         edi::Interleaver edi_interleaver;
-#endif // HAVE_OUTPUT_EDI
 
         /* New FIG Carousel */
         FIC::FIGCarousel fig_carousel;
diff --git a/src/DabMux.cpp b/src/DabMux.cpp
index 3185fb3..4b06eb4 100644
--- a/src/DabMux.cpp
+++ b/src/DabMux.cpp
@@ -291,7 +291,6 @@ int main(int argc, char *argv[])
             }
 
             if (outputuid == "edi") {
-#if HAVE_OUTPUT_EDI
                 ptree pt_edi = pt_outputs.get_child("edi");
                 for (auto pt_edi_dest : pt_edi.get_child("destinations")) {
                     edi_destination_t dest;
@@ -330,9 +329,6 @@ int main(int argc, char *argv[])
                 edi_conf.tagpacket_alignment = pt_edi.get<unsigned int>("tagpacket_alignment", 8);
 
                 mux.set_edi_config(edi_conf);
-#else
-                throw runtime_error("EDI output not compiled in");
-#endif
             }
             else if (outputuid == "zeromq") {
 #if defined(HAVE_OUTPUT_ZEROMQ)
@@ -463,7 +459,6 @@ int main(int argc, char *argv[])
         etiLog.log(info, "--- Output list ---");
         printOutputs(outputs);
 
-#if HAVE_OUTPUT_EDI
         if (edi_conf.enabled()) {
             etiLog.level(info) << "EDI";
             etiLog.level(info) << " verbose     " << edi_conf.verbose;
@@ -479,7 +474,6 @@ int main(int argc, char *argv[])
                 etiLog.level(info) << " interleave     " << edi_conf.latency_frames * 24 << " ms";
             }
         }
-#endif
 
         size_t limit = pt.get("general.nbframes", 0);
 
diff --git a/src/dabOutput/edi/TagItems.cpp b/src/dabOutput/edi/TagItems.cpp
index 631b88d..5511efb 100644
--- a/src/dabOutput/edi/TagItems.cpp
+++ b/src/dabOutput/edi/TagItems.cpp
@@ -132,13 +132,13 @@ std::vector<uint8_t> TagDETI::Assemble()
     return packet;
 }
 
-void TagDETI::set_edi_time(const std::chrono::system_clock::time_point& t, int tai_utc_offset)
+void TagDETI::set_edi_time(const std::time_t t, int tai_utc_offset)
 {
     utco = tai_utc_offset - 32;
 
     const std::time_t posix_timestamp_1_jan_2000 = 946684800;
 
-    seconds = std::chrono::system_clock::to_time_t(t) - posix_timestamp_1_jan_2000 + utco;
+    seconds = t - posix_timestamp_1_jan_2000 + utco;
 }
 
 std::vector<uint8_t> TagESTn::Assemble()
diff --git a/src/dabOutput/edi/TagItems.h b/src/dabOutput/edi/TagItems.h
index 8666053..0559bf3 100644
--- a/src/dabOutput/edi/TagItems.h
+++ b/src/dabOutput/edi/TagItems.h
@@ -86,7 +86,7 @@ class TagDETI : public TagItem
         uint8_t utco = 0;
 
         /* Update the EDI time. t is in UTC */
-        void set_edi_time(const std::chrono::system_clock::time_point &t, int tai_utc_offset);
+        void set_edi_time(const std::time_t t, int tai_utc_offset);
 
         /* The number of SI seconds since 2000-01-01 T 00:00:00 UTC as an
          * unsigned 32-bit quantity. Contrary to POSIX, this value also