Add load_entire_file option for file inputs

4 files changed, 159 insertions, 27 deletions

diff --git a/doc/advanced.mux b/doc/advanced.mux
index fd7a06c..61328d7 100644
--- a/doc/advanced.mux
+++ b/doc/advanced.mux
@@ -278,6 +278,20 @@ subchannels {
        ; (1 << 20) + (1 << 17) + (1 << 0) = 0x120001
        ;inputuri "prbs://:0x120001
    }
+
+   ; An example using 'enhancedpacket' to send out epg data
+   ; See http://wiki.opendigitalradio.org/How_to_configure_SPI_(Service_and_Programme_Information)_for_ODR-DabMux
+    sub-data {
+        id 6
+        type enhancedpacket
+        ; Settings this flag to true makes ODR-DabMux preload the entire file contents into memory
+        ; This allows you to replace the file contents while the current file data still gets transmitted to the
+        ; end.
+        load_entire_file true
+        inputfile "./epg.dat"
+        protection 1
+        bitrate 32
+    }
 }
 
 ; For now, each component links one service to one subchannel
@@ -299,6 +313,7 @@ components {
 
         ; FIG 0/13 user application was previously configured using the figtype setting, which
         ; allowed only a single user application.
+        ; Now more than one user application can be defined per component, using the
         ; Using the 'user-applications' section, several can be defined per component.
         ; Do not use both figtype and user-applications.
 
diff --git a/src/ConfigParser.cpp b/src/ConfigParser.cpp
index dbbc2fb..a845bef 100644
--- a/src/ConfigParser.cpp
+++ b/src/ConfigParser.cpp
@@ -1090,16 +1090,24 @@ static void setup_subchannel_from_ptree(shared_ptr<DabSubchannel>& subchan,
         throw runtime_error(ss.str());
     }
 
-    const bool nonblock = pt.get("nonblock", false);
-    if (nonblock) {
+    if (pt.get("nonblock", false)) {
         if (auto filein = dynamic_pointer_cast<Inputs::FileBase>(subchan->input)) {
-            filein->setNonblocking(nonblock);
+            filein->setNonblocking(true);
         }
         else {
             etiLog.level(warn) << "The nonblock option is not supported";
         }
     }
 
+    if (pt.get("load_entire_file", false)) {
+        if (auto filein = dynamic_pointer_cast<Inputs::FileBase>(subchan->input)) {
+            filein->setLoadEntireFile(true);
+        }
+        else {
+            etiLog.level(warn) << "The load_entire_file option is not supported";
+        }
+    }
+
     const string bufferManagement = pt.get("buffer-management", "prebuffering");
     if (bufferManagement == "prebuffering") {
         subchan->input->setBufferManagement(Inputs::BufferManagement::Prebuffering);
diff --git a/src/input/File.cpp b/src/input/File.cpp
index 46bfb59..8f5d3e3 100644
--- a/src/input/File.cpp
+++ b/src/input/File.cpp
@@ -2,7 +2,7 @@
    Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications
    Research Center Canada)
 
-   Copyright (C) 2019 Matthias P. Braendli
+   Copyright (C) 2022 Matthias P. Braendli
     http://www.opendigitalradio.org
 
    */
@@ -62,15 +62,22 @@ __attribute((packed))
 
 void FileBase::open(const std::string& name)
 {
-    int flags = O_RDONLY | O_BINARY;
-    if (m_nonblock) {
-        flags |= O_NONBLOCK;
+    m_filename = name;
+
+    if (m_load_entire_file) {
+        load_entire_file();
     }
+    else {
+        int flags = O_RDONLY | O_BINARY;
+        if (m_nonblock) {
+            flags |= O_NONBLOCK;
+        }
 
-    m_fd = ::open(name.c_str(), flags);
-    if (m_fd == -1) {
-        throw runtime_error("Could not open input file " + name + ": " +
-            strerror(errno));
+        m_fd = ::open(name.c_str(), flags);
+        if (m_fd == -1) {
+            throw runtime_error("Could not open input file " + name + ": " +
+                    strerror(errno));
+        }
     }
 }
 
@@ -102,15 +109,82 @@ void FileBase::close()
 
 void FileBase::setNonblocking(bool nonblock)
 {
+    if (m_load_entire_file) {
+        throw runtime_error("Cannot set both nonblock and load_entire_file");
+    }
     m_nonblock = nonblock;
 }
 
-int FileBase::rewind()
+void FileBase::setLoadEntireFile(bool load_entire_file)
 {
-    return ::lseek(m_fd, 0, SEEK_SET);
+    if (m_nonblock) {
+        throw runtime_error("Cannot set both nonblock and load_entire_file");
+    }
+    m_load_entire_file = load_entire_file;
+}
+
+ssize_t FileBase::rewind()
+{
+    if (m_load_entire_file) {
+        return load_entire_file();
+    }
+    else if (m_fd) {
+        return ::lseek(m_fd, 0, SEEK_SET);
+    }
+    else {
+        throw runtime_error("Cannot rewind");
+    }
+}
+
+ssize_t FileBase::load_entire_file()
+{
+    m_file_contents_offset = 0;
+    // Read entire file in chunks of 4MiB
+    constexpr size_t blocksize = 4 * 1024 * 1024;
+    constexpr int flags = O_RDONLY | O_BINARY;
+    m_fd = ::open(m_filename.c_str(), flags);
+    if (m_fd == -1) {
+        if (not m_file_open_alert_shown) {
+            etiLog.level(error) << "Could not open input file " << m_filename << ": " <<
+                strerror(errno);
+        }
+        m_file_open_alert_shown = true;
+        return -1;
+    }
+
+    // Do not clear the buffer if the file open fails
+    m_file_contents.clear();
+    ssize_t offset = 0;
+    ssize_t r = 0;
+    do {
+        m_file_contents.resize(offset + blocksize);
+        uint8_t *buffer = m_file_contents.data() + offset;
+
+        r = read(m_fd, buffer, blocksize);
+
+        if (r == -1) {
+            if (not m_file_open_alert_shown) {
+                etiLog.level(error) << "Can't read file " << strerror(errno);
+            }
+            m_file_contents.clear(); // ensures size is not larger than what we read
+            close();
+            m_file_open_alert_shown = true;
+            return -1;
+        }
+
+        m_file_contents.resize(offset + r);
+        offset += r;
+    } while (r > 0);
+
+    close();
+    if (m_file_open_alert_shown) {
+        etiLog.level(info) << "Loaded " << m_file_contents.size() << " bytes from " << m_filename;
+    }
+    m_file_open_alert_shown = false;
+    return m_file_contents.size();
 }
 
-ssize_t FileBase::readFromFile(uint8_t* buffer, size_t size)
+ssize_t FileBase::readFromFile(uint8_t *buffer, size_t size)
 {
     using namespace std;
 
@@ -133,7 +207,7 @@ ssize_t FileBase::readFromFile(uint8_t* buffer, size_t size)
                 return 0;
             }
             else if (ret == -1) {
-                etiLog.level(alert) << "ERROR: Can't read file " << strerror(errno);
+                etiLog.level(error) << "Can't read file " << strerror(errno);
                 return -1;
             }
 
@@ -156,11 +230,37 @@ ssize_t FileBase::readFromFile(uint8_t* buffer, size_t size)
             return 0;
         }
     }
+    else if (m_load_entire_file) {
+        // Handle file read errors.
+        if (m_file_contents.size() == 0) {
+            rewind();
+        }
+
+        if (m_file_contents.size() == 0) {
+            memset(buffer, 0, size);
+        }
+        else {
+            size_t remain = size;
+
+            while (m_file_contents_offset + remain > m_file_contents.size()) {
+                copy(   m_file_contents.begin() + m_file_contents_offset,
+                        m_file_contents.end(), buffer);
+                size_t copied = m_file_contents.size() - m_file_contents_offset;
+                remain -= copied;
+                rewind();
+            }
+
+            copy(   m_file_contents.begin() + m_file_contents_offset,
+                    m_file_contents.begin() + m_file_contents_offset + remain, buffer);
+            m_file_contents_offset += remain;
+        }
+        return size;
+    }
     else {
         ret = read(m_fd, buffer, size);
 
         if (ret == -1) {
-            etiLog.level(alert) << "ERROR: Can't read file " << strerror(errno);
+            etiLog.level(error) << "Can't read file " << strerror(errno);
             return -1;
         }
 
@@ -168,19 +268,19 @@ ssize_t FileBase::readFromFile(uint8_t* buffer, size_t size)
             ssize_t sizeOut = ret;
             etiLog.log(info, "reach end of file -> rewinding");
             if (rewind() == -1) {
-                etiLog.log(alert, "ERROR: Can't rewind file");
+                etiLog.log(error, "Can't rewind file");
                 return -1;
             }
 
             ret = read(m_fd, buffer + sizeOut, size - sizeOut);
             if (ret == -1) {
-                etiLog.log(alert, "ERROR: Can't read file");
+                etiLog.log(error, "Can't read file");
                 perror("");
                 return -1;
             }
 
             if (ret < (ssize_t)size) {
-                etiLog.log(alert, "ERROR: Not enough data in file");
+                etiLog.log(error, "Not enough data in file");
                 return -1;
             }
         }
@@ -228,20 +328,20 @@ READ_SUBCHANNEL:
                 goto READ_SUBCHANNEL;
             }
         case MPEG_FILE_ERROR:
-            etiLog.log(alert, "can't read file (%i) -> frame muted", errno);
+            etiLog.log(error, "can't read file (%i) -> frame muted", errno);
             perror("");
             goto MUTE_SUBCHANNEL;
         case MPEG_SYNC_NOT_FOUND:
-            etiLog.log(alert, "mpeg sync not found, maybe is not a valid file "
+            etiLog.log(error, "mpeg sync not found, maybe is not a valid file "
                     "-> frame muted");
             goto MUTE_SUBCHANNEL;
         case MPEG_INVALID_FRAME:
-            etiLog.log(alert, "file is not a valid mpeg file "
+            etiLog.log(error, "file is not a valid mpeg file "
                     "-> frame muted");
             goto MUTE_SUBCHANNEL;
         default:
             if (result < 0) {
-                etiLog.log(alert,
+                etiLog.log(error,
                         "unknown error (code = %i) -> frame muted",
                         result);
 MUTE_SUBCHANNEL:
@@ -275,7 +375,7 @@ MUTE_SUBCHANNEL:
                         break;
                     default:
                         if (result < 0) {
-                            etiLog.log(alert, "mpeg file has an invalid DAB "
+                            etiLog.log(error, "mpeg file has an invalid DAB "
                                     "mpeg frame (unknown reason: %i)", result);
                         }
                         break;
diff --git a/src/input/File.h b/src/input/File.h
index 39ce7fd..f317d46 100644
--- a/src/input/File.h
+++ b/src/input/File.h
@@ -2,7 +2,7 @@
    Copyright (C) 2009 Her Majesty the Queen in Right of Canada (Communications
    Research Center Canada)
 
-   Copyright (C) 2018 Matthias P. Braendli
+   Copyright (C) 2022 Matthias P. Braendli
     http://www.opendigitalradio.org
 
    */
@@ -43,23 +43,32 @@ class FileBase : public InputBase {
         virtual void close();
 
         virtual void setNonblocking(bool nonblock);
+        virtual void setLoadEntireFile(bool load_entire_file);
 
+    protected:
         /* Rewind the file
          * Returns -1 on failure, 0 on success
          */
-        virtual int rewind();
-    protected:
+        virtual ssize_t rewind();
         /* Read len bytes from the file into buf, and return
          * the number of bytes read, or -1 in case of error.
          */
         virtual ssize_t readFromFile(uint8_t* buf, size_t len);
 
+        virtual ssize_t load_entire_file();
+
         // We use unix open() instead of fopen() because
         // of non-blocking I/O
         int m_fd = -1;
 
+        std::string m_filename;
         bool m_nonblock = false;
+        bool m_load_entire_file = false;
         std::vector<uint8_t> m_nonblock_buffer;
+
+        size_t m_file_contents_offset = 0;
+        std::vector<uint8_t> m_file_contents;
+        bool m_file_open_alert_shown = false;
 };
 
 class MPEGFile : public FileBase {