Remove mot-encoder

7 files changed, 0 insertions, 2921 deletions

diff --git a/src/charset-test.cpp b/src/charset-test.cpp
deleted file mode 100644
index d65fcc7..0000000
--- a/src/charset-test.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-    Copyright (C) 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-    charset.cpp
-         This is test code for the character set conversion. It does not
-         get compiled by default.
-         Please use
-         g++ -Wall -Wextra -std=c++11 -o charset-test charset-test.cpp
-         to compile this, and create a input.txt file with test data.
-
-    Authors:
-         Matthias P. Braendli <matthias@mpb.li>
-*/
-
-#include "charset.h"
-#include <iostream>
-#include <fstream>
-
-#include <cstdio>
-
-using namespace std;
-
-int main(int argc, char** argv)
-{
-    string test_file_path("input.txt");
-
-    ifstream fs8(test_file_path);
-    if (!fs8.is_open()) {
-        cerr << "Could not open " << test_file_path << endl;
-        return 1;
-    }
-
-    CharsetConverter conv;
-
-    string line;
-    // Play with all the lines in the file
-    while (getline(fs8, line)) {
-        cout << conv.convert(line) << endl;
-    }
-
-    return 0;
-}
-
-
diff --git a/src/charset.h b/src/charset.h
deleted file mode 100644
index 8d1e1a2..0000000
--- a/src/charset.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-    Copyright (C) 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-/*!
-    \file charset.h
-    \brief Define the EBU charset according to ETSI TS 101 756v1.8.1 for DLS encoding
-
-    \author Matthias P. Braendli <matthias@mpb.li>
-    \author Lindsay Cornell
-*/
-
-#ifndef __CHARSET_H_
-#define __CHARSET_H_
-
-#include "utf8.h"
-#include <string>
-#include <vector>
-#include <algorithm>
-
-/**********************************************/
-/************* BIG FAT WARNING ****************/
-/**********************************************/
-/**** Make sure this file is always saved  ****/
-/**** encoded in UTF-8, otherwise you will ****/
-/****      mess up the table below !       ****/
-/**********************************************/
-/********* END OF BIG FAT WARNING *************/
-/**********************************************/
-
-
-#define CHARSET_TABLE_OFFSET 1 // NUL at index 0 cannot be represented
-#define CHARSET_TABLE_ENTRIES (256 - CHARSET_TABLE_OFFSET)
-const char* utf8_encoded_EBU_Latin[CHARSET_TABLE_ENTRIES] = {
-     "Ę", "Į", "Ų", "Ă", "Ė", "Ď", "Ș", "Ț", "Ċ", "\n","\v","Ġ", "Ĺ", "Ż", "Ń",
-"ą", "ę", "į", "ų", "ă", "ė", "ď", "ș", "ț", "ċ", "Ň", "Ě", "ġ", "ĺ", "ż", "\u0082",
-" ", "!", "\"","#", "ł", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/",
-"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?",
-"@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O",
-"P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "Ů", "]", "Ł", "_",
-"Ą", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
-"p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "«", "ů", "»", "Ľ", "Ħ",
-"á", "à", "é", "è", "í", "ì", "ó", "ò", "ú", "ù", "Ñ", "Ç", "Ş", "ß", "¡", "Ÿ",
-"â", "ä", "ê", "ë", "î", "ï", "ô", "ö", "û", "ü", "ñ", "ç", "ş", "ğ", "ı", "ÿ",
-"Ķ", "Ņ", "©", "Ģ", "Ğ", "ě", "ň", "ő", "Ő", "€", "£", "$", "Ā", "Ē", "Ī", "Ū",
-"ķ", "ņ", "Ļ", "ģ", "ļ", "İ", "ń", "ű", "Ű", "¿", "ľ", "°", "ā", "ē", "ī", "ū",
-"Á", "À", "É", "È", "Í", "Ì", "Ó", "Ò", "Ú", "Ù", "Ř", "Č", "Š", "Ž", "Ð", "Ŀ",
-"Â", "Ä", "Ê", "Ë", "Î", "Ï", "Ô", "Ö", "Û", "Ü", "ř", "č", "š", "ž", "đ", "ŀ",
-"Ã", "Å", "Æ", "Œ", "ŷ", "Ý", "Õ", "Ø", "Þ", "Ŋ", "Ŕ", "Ć", "Ś", "Ź", "Ť", "ð",
-"ã", "å", "æ", "œ", "ŵ", "ý", "õ", "ø", "þ", "ŋ", "ŕ", "ć", "ś", "ź", "ť", "ħ"};
-
-class CharsetConverter
-{
-    public:
-        CharsetConverter() {
-            /*! Build the converstion table that contains the known code points,
-             * at the indices corresponding to the EBU Latin table
-             */
-            using namespace std;
-            for (size_t i = 0; i < CHARSET_TABLE_ENTRIES; i++) {
-                string table_entry(utf8_encoded_EBU_Latin[i]);
-                string::iterator it = table_entry.begin();
-                uint32_t code_point = utf8::next(it, table_entry.end());
-                m_conversion_table.push_back(code_point);
-            }
-        }
-
-        /*! Convert a UTF-8 encoded text line into an EBU Latin encoded byte stream
-         */
-        std::string convert(std::string line_utf8) {
-            using namespace std;
-
-            // check for invalid utf-8, we only convert up to the first error
-            string::iterator end_it = utf8::find_invalid(line_utf8.begin(), line_utf8.end());
-
-            // Convert it to utf-32
-            vector<uint32_t> utf32line;
-            utf8::utf8to32(line_utf8.begin(), end_it, back_inserter(utf32line));
-
-            string encoded_line(utf32line.size(), '0');
-
-            // Try to convert each codepoint
-            for (size_t i = 0; i < utf32line.size(); i++) {
-                vector<uint32_t>::iterator iter = find(m_conversion_table.begin(),
-                        m_conversion_table.end(), utf32line[i]);
-                if (iter != m_conversion_table.end()) {
-                    size_t index = std::distance(m_conversion_table.begin(), iter);
-
-                    encoded_line[i] = (char)(index + CHARSET_TABLE_OFFSET);
-                }
-                else {
-                    encoded_line[i] = ' ';
-                }
-            }
-            return encoded_line;
-        }
-
-    private:
-
-        std::vector<uint32_t> m_conversion_table;
-};
-
-#endif
diff --git a/src/mot-encoder.cpp b/src/mot-encoder.cpp
deleted file mode 100644
index 4cb6688..0000000
--- a/src/mot-encoder.cpp
+++ /dev/null
@@ -1,1831 +0,0 @@
-/*
-    Copyright (C) 2014 CSP Innovazione nelle ICT s.c.a r.l. (http://rd.csp.it/)
-
-    Copyright (C) 2014, 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
-    Copyright (C) 2015, 2016 Stefan Pöschel (http://opendigitalradio.org)
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-/*!
-    \file mot-encoder.cpp
-    \brief Generete PAD data for MOT Slideshow and DLS
-
-    \author Sergio Sagliocco <sergio.sagliocco@csp.it>
-    \author Matthias P. Braendli <matthias@mpb.li>
-    \author Stefan Pöschel <odr@basicmaster.de>
-*/
-
-#include <cstdio>
-#include <stdlib.h>
-#include <stdint.h>
-#include <unistd.h>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <fstream>
-#include <algorithm>
-#include <vector>
-#include <deque>
-#include <list>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <getopt.h>
-#include "config.h"
-#include "charset.h"
-
-
-#if HAVE_MAGICKWAND
-#  include <wand/magick_wand.h>
-#endif
-
-#define DEBUG 0
-
-#define SLEEPDELAY_DEFAULT 10 //seconds
-
-extern "C" {
-#include "lib_crc.h"
-}
-
-#define MIN(a,b) (((a)<(b))?(a):(b))
-#define MAX(a,b) (((a)>(b))?(a):(b))
-
-#define XSTR(x) #x
-#define STR(x) XSTR(x)
-
-#define MAXSEGLEN 8189 // Bytes (EN 301 234 v2.1.1, ch. 5.1.1)
-#define MAXDLS 128 // chars
-#define MAXSLIDESIZE 51200 // Bytes (TS 101 499 v3.1.1, ch. 9.1.2)
-
-// Roll-over value for fidx
-#define MAXSLIDEID 9999
-
-// How many slides to keep in history
-#define MAXHISTORYLEN 50
-
-// Do not allow the image compressor to go below
-// JPEG quality 40
-#define MINQUALITY 40
-
-// Charsets from TS 101 756
-#define CHARSET_COMPLETE_EBU_LATIN 0 //!< Complete EBU Latin based repertoire
-#define CHARSET_EBU_LATIN_CY_GR 1 //!< EBU Latin based common core, Cyrillic, Greek
-#define CHARSET_EBU_LATIN_AR_HE_CY_GR 2 //!< EBU Latin based core, Arabic, Hebrew, Cyrillic and Greek
-#define CHARSET_ISO_LATIN_ALPHABET_2 3 //!< ISO Latin Alphabet No 2
-#define CHARSET_UCS2_BE 6 //!< ISO/IEC 10646 using UCS-2 transformation format, big endian byte order
-#define CHARSET_UTF8 15 //!< ISO Latin Alphabet No 2
-
-
-typedef std::vector<uint8_t> uint8_vector_t;
-static int verbose = 0;
-
-
-struct MSCDG {
-    // MSC Data Group Header (extension field not supported)
-    unsigned char extflag;      //  1 bit
-    unsigned char crcflag;      //  1 bit
-    unsigned char segflag;      //  1 bit
-    unsigned char accflag;      //  1 bit
-    unsigned char dgtype;       //  4 bits
-    unsigned char cindex;       //  4 bits
-    unsigned char rindex;       //  4 bits
-    /// Session header - Segment field
-    unsigned char last;         //  1 bit
-    unsigned short int segnum;  // 16 bits
-    // Session header - User access field
-    unsigned char rfa;          //  3 bits
-    unsigned char tidflag;      //  1 bit
-    unsigned char lenid;        //  4 bits - Fixed to value 2 in this implemntation
-    unsigned short int tid;     // 16 bits
-    // MSC data group data field
-    //  Mot Segmentation header
-    unsigned char rcount;       //  3 bits
-    unsigned short int seglen;  // 13 bits
-    // Mot segment
-    unsigned char* segdata;
-    // MSC data group CRC
-    unsigned short int crc;     // 16 bits
-};
-
-/*! Between collection of slides and transmission, the slide data is saved
- * in this structure.
- */
-struct slide_metadata_t {
-    // complete path to slide
-    std::string filepath;
-
-    // index, values from 0 to MAXSLIDEID, rolls over
-    int fidx;
-
-    // This is used to define the order in which several discovered
-    // slides are transmitted
-    bool operator<(const slide_metadata_t& other) const {
-        return this->fidx < other.fidx;
-    }
-};
-
-/*! A simple fingerprint for each slide transmitted.
- * Allows us to reuse the same fidx if the same slide
- * is transmitted more than once.
- */
-struct fingerprint_t {
-    // file name
-    std::string s_name;
-    // file size, in bytes
-    off_t s_size;
-    // time of last modification
-    unsigned long s_mtime;
-
-    // assigned fidx, -1 means invalid
-    int fidx;
-
-    /*! The comparison is not done on fidx, only
-     * on the file-specific data
-     */
-    bool operator==(const fingerprint_t& other) const {
-        return (((s_name == other.s_name &&
-                 s_size == other.s_size) &&
-                s_mtime == other.s_mtime));
-    }
-
-    void disp(void) {
-        printf("%s_%ld_%lu:%d\n", s_name.c_str(), s_size, s_mtime, fidx);
-    }
-
-    void load_from_file(const char* filepath)
-    {
-        struct stat file_attribue;
-        const char * final_slash;
-
-        stat(filepath, &file_attribue);
-        final_slash = strrchr(filepath, '/');
-
-        // load filename, size and mtime
-        // Save only the basename of the filepath
-        this->s_name.assign((final_slash == NULL) ? filepath : final_slash + 1);
-        this->s_size = file_attribue.st_size;
-        this->s_mtime = file_attribue.st_mtime;
-
-        this->fidx = -1;
-    }
-};
-
-/*! We keep track of transmitted files so that we can retransmit
- * identical slides with the same index, in case the receivers cache
- * them.
- *
- * \c MAXHISTORYLEN defines for how how many slides we want to keep this
- * history.
- */
-class History {
-    public:
-        History(size_t hist_size) :
-            m_hist_size(hist_size),
-            m_last_given_fidx(0) {}
-        void disp_database();
-        // controller of id base on database
-        int get_fidx(const char* filepath);
-
-    private:
-        std::deque<fingerprint_t> m_database;
-
-        size_t m_hist_size;
-
-        int m_last_given_fidx;
-
-        // find the fingerprint fp in database.
-        // returns the fidx when found,
-        //    or   -1 if not found
-        int find(const fingerprint_t& fp) const;
-
-        // add a new fingerprint into database
-        // returns its fidx
-        void add(fingerprint_t& fp);
-};
-
-
-int encodeFile(int output_fd, const std::string& fname, int fidx, bool raw_slides);
-
-uint8_vector_t createMotHeader(
-        size_t blobsize,
-        int fidx,
-        bool jfif_not_png);
-
-void createMscDG(MSCDG* msc, unsigned short int dgtype, int *cindex, unsigned short int segnum,
-        unsigned short int lastseg, unsigned short int tid, unsigned char* data,
-        unsigned short int datalen);
-
-struct DATA_GROUP;
-DATA_GROUP* packMscDG(MSCDG* msc);
-
-struct DL_STATE;
-void prepend_dl_dgs(const DL_STATE& dl_state, uint8_t charset);
-void writeDLS(int output_fd, const std::string& dls_file, uint8_t charset, bool raw_dls, bool remove_dls);
-
-// PAD related
-#define CRC_LEN 2
-
-struct DATA_GROUP {
-    uint8_vector_t data;
-    int apptype_start;
-    int apptype_cont;
-    size_t written;
-
-    DATA_GROUP(size_t len, int apptype_start, int apptype_cont) {
-        this->data.resize(len);
-        this->apptype_start = apptype_start;
-        this->apptype_cont = apptype_cont;
-        written = 0;
-    }
-
-    void AppendCRC() {
-        uint16_t crc = 0xFFFF;
-        for (size_t i = 0; i < data.size(); i++)
-            crc = update_crc_ccitt(crc, data[i]);
-        crc = ~crc;
-#if DEBUG
-        fprintf(stderr, "crc=%04x ~crc=%04x\n", crc, ~crc);
-#endif
-
-        data.push_back((crc & 0xFF00) >> 8);
-        data.push_back((crc & 0x00FF));
-    }
-
-    size_t Available() {
-        return data.size() - written;
-    }
-
-    int Write(uint8_t *write_data, size_t len, int *cont_apptype) {
-        size_t written_now = std::min(len, Available());
-
-        // fill up remaining bytes with zero padding
-        memcpy(write_data, &data[written], written_now);
-        memset(write_data + written_now, 0x00, len - written_now);
-
-        // set app type depending on progress
-        int apptype = written > 0 ? apptype_cont : apptype_start;
-
-        written += written_now;
-
-        // prevent continuation of a different DG having the same type
-        if (cont_apptype)
-            *cont_apptype = Available() > 0 ? apptype_cont : -1;
-
-        return apptype;
-    }
-};
-
-#define SHORT_PAD 6         // F-PAD + 1x CI              + 1x  3 bytes data sub-field
-#define VARSIZE_PAD_MIN 8   // F-PAD + 1x CI + end marker + 1x  4 bytes data sub-field
-#define VARSIZE_PAD_MAX 196 // F-PAD + 4x CI              + 4x 48 bytes data sub-field
-#define ALLOWED_PADLEN "6 (short X-PAD), 8 to 196 (variable size X-PAD)"
-
-
-
-// MOT Slideshow related
-static int cindex_header = 0;
-static int cindex_body = 0;
-
-
-class MOTHeader {
-private:
-    size_t header_size;
-    uint8_vector_t data;
-
-    void IncrementHeaderSize(size_t size);
-    void AddParamHeader(int pli, int param_id) {data.push_back((pli << 6) | (param_id & 0x3F));}
-public:
-    MOTHeader(size_t body_size, int content_type, int content_subtype);
-
-    void AddExtension(int param_id);
-    void AddExtension8Bit(int param_id, uint8_t data_field);
-    void AddExtension32Bit(int param_id, uint32_t data_field);
-    void AddExtensionVarSize(int param_id, const uint8_t* data_field, size_t data_field_len);
-    const uint8_vector_t GetData() {return data;}
-};
-
-MOTHeader::MOTHeader(size_t body_size, int content_type, int content_subtype)
-: header_size(0), data(uint8_vector_t(7, 0x00)) {
-    // init header core
-
-    // body size
-    data[0] = (body_size >> 20) & 0xFF;
-    data[1] = (body_size >> 12) & 0xFF;
-    data[2] = (body_size >>  4) & 0xFF;
-    data[3] = (body_size <<  4) & 0xF0;
-
-    // header size
-    IncrementHeaderSize(data.size());
-
-    // content type
-    data[5] |= (content_type << 1) & 0x7E;
-
-    // content subtype
-    data[5] |= (content_subtype >> 8) & 0x01;
-    data[6] |=  content_subtype       & 0xFF;
-}
-
-void MOTHeader::IncrementHeaderSize(size_t size) {
-    header_size += size;
-
-    data[3] &= 0xF0;
-    data[3] |= (header_size >> 9) & 0x0F;
-
-    data[4]  = (header_size >> 1) & 0xFF;
-
-    data[5] &= 0x7F;
-    data[5] |= (header_size << 7) & 0x80;
-}
-
-void MOTHeader::AddExtension(int param_id) {
-    AddParamHeader(0b00, param_id);
-
-    IncrementHeaderSize(1);
-}
-
-void MOTHeader::AddExtension8Bit(int param_id, uint8_t data_field) {
-    AddParamHeader(0b01, param_id);
-    data.push_back(data_field);
-
-    IncrementHeaderSize(2);
-}
-
-void MOTHeader::AddExtension32Bit(int param_id, uint32_t data_field) {
-    AddParamHeader(0b10, param_id);
-    data.push_back((data_field >> 24) & 0xFF);
-    data.push_back((data_field >> 16) & 0xFF);
-    data.push_back((data_field >>  8) & 0xFF);
-    data.push_back( data_field        & 0xFF);
-
-    IncrementHeaderSize(5);
-}
-
-void MOTHeader::AddExtensionVarSize(int param_id, const uint8_t* data_field, size_t data_field_len) {
-    AddParamHeader(0b11, param_id);
-
-    // longer field lens use 15 instead of 7 bits
-    bool ext = data_field_len > 127;
-    if (ext) {
-        data.push_back(0x80 | ((data_field_len >> 8) & 0x7F));
-        data.push_back(data_field_len & 0xFF);
-    } else {
-        data.push_back(data_field_len & 0x7F);
-    }
-
-    for (size_t i = 0; i < data_field_len; i++)
-        data.push_back(data_field[i]);
-
-    IncrementHeaderSize(1 + (ext ? 2 : 1) + data_field_len);
-}
-
-
-
-// DLS related
-#define FPAD_LEN 2
-#define DLS_SEG_LEN_PREFIX       2
-#define DLS_SEG_LEN_CHAR_MAX    16
-#define DLS_CMD_REMOVE_LABEL    0b0001
-#define DLS_CMD_DL_PLUS         0b0010
-#define DL_PLUS_CMD_TAGS        0b0000
-
-#define DL_PARAMS_OPEN          "##### parameters { #####"
-#define DL_PARAMS_CLOSE         "##### parameters } #####"
-
-CharsetConverter charset_converter;
-
-struct DL_PLUS_TAG {
-    int content_type;
-    int start_marker;
-    int length_marker;
-
-    DL_PLUS_TAG() :
-        content_type(0),    // = DUMMY
-        start_marker(0),
-        length_marker(0)
-    {}
-
-    DL_PLUS_TAG(int content_type, int start_marker, int length_marker) :
-        content_type(content_type),
-        start_marker(start_marker),
-        length_marker(length_marker)
-    {}
-
-    bool operator==(const DL_PLUS_TAG& other) const {
-        return
-            content_type == other.content_type &&
-            start_marker == other.start_marker &&
-            length_marker == other.length_marker;
-    }
-    bool operator!=(const DL_PLUS_TAG& other) const {
-        return !(*this == other);
-    }
-};
-
-typedef std::vector<DL_PLUS_TAG> dl_plus_tags_t;
-
-struct DL_STATE {
-    std::string dl_text;
-
-    bool dl_plus_enabled;
-    bool dl_plus_item_toggle;
-    bool dl_plus_item_running;
-    dl_plus_tags_t dl_plus_tags;
-
-    DL_STATE() :
-        dl_plus_enabled(false),
-        dl_plus_item_toggle(false),
-        dl_plus_item_running(false)
-    {}
-
-    bool operator==(const DL_STATE& other) const {
-        if (dl_text != other.dl_text)
-            return false;
-        if (dl_plus_enabled != other.dl_plus_enabled)
-            return false;
-        if (dl_plus_enabled) {
-            if (dl_plus_item_toggle != other.dl_plus_item_toggle)
-                return false;
-            if (dl_plus_item_running != other.dl_plus_item_running)
-                return false;
-            if (dl_plus_tags != other.dl_plus_tags)
-                return false;
-        }
-        return true;
-    }
-    bool operator!=(const DL_STATE& other) const {
-        return !(*this == other);
-    }
-};
-
-
-typedef uint8_vector_t pad_t;
-static bool dls_toggle = false;
-static DL_STATE dl_state_prev;
-
-
-class PADPacketizer {
-private:
-    static const size_t subfield_lens[];
-
-    const size_t xpad_size_max;
-    const bool short_xpad;
-    const size_t max_cis;
-
-    size_t xpad_size;
-    uint8_t subfields[4*48];
-    size_t subfields_size;
-
-    // PAD w/  CI list
-    int ci_type[4];
-    size_t ci_len_index[4];
-    size_t used_cis;
-
-    // PAD w/o CI list
-    int last_ci_type;
-    size_t last_ci_size;
-
-    size_t AddCINeededBytes();
-    void AddCI(int apptype, int len_index);
-
-    int OptimalSubFieldSizeIndex(size_t available_bytes);
-    int WriteDGToSubField(DATA_GROUP* dg, size_t len);
-
-    bool AppendDG(DATA_GROUP* dg);
-    void AppendDGWithCI(DATA_GROUP* dg);
-    void AppendDGWithoutCI(DATA_GROUP* dg);
-
-    void ResetPAD();
-    pad_t* FlushPAD();
-public:
-    std::deque<DATA_GROUP*> queue;
-
-    PADPacketizer(size_t pad_size);
-    ~PADPacketizer();
-
-    pad_t* GetPAD();
-
-    // will be removed, when pull (instead of push) approach is implemented!
-    void WriteAllPADs(int output_fd);
-};
-
-
-const size_t PADPacketizer::subfield_lens[] = {4, 6, 8, 12, 16, 24, 32, 48};
-
-PADPacketizer::PADPacketizer(size_t pad_size) :
-    xpad_size_max(pad_size - FPAD_LEN),
-    short_xpad(pad_size == SHORT_PAD),
-    max_cis(short_xpad ? 1 : 4),
-    last_ci_type(-1)
-{
-    ResetPAD();
-}
-
-PADPacketizer::~PADPacketizer() {
-    while (!queue.empty()) {
-        delete queue.front();
-        queue.pop_front();
-    }
-}
-
-
-pad_t* PADPacketizer::GetPAD() {
-    bool pad_flushable = false;
-
-    // process DG queue
-    while (!pad_flushable && !queue.empty()) {
-        DATA_GROUP* dg = queue.front();
-
-        // repeatedly append DG
-        while (!pad_flushable && dg->Available() > 0)
-            pad_flushable = AppendDG(dg);
-
-        if (dg->Available() == 0) {
-            delete dg;
-            queue.pop_front();
-        }
-    }
-
-    // (possibly empty) PAD
-    return FlushPAD();
-}
-
-void PADPacketizer::WriteAllPADs(int output_fd) {
-    for (;;) {
-        pad_t* pad = GetPAD();
-
-        // if only F-PAD present, abort
-        if (pad->back() == FPAD_LEN) {
-            delete pad;
-            break;
-        }
-
-        if (write(output_fd, &(*pad)[0], pad->size()) != (signed) pad->size())
-            fprintf(stderr, "mot-encoder Error: Could not write PAD\n");
-
-        delete pad;
-    }
-}
-
-
-size_t PADPacketizer::AddCINeededBytes() {
-    // returns the amount of additional bytes needed for the next CI
-
-    // special cases: end marker added/replaced
-    if (!short_xpad && used_cis == 0)
-        return 2;
-    if (!short_xpad && used_cis == (max_cis - 1))
-        return 0;
-    return 1;
-}
-
-void PADPacketizer::AddCI(int apptype, int len_index) {
-    ci_type[used_cis] = apptype;
-    ci_len_index[used_cis] = len_index;
-
-    xpad_size += AddCINeededBytes();
-    used_cis++;
-}
-
-
-int PADPacketizer::OptimalSubFieldSizeIndex(size_t available_bytes) {
-    /*! Return the index of the optimal sub-field size by stepwise search (regards only Variable Size X-PAD):
-     * - find the smallest sub-field able to hold (at least) all available bytes
-     * - find the biggest regarding sub-field we have space for (which definitely exists - otherwise previously the PAD would have been flushed)
-     * - if the wasted space is at least as big as the smallest possible sub-field, use a sub-field one size smaller
-     */
-    int len_index = 0;
-
-    while ((len_index + 1) < 8 && subfield_lens[len_index] < available_bytes)
-        len_index++;
-    while ((len_index - 1) >= 0 && (subfield_lens[len_index] + AddCINeededBytes()) > (xpad_size_max - xpad_size))
-        len_index--;
-    if ((len_index - 1) >= 0 && ((int) subfield_lens[len_index] - (int) available_bytes) >= (int) subfield_lens[0])
-        len_index--;
-
-    return len_index;
-}
-
-int PADPacketizer::WriteDGToSubField(DATA_GROUP* dg, size_t len) {
-    int apptype = dg->Write(&subfields[subfields_size], len, &last_ci_type);
-    subfields_size += len;
-    xpad_size += len;
-    return apptype;
-}
-
-
-bool PADPacketizer::AppendDG(DATA_GROUP* dg) {
-    /*! use X-PAD w/o CIs instead of X-PAD w/ CIs, if we can save some bytes or at least do not waste additional bytes
-     *
-     * Omit CI list in case:
-     * 1.   no pending data sub-fields
-     * 2.   last CI type valid
-     * 3.   last CI type matching current (continuity) CI type
-     * 4a.  short X-PAD; OR
-     * 4ba. size of the last X-PAD being at least as big as the available X-PAD payload in case all CIs are used AND
-     * 4bb. the amount of available DG bytes being at least as big as the size of the last X-PAD in case all CIs are used
-     */
-    if (
-            used_cis == 0 &&
-            last_ci_type != -1 &&
-            last_ci_type == dg->apptype_cont &&
-            (short_xpad ||
-                    (last_ci_size >= (xpad_size_max - max_cis) &&
-                            dg->Available() >= (last_ci_size - max_cis)))
-            ) {
-        AppendDGWithoutCI(dg);
-        return true;
-    } else {
-        AppendDGWithCI(dg);
-
-        // if no further sub-fields could be added, PAD must be flushed
-        if (used_cis == max_cis || subfield_lens[0] + AddCINeededBytes() > (xpad_size_max - xpad_size))
-            return true;
-    }
-    return false;
-}
-
-
-void PADPacketizer::AppendDGWithCI(DATA_GROUP* dg) {
-    int len_index = short_xpad ? 0 : OptimalSubFieldSizeIndex(dg->Available());
-    size_t len_size = short_xpad ? 3 : subfield_lens[len_index];
-
-    int apptype = WriteDGToSubField(dg, len_size);
-    AddCI(apptype, len_index);
-
-#if DEBUG
-    fprintf(stderr, "PADPacketizer: added sub-field w/  CI - type: %2d, size: %2zu\n", apptype, len_size);
-#endif
-}
-
-void PADPacketizer::AppendDGWithoutCI(DATA_GROUP* dg) {
-#if DEBUG
-    int old_last_ci_type = last_ci_type;
-#endif
-
-    WriteDGToSubField(dg, last_ci_size);
-
-#if DEBUG
-    fprintf(stderr, "PADPacketizer: added sub-field w/o CI - type: %2d, size: %2zu\n", old_last_ci_type, last_ci_size);
-#endif
-}
-
-void PADPacketizer::ResetPAD() {
-    xpad_size = 0;
-    subfields_size = 0;
-    used_cis = 0;
-}
-
-pad_t* PADPacketizer::FlushPAD() {
-    pad_t* result = new pad_t(xpad_size_max + FPAD_LEN + 1);
-    pad_t &pad = *result;
-
-    size_t pad_offset = xpad_size_max;
-
-    if (subfields_size > 0) {
-        if (used_cis > 0) {
-            // X-PAD: CIs
-            for (size_t i = 0; i < used_cis; i++)
-                pad[--pad_offset] = (short_xpad ? 0 : ci_len_index[i]) << 5 | ci_type[i];
-
-            // X-PAD: end marker (if needed)
-            if (used_cis < max_cis)
-                pad[--pad_offset] = 0x00;
-        }
-
-        // X-PAD: sub-fields (reversed on-the-fly)
-        for (size_t off = 0; off < subfields_size; off++)
-            pad[--pad_offset] = subfields[off];
-    } else {
-        // no X-PAD
-        last_ci_type = -1;
-    }
-
-    // zero padding
-    memset(&pad[0], 0x00, pad_offset);
-
-    // F-PAD
-    pad[xpad_size_max + 0] = subfields_size > 0 ? (short_xpad ? 0x10 : 0x20) : 0x00;
-    pad[xpad_size_max + 1] = subfields_size > 0 ? (used_cis > 0 ? 0x02 : 0x00) : 0x00;
-
-    // used PAD len
-    pad[xpad_size_max + FPAD_LEN] = xpad_size + FPAD_LEN;
-
-    last_ci_size = xpad_size;
-    ResetPAD();
-    return result;
-}
-
-
-static PADPacketizer *pad_packetizer;
-
-
-
-
-
-
-
-void usage(char* name)
-{
-    fprintf(stderr, "DAB MOT encoder %s for slideshow and DLS\n\n"
-                    "By CSP Innovazione nelle ICT s.c.a r.l. (http://rd.csp.it/) and\n"
-                    "Opendigitalradio.org\n\n"
-                    "Reads image data from the specified directory, DLS text from a file,\n"
-                    "and outputs PAD data to the given FIFO.\n"
-                    "  http://opendigitalradio.org\n\n",
-#if defined(GITVERSION)
-                    GITVERSION
-#else
-                    PACKAGE_VERSION
-#endif
-                    );
-    fprintf(stderr, "Usage: %s [OPTIONS...]\n", name);
-    fprintf(stderr, " -d, --dir=DIRNAME      Directory to read images from.\n"
-                    " -e, --erase            Erase slides from DIRNAME once they have\n"
-                    "                          been encoded.\n"
-                    " -s, --sleep=DELAY      Wait DELAY seconds between each slide\n"
-                    "                          Default: " STR(SLEEPDELAY_DEFAULT) "\n"
-                    " -o, --output=FILENAME  Fifo to write PAD data into.\n"
-                    "                          Default: /tmp/pad.fifo\n"
-                    " -t, --dls=FILENAME     Fifo or file to read DLS text from.\n"
-                    " -p, --pad=LENGTH       Set the pad length.\n"
-                    "                          Possible values: " ALLOWED_PADLEN "\n"
-                    "                          Default: 58\n"
-                    " -c, --charset=ID       ID of the character set encoding used for DLS text input.\n"
-                    "                          ID =  0: Complete EBU Latin based repertoire\n"
-                    "                          ID =  6: ISO/IEC 10646 using UCS-2 BE\n"
-                    "                          ID = 15: ISO/IEC 10646 using UTF-8\n"
-                    "                          Default: 15\n"
-                    " -r, --remove-dls       Always insert a DLS Remove Label command when replacing a DLS text.\n"
-                    " -C, --raw-dls          Do not convert DLS texts to Complete EBU Latin based repertoire\n"
-                    "                          character set encoding.\n"
-                    " -R, --raw-slides       Do not process slides. Integrity checks and resizing\n"
-                    "                          slides is skipped. Use this if you know what you are doing !\n"
-                    "                          It is useful only when -d is used\n"
-                    " -v, --verbose          Print more information to the console\n"
-           );
-}
-
-#define no_argument 0
-#define required_argument 1
-#define optional_argument 2
-int main(int argc, char *argv[])
-{
-    int ret;
-    struct dirent *pDirent;
-    int  padlen = 58;
-    bool erase_after_tx = false;
-    int  sleepdelay = SLEEPDELAY_DEFAULT;
-    bool raw_slides = false;
-    int  charset = CHARSET_UTF8;
-    bool raw_dls = false;
-    bool remove_dls = false;
-
-    const char* sls_dir = NULL;
-    const char* output = "/tmp/pad.fifo";
-    std::string dls_file;
-
-    const struct option longopts[] = {
-        {"charset",    required_argument,  0, 'c'},
-        {"raw-dls",    no_argument,        0, 'C'},
-        {"remove-dls", no_argument,        0, 'r'},
-        {"dir",        required_argument,  0, 'd'},
-        {"erase",      no_argument,        0, 'e'},
-        {"output",     required_argument,  0, 'o'},
-        {"dls",        required_argument,  0, 't'},
-        {"pad",        required_argument,  0, 'p'},
-        {"sleep",      required_argument,  0, 's'},
-        {"raw-slides", no_argument,        0, 'R'},
-        {"help",       no_argument,        0, 'h'},
-        {"verbose",    no_argument,        0, 'v'},
-        {0,0,0,0},
-    };
-
-    int ch=0;
-    int index;
-    while(ch != -1) {
-        ch = getopt_long(argc, argv, "eChRrc:d:o:p:s:t:v", longopts, &index);
-        switch (ch) {
-            case 'c':
-                charset = atoi(optarg);
-                break;
-            case 'C':
-                raw_dls = true;
-                break;
-            case 'r':
-                remove_dls = true;
-                break;
-            case 'd':
-                sls_dir = optarg;
-                break;
-            case 'e':
-                erase_after_tx = true;
-                break;
-            case 'o':
-                output = optarg;
-                break;
-            case 's':
-                sleepdelay = atoi(optarg);
-                break;
-            case 't':
-                dls_file = optarg;
-                break;
-            case 'p':
-                padlen = atoi(optarg);
-                break;
-            case 'R':
-                raw_slides = true;
-                break;
-            case 'v':
-                verbose++;
-                break;
-            case '?':
-            case 'h':
-                usage(argv[0]);
-                return 0;
-        }
-    }
-
-    if (padlen != SHORT_PAD && (padlen < VARSIZE_PAD_MIN || padlen > VARSIZE_PAD_MAX)) {
-        fprintf(stderr, "mot-encoder Error: pad length %d invalid: Possible values: "
-                ALLOWED_PADLEN "\n",
-                padlen);
-        return 2;
-    }
-
-    if (sls_dir && not dls_file.empty()) {
-        fprintf(stderr, "mot-encoder encoding Slideshow from '%s' and DLS from '%s' to '%s'\n",
-                sls_dir, dls_file.c_str(), output);
-    }
-    else if (sls_dir) {
-        fprintf(stderr, "mot-encoder encoding Slideshow from '%s' to '%s'. No DLS.\n",
-                sls_dir, output);
-    }
-    else if (not dls_file.empty()) {
-        fprintf(stderr, "mot-encoder encoding DLS from '%s' to '%s'. No Slideshow.\n",
-                dls_file.c_str(), output);
-    }
-    else {
-        fprintf(stderr, "mot-encoder Error: No DLS nor slideshow to encode !\n");
-        usage(argv[0]);
-        return 1;
-    }
-
-    const char* user_charset;
-    switch (charset) {
-        case CHARSET_COMPLETE_EBU_LATIN:
-            user_charset = "Complete EBU Latin";
-            break;
-        case CHARSET_EBU_LATIN_CY_GR:
-            user_charset = "EBU Latin core, Cyrillic, Greek";
-            break;
-        case CHARSET_EBU_LATIN_AR_HE_CY_GR:
-            user_charset = "EBU Latin core, Arabic, Hebrew, Cyrillic, Greek";
-            break;
-        case CHARSET_ISO_LATIN_ALPHABET_2:
-            user_charset = "ISO Latin Alphabet 2";
-            break;
-        case CHARSET_UCS2_BE:
-            user_charset = "UCS-2 BE";
-            break;
-        case CHARSET_UTF8:
-            user_charset = "UTF-8";
-            break;
-        default:
-            user_charset = "Invalid";
-            charset = -1;
-            break;
-    }
-
-    if (charset == -1) {
-        fprintf(stderr, "mot-encoder Error: Invalid charset!\n");
-        usage(argv[0]);
-        return 1;
-    }
-    else {
-        fprintf(stderr, "mot-encoder using charset %s (%d)\n",
-               user_charset, charset);
-    }
-
-    if (not raw_dls) {
-        switch (charset) {
-        case CHARSET_COMPLETE_EBU_LATIN:
-            // no conversion needed
-            break;
-        case CHARSET_UTF8:
-            fprintf(stderr, "mot-encoder converting DLS texts to Complete EBU Latin\n");
-            break;
-        default:
-            fprintf(stderr, "mot-encoder Error: DLS conversion to EBU is currently only supported for UTF-8 input!\n");
-            return 1;
-        }
-    }
-
-    int output_fd = open(output, O_WRONLY);
-    if (output_fd == -1) {
-        perror("mot-encoder Error: failed to open output");
-        return 3;
-    }
-
-#if HAVE_MAGICKWAND
-    MagickWandGenesis();
-#endif
-
-    pad_packetizer = new PADPacketizer(padlen);
-
-    std::list<slide_metadata_t> slides_to_transmit;
-    History slides_history(MAXHISTORYLEN);
-
-    while(1) {
-        if (sls_dir) { // slide + possibly DLS
-            DIR *pDir = opendir(sls_dir);
-            if (pDir == NULL) {
-                fprintf(stderr, "mot-encoder Error: cannot open directory '%s'\n", sls_dir);
-                return 1;
-            }
-
-            // Add new slides to transmit to list
-            while ((pDirent = readdir(pDir)) != NULL) {
-                if (pDirent->d_name[0] != '.') {
-                    char imagepath[256];
-                    sprintf(imagepath, "%s/%s", sls_dir, pDirent->d_name);
-
-                    slide_metadata_t md;
-                    md.filepath = imagepath;
-                    md.fidx     = slides_history.get_fidx(imagepath);
-                    slides_to_transmit.push_back(md);
-
-                    if (verbose) {
-                        fprintf(stderr, "mot-encoder found slide '%s', fidx %d\n", imagepath, md.fidx);
-                    }
-                }
-            }
-
-            closedir(pDir);
-
-#if DEBUG
-            slides_history.disp_database();
-#endif
-
-            // if ATM no slides, transmit at least DLS
-            if (slides_to_transmit.empty()) {
-                if (not dls_file.empty())
-                    writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
-                sleep(sleepdelay);
-            } else {
-                // Sort the list in fidx order
-                slides_to_transmit.sort();
-
-                // Encode the slides
-                for (std::list<slide_metadata_t>::const_iterator it = slides_to_transmit.cbegin();
-                        it != slides_to_transmit.cend();
-                        ++it) {
-
-                    ret = encodeFile(output_fd, it->filepath, it->fidx, raw_slides);
-                    if (ret != 1)
-                        fprintf(stderr, "mot-encoder Error: cannot encode file '%s'\n", it->filepath.c_str());
-
-                    if (erase_after_tx) {
-                        if (unlink(it->filepath.c_str()) == -1) {
-                            fprintf(stderr, "mot-encoder Error: erasing file '%s' failed: ", it->filepath.c_str());
-                            perror("");
-                        }
-                    }
-
-                    // Always retransmit DLS after each slide, we want it to be updated frequently
-                    if (not dls_file.empty())
-                        writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
-                    sleep(sleepdelay);
-                }
-
-                slides_to_transmit.resize(0);
-            }
-        } else { // only DLS
-            // Always retransmit DLS, we want it to be updated frequently
-            writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
-            sleep(sleepdelay);
-        }
-    }
-
-    delete pad_packetizer;
-
-    return 1;
-}
-
-
-DATA_GROUP* createDataGroupLengthIndicator(size_t len) {
-    DATA_GROUP* dg = new DATA_GROUP(2, 1, 1);    // continuation never used (except for comparison at short X-PAD)
-    uint8_vector_t &data = dg->data;
-
-    // Data Group length
-    data[0] = (len & 0x3F00) >> 8;
-    data[1] = (len & 0x00FF);
-
-    // CRC
-    dg->AppendCRC();
-
-    return dg;
-}
-
-
-void warnOnSmallerImage(size_t height, size_t width, const std::string& fname) {
-    if (height < 240 || width < 320)
-        fprintf(stderr, "mot-encoder Warning: Image '%s' smaller than recommended size (%zu x %zu < 320 x 240 px)\n", fname.c_str(), width, height);
-}
-
-
-/*! Scales the image down if needed,
- * so that it is 320x240 pixels.
- * Automatically reduces the quality to make sure the
- * blobsize is not too large.
- *
- * \return the blobsize
- */
-#if HAVE_MAGICKWAND
-size_t resizeImage(MagickWand* m_wand, unsigned char** blob, const std::string& fname, bool* jfif_not_png)
-{
-    unsigned char* blob_png;
-    unsigned char* blob_jpg;
-    size_t blobsize_png;
-    size_t blobsize_jpg;
-
-    size_t height = MagickGetImageHeight(m_wand);
-    size_t width  = MagickGetImageWidth(m_wand);
-
-    while (height > 240 || width > 320) {
-        if (height/240.0 > width/320.0) {
-            width = width * 240.0 / height;
-            height = 240;
-        }
-        else {
-            height = height * 320.0 / width;
-            width = 320;
-        }
-        MagickResizeImage(m_wand, width, height, LanczosFilter, 1);
-    }
-
-    height = MagickGetImageHeight(m_wand);
-    width  = MagickGetImageWidth(m_wand);
-
-    // try PNG (zlib level 9 / possibly adaptive filtering)
-    MagickSetImageFormat(m_wand, "png");
-    MagickSetImageCompressionQuality(m_wand, 95);
-    blob_png = MagickGetImageBlob(m_wand, &blobsize_png);
-
-    // try JPG
-    MagickSetImageFormat(m_wand, "jpg");
-
-    blob_jpg = NULL;
-    int quality_jpg = 100;
-    do {
-        free(blob_jpg);
-        quality_jpg -= 5;
-
-        MagickSetImageCompressionQuality(m_wand, quality_jpg);
-        blob_jpg = MagickGetImageBlob(m_wand, &blobsize_jpg);
-    } while (blobsize_jpg > MAXSLIDESIZE && quality_jpg > MINQUALITY);
-
-
-    // check for max size
-    if (blobsize_png > MAXSLIDESIZE && blobsize_jpg > MAXSLIDESIZE) {
-        fprintf(stderr, "mot-encoder: Image Size too large after compression: %zu bytes (PNG), %zu bytes (JPEG)\n",
-                blobsize_png, blobsize_jpg);
-        free(blob_png);
-        free(blob_jpg);
-        return 0;
-    }
-
-    // choose the smaller one (at least one does not exceed the max size)
-    *jfif_not_png = blobsize_jpg < blobsize_png;
-
-    if (verbose) {
-        if (*jfif_not_png)
-            fprintf(stderr, "mot-encoder resized image to %zu x %zu. Size after compression %zu bytes (JPEG, q=%d; PNG was %zu bytes)\n",
-                    width, height, blobsize_jpg, quality_jpg, blobsize_png);
-        else
-            fprintf(stderr, "mot-encoder resized image to %zu x %zu. Size after compression %zu bytes (PNG; JPEG was %zu bytes)\n",
-                    width, height, blobsize_png, blobsize_jpg);
-    }
-
-    // warn if resized image smaller than default dimension
-    warnOnSmallerImage(height, width, fname);
-
-    free(*jfif_not_png ? blob_png : blob_jpg);
-    *blob = *jfif_not_png ? blob_jpg : blob_png;
-    return *jfif_not_png ? blobsize_jpg : blobsize_png;
-}
-#endif
-
-int encodeFile(int output_fd, const std::string& fname, int fidx, bool raw_slides)
-{
-    int ret = 0;
-    int nseg, lastseglen, i, last, curseglen;
-#if HAVE_MAGICKWAND
-    MagickWand *m_wand = NULL;
-    MagickBooleanType err;
-#endif
-    size_t blobsize, height, width;
-    bool jpeg_progr;
-    unsigned char *blob = NULL;
-    unsigned char *curseg = NULL;
-    MSCDG msc;
-    DATA_GROUP* dgli;
-    DATA_GROUP* mscdg;
-
-    size_t orig_quality;
-    char*  orig_format = NULL;
-    /*! We handle JPEG differently, because we want to avoid recompressing the
-     * image if it is suitable as is
-     */
-    bool orig_is_jpeg = false;
-
-    /*! If the original is a PNG, we transmit it as is, if the resolution is correct
-     * and the file is not too large. Otherwise it gets resized and sent as JPEG.
-     */
-    bool orig_is_png = false;
-
-    /*! By default, we do resize the image to 320x240, with a quality such that
-     * the blobsize is at most MAXSLIDESIZE.
-     *
-     * For JPEG input files that are already at the right resolution and at the
-     * right blobsize, we disable this to avoid quality loss due to recompression
-     *
-     * As device support of this feature is optional, we furthermore require JPEG input
-     * files to not have progressive coding.
-     */
-    bool resize_required = true;
-
-    bool jfif_not_png = true;
-
-    if (!raw_slides) {
-#if HAVE_MAGICKWAND
-
-        m_wand = NewMagickWand();
-
-        err = MagickReadImage(m_wand, fname.c_str());
-        if (err == MagickFalse) {
-            fprintf(stderr, "mot-encoder Error: Unable to load image '%s'\n",
-                    fname.c_str());
-
-            goto encodefile_out;
-        }
-
-        height       = MagickGetImageHeight(m_wand);
-        width        = MagickGetImageWidth(m_wand);
-        orig_format  = MagickGetImageFormat(m_wand);
-        jpeg_progr   = MagickGetImageInterlaceScheme(m_wand) == JPEGInterlace;
-
-        // By default assume that the image has full quality and can be reduced
-        orig_quality = 100;
-
-        // strip unneeded information (profiles, meta data)
-        MagickStripImage(m_wand);
-
-        if (orig_format) {
-            if (strcmp(orig_format, "JPEG") == 0) {
-                orig_quality = MagickGetImageCompressionQuality(m_wand);
-                orig_is_jpeg = true;
-
-                if (verbose) {
-                    fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
-                            " Original size: %zu x %zu. (%s, q=%zu, progr=%s)\n",
-                            fname.c_str(), fidx, width, height, orig_format, orig_quality, jpeg_progr ? "y" : "n");
-                }
-            }
-            else if (strcmp(orig_format, "PNG") == 0) {
-                orig_is_png = true;
-                jfif_not_png = false;
-
-                if (verbose) {
-                    fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
-                            " Original size: %zu x %zu. (%s)\n",
-                            fname.c_str(), fidx, width, height, orig_format);
-                }
-            }
-            else if (verbose) {
-                fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
-                        " Original size: %zu x %zu. (%s)\n",
-                        fname.c_str(), fidx, width, height, orig_format);
-            }
-
-            free(orig_format);
-        }
-        else {
-            fprintf(stderr, "mot-encoder Warning: Unable to detect image format of '%s'\n",
-                    fname.c_str());
-
-            fprintf(stderr, "mot-encoder image: '%s' (id=%d).  Original size: %zu x %zu.\n",
-                    fname.c_str(), fidx, width, height);
-        }
-
-        if ((orig_is_jpeg || orig_is_png) && height <= 240 && width <= 320 && not jpeg_progr) {
-            // Don't recompress the image and check if the blobsize is suitable
-            blob = MagickGetImageBlob(m_wand, &blobsize);
-
-            if (blobsize <= MAXSLIDESIZE) {
-                if (verbose) {
-                    fprintf(stderr, "mot-encoder image: '%s' (id=%d).  No resize needed: %zu Bytes\n",
-                            fname.c_str(), fidx, blobsize);
-                }
-                resize_required = false;
-            }
-        }
-
-        if (resize_required) {
-            blobsize = resizeImage(m_wand, &blob, fname, &jfif_not_png);
-        }
-        else {
-            // warn if unresized image smaller than default dimension
-            warnOnSmallerImage(height, width, fname);
-        }
-
-#else
-        fprintf(stderr, "mot-encoder has not been compiled with MagickWand, only RAW slides are supported!\n");
-        ret = -1;
-        goto encodefile_out;
-#endif
-    }
-    else { // Use RAW data, it might not even be a jpg !
-        // read file
-        FILE* pFile = fopen(fname.c_str(), "rb");
-        if (pFile == NULL) {
-            fprintf(stderr, "mot-encoder Error: Unable to load file '%s'\n",
-                    fname.c_str());
-            goto encodefile_out;
-        }
-
-        // obtain file size:
-        fseek(pFile, 0, SEEK_END);
-        blobsize = ftell(pFile);
-        rewind(pFile);
-
-        if (blobsize > MAXSLIDESIZE) {
-            fprintf(stderr, "mot-encoder Warning: blob in raw-slide '%s' too large\n",
-                    fname.c_str());
-        }
-
-        // allocate memory to contain the whole file:
-        blob = (unsigned char*)malloc(sizeof(char) * blobsize);
-        if (blob == NULL) {
-            fprintf(stderr, "mot-encoder Error: Memory allocation error\n");
-            goto encodefile_out;
-        }
-
-        // copy the file into the buffer:
-        if (fread(blob, blobsize, 1, pFile) != 1) {
-            fprintf(stderr, "mot-encoder Error: Could not read file\n");
-            goto encodefile_out;
-        }
-
-        size_t last_dot = fname.rfind(".");
-
-        // default:
-        jfif_not_png = true; // This is how we did it in the past.
-                             // It's wrong anyway, so we're at least compatible
-
-        if (last_dot != std::string::npos) {
-            std::string file_extension = fname.substr(last_dot, std::string::npos);
-
-            std::transform(file_extension.begin(), file_extension.end(), file_extension.begin(), ::tolower);
-
-            if (file_extension == ".png") {
-                jfif_not_png = false;
-            }
-        }
-
-        if (pFile != NULL) {
-            fclose(pFile);
-        }
-    }
-
-    if (blobsize) {
-        nseg = blobsize / MAXSEGLEN;
-        lastseglen = blobsize % MAXSEGLEN;
-        if (lastseglen != 0) {
-            nseg++;
-        }
-
-        uint8_vector_t mothdr = createMotHeader(blobsize, fidx, jfif_not_png);
-        // Create the MSC Data Group C-Structure
-        createMscDG(&msc, 3, &cindex_header, 0, 1, fidx, &mothdr[0], mothdr.size());
-        // Generate the MSC DG frame (Figure 9 en 300 401)
-        mscdg = packMscDG(&msc);
-        dgli = createDataGroupLengthIndicator(mscdg->data.size());
-
-        pad_packetizer->queue.push_back(dgli);
-        pad_packetizer->queue.push_back(mscdg);
-
-        for (i = 0; i < nseg; i++) {
-            curseg = blob + i * MAXSEGLEN;
-            if (i == nseg-1) {
-                curseglen = lastseglen;
-                last = 1;
-            }
-            else {
-                curseglen = MAXSEGLEN;
-                last = 0;
-            }
-
-            createMscDG(&msc, 4, &cindex_body, i, last, fidx, curseg, curseglen);
-            mscdg = packMscDG(&msc);
-            dgli = createDataGroupLengthIndicator(mscdg->data.size());
-
-            pad_packetizer->queue.push_back(dgli);
-            pad_packetizer->queue.push_back(mscdg);
-        }
-
-        pad_packetizer->WriteAllPADs(output_fd);
-
-        ret = 1;
-    }
-
-encodefile_out:
-#if HAVE_MAGICKWAND
-    if (m_wand) {
-        m_wand = DestroyMagickWand(m_wand);
-    }
-#endif
-
-    if (blob) {
-        free(blob);
-    }
-    return ret;
-}
-
-
-uint8_vector_t createMotHeader(size_t blobsize, int fidx, bool jfif_not_png)
-{
-    // prepare ContentName
-    uint8_t cntemp[10];     // = 1 + 8 + 1 = charset + name + terminator
-    cntemp[0] = 0x0 << 4;   // charset: 0 (Complete EBU Latin based) - doesn't really matter here
-    snprintf((char*) (cntemp + 1), sizeof(cntemp) - 1, "%04d.%s", fidx, jfif_not_png ? "jpg" : "png");
-    if (verbose)
-        fprintf(stderr, "mot-encoder writing image as '%s'\n", cntemp + 1);
-
-    // MOT header - content type: image, content subtype: JFIF / PNG
-    MOTHeader header(blobsize, 0x02, jfif_not_png ? 0x001 : 0x003);
-
-    // TriggerTime: NOW
-    header.AddExtension32Bit(0x05, 0x00000000);
-
-    // ContentName: XXXX.jpg / XXXX.png
-    header.AddExtensionVarSize(0x0C, cntemp, sizeof(cntemp) - 1);   // omit terminator
-
-    return header.GetData();
-}
-
-
-void createMscDG(MSCDG* msc, unsigned short int dgtype,
-        int *cindex, unsigned short int segnum, unsigned short int lastseg,
-        unsigned short int tid, unsigned char* data,
-        unsigned short int datalen)
-{
-    msc->extflag = 0;
-    msc->crcflag = 1;
-    msc->segflag = 1;
-    msc->accflag = 1;
-    msc->dgtype = dgtype;
-    msc->cindex = *cindex;
-    msc->rindex = 0;
-    msc->last = lastseg;
-    msc->segnum = segnum;
-    msc->rfa = 0;
-    msc->tidflag = 1;
-    msc->lenid = 2;
-    msc->tid = tid;
-    msc->segdata = data;
-    msc->rcount = 0;
-    msc->seglen = datalen;
-
-    *cindex = (*cindex + 1) % 16;   // increment continuity index
-}
-
-
-DATA_GROUP* packMscDG(MSCDG* msc)
-{
-    DATA_GROUP* dg = new DATA_GROUP(9 + msc->seglen, 12, 13);
-    uint8_vector_t &b = dg->data;
-
-    // headers
-    b[0] = (msc->extflag<<7) | (msc->crcflag<<6) | (msc->segflag<<5) |
-           (msc->accflag<<4) | msc->dgtype;
-
-    b[1] = (msc->cindex<<4) | msc->rindex;
-    b[2] = (msc->last<<7) | ((msc->segnum & 0x7F00) >> 8);
-    b[3] =  msc->segnum & 0x00FF;
-    b[4] = 0;
-    b[4] = (msc->rfa << 5) | (msc->tidflag << 4) | msc->lenid;
-    b[5] = (msc->tid & 0xFF00) >> 8;
-    b[6] =  msc->tid & 0x00FF;
-    b[7] = (msc->rcount << 5) | ((msc->seglen & 0x1F00)>>8);
-    b[8] =  msc->seglen & 0x00FF;
-
-    // data field
-    memcpy(&b[9], msc->segdata, msc->seglen);
-
-    // CRC
-    dg->AppendCRC();
-
-    return dg;
-}
-
-
-DATA_GROUP* createDynamicLabelCommand(uint8_t command) {
-    DATA_GROUP* dg = new DATA_GROUP(2, 2, 3);
-    uint8_vector_t &seg_data = dg->data;
-
-    // prefix: toggle? + first seg + last seg + command flag + command
-    seg_data[0] =
-            (dls_toggle ? (1 << 7) : 0) +
-            (1 << 6) +
-            (1 << 5) +
-            (1 << 4) +
-            command;
-
-    // prefix: charset (though irrelevant here)
-    seg_data[1] = CHARSET_COMPLETE_EBU_LATIN;
-
-    // CRC
-    dg->AppendCRC();
-
-    return dg;
-}
-
-DATA_GROUP* createDynamicLabelPlus(const DL_STATE& dl_state) {
-    size_t tags_size = dl_state.dl_plus_tags.size();
-    size_t len_dl_plus_cmd_field = 1 + 3 * tags_size;
-    DATA_GROUP* dg = new DATA_GROUP(2 + len_dl_plus_cmd_field, 2, 3);
-    uint8_vector_t &seg_data = dg->data;
-
-    // prefix: toggle? + first seg + last seg + command flag + command
-    seg_data[0] =
-            (dls_toggle ? (1 << 7) : 0) +
-            (1 << 6) +
-            (1 << 5) +
-            (1 << 4) +
-            DLS_CMD_DL_PLUS;
-
-    // prefix: link bit + length
-    seg_data[1] =
-            (dls_toggle ? (1 << 7) : 0) +
-            (len_dl_plus_cmd_field - 1);    // -1 !
-
-    // DL Plus tags command: CId + IT + IR + NT
-    seg_data[2] =
-            (DL_PLUS_CMD_TAGS << 4) +
-            (dl_state.dl_plus_item_toggle ? (1 << 3) : 0) +
-            (dl_state.dl_plus_item_running ? (1 << 2) : 0) +
-            (tags_size - 1);                // -1 !
-
-    for (size_t i = 0; i < tags_size; i++) {
-        // DL Plus tags command: Content Type + Start Marker + Length Marker
-        seg_data[3 + 3 * i] = dl_state.dl_plus_tags[i].content_type & 0x7F;
-        seg_data[4 + 3 * i] = dl_state.dl_plus_tags[i].start_marker & 0x7F;
-        seg_data[5 + 3 * i] = dl_state.dl_plus_tags[i].length_marker & 0x7F;
-    }
-
-    // CRC
-    dg->AppendCRC();
-
-    return dg;
-}
-
-
-std::vector<std::string> split_string(const std::string &s, const char delimiter) {
-    std::vector<std::string> result;
-    std::stringstream ss(s);
-    std::string part;
-
-    while (std::getline(ss, part, delimiter))
-        result.push_back(part);
-    return result;
-}
-
-bool parse_dl_param_bool(const std::string &key, const std::string &value, bool &target) {
-    if (value == "0") {
-        target = 0;
-        return true;
-    }
-    if (value == "1") {
-        target = 1;
-        return true;
-    }
-    fprintf(stderr, "mot-encoder Warning: DL parameter '%s' has unsupported value '%s' - ignored\n", key.c_str(), value.c_str());
-    return false;
-}
-
-bool parse_dl_param_int_dl_plus_tag(const std::string &key, const std::string &value, int &target) {
-    int value_int = atoi(value.c_str());
-    if (value_int >= 0x00 && value_int <= 0x7F) {
-        target = value_int;
-        return true;
-    }
-    fprintf(stderr, "mot-encoder Warning: DL Plus tag parameter '%s' %d out of range - ignored\n", key.c_str(), value_int);
-    return false;
-}
-
-void parse_dl_params(std::ifstream &dls_fstream, DL_STATE &dl_state) {
-    std::string line;
-    while (std::getline(dls_fstream, line)) {
-        // return on params close
-        if (line == DL_PARAMS_CLOSE)
-            return;
-
-        // ignore empty lines and comments
-        if (line.empty() || line[0] == '#')
-            continue;
-
-        // parse key/value pair
-        size_t separator_pos = line.find('=');
-        if (separator_pos == std::string::npos) {
-            fprintf(stderr, "mot-encoder Warning: DL parameter line '%s' without separator - ignored\n", line.c_str());
-            continue;
-        }
-        std::string key = line.substr(0, separator_pos);
-        std::string value = line.substr(separator_pos + 1);
-#if DEBUG
-        fprintf(stderr, "parse_dl_params: key: '%s', value: '%s'\n", key.c_str(), value.c_str());
-#endif
-
-        if (key == "DL_PLUS") {
-            parse_dl_param_bool(key, value, dl_state.dl_plus_enabled);
-            continue;
-        }
-        if (key == "DL_PLUS_ITEM_TOGGLE") {
-            parse_dl_param_bool(key, value, dl_state.dl_plus_item_toggle);
-            continue;
-        }
-        if (key == "DL_PLUS_ITEM_RUNNING") {
-            parse_dl_param_bool(key, value, dl_state.dl_plus_item_running);
-            continue;
-        }
-        if (key == "DL_PLUS_TAG") {
-            if (dl_state.dl_plus_tags.size() == 4) {
-                fprintf(stderr, "mot-encoder Warning: DL Plus tag ignored, as already four tags present\n");
-                continue;
-            }
-
-            // split value
-            std::vector<std::string> params = split_string(value, ' ');
-            if (params.size() != 3) {
-                fprintf(stderr, "mot-encoder Warning: DL Plus tag value '%s' does not have three parts - ignored\n", value.c_str());
-                continue;
-            }
-
-            int content_type, start_marker, length_marker;
-            if (parse_dl_param_int_dl_plus_tag("content_type", params[0], content_type) &
-                parse_dl_param_int_dl_plus_tag("start_marker", params[1], start_marker) &
-                parse_dl_param_int_dl_plus_tag("length_marker", params[2], length_marker))
-                dl_state.dl_plus_tags.push_back(DL_PLUS_TAG(content_type, start_marker, length_marker));
-            continue;
-        }
-
-        fprintf(stderr, "mot-encoder Warning: DL parameter '%s' unknown - ignored\n", key.c_str());
-    }
-
-    fprintf(stderr, "mot-encoder Warning: no param closing tag, so the DLS text will be empty\n");
-}
-
-
-void writeDLS(int output_fd, const std::string& dls_file, uint8_t charset, bool raw_dls, bool remove_dls)
-{
-    std::ifstream dls_fstream(dls_file);
-    if (!dls_fstream.is_open()) {
-        std::cerr << "Could not open " << dls_file << std::endl;
-        return;
-    }
-
-    DL_STATE dl_state;
-
-    std::vector<std::string> dls_lines;
-
-    std::string line;
-    // Read and convert lines one by one because the converter doesn't understand
-    // line endings
-    while (std::getline(dls_fstream, line)) {
-        if (line.empty())
-            continue;
-        if (line == DL_PARAMS_OPEN) {
-            parse_dl_params(dls_fstream, dl_state);
-        } else {
-            if (not raw_dls && charset == CHARSET_UTF8) {
-                dls_lines.push_back(charset_converter.convert(line));
-            }
-            else {
-                dls_lines.push_back(line);
-            }
-            // TODO handle the other charsets accordingly
-        }
-    }
-
-    std::stringstream ss;
-    for (size_t i = 0; i < dls_lines.size(); i++) {
-        if (i != 0) {
-            if (charset == CHARSET_UCS2_BE)
-                ss << '\0' << '\n';
-            else
-                ss << '\n';
-        }
-
-        // UCS-2 BE: if from file the first byte of \0\n remains, remove it
-        if (charset == CHARSET_UCS2_BE && dls_lines[i].size() % 2) {
-            dls_lines[i].resize(dls_lines[i].size() - 1);
-        }
-
-        ss << dls_lines[i];
-    }
-
-    dl_state.dl_text = ss.str();
-    if (dl_state.dl_text.size() > MAXDLS)
-        dl_state.dl_text.resize(MAXDLS);
-
-
-    // if DL Plus enabled, but no DL Plus tags were added, add the required DUMMY tag
-    if (dl_state.dl_plus_enabled && dl_state.dl_plus_tags.empty())
-        dl_state.dl_plus_tags.push_back(DL_PLUS_TAG());
-
-    if (not raw_dls)
-        charset = CHARSET_COMPLETE_EBU_LATIN;
-
-
-    // toggle the toggle bit only on new DL state
-    bool dl_state_is_new = dl_state != dl_state_prev;
-    if (verbose) {
-        fprintf(stderr, "mot-encoder writing %s DLS text \"%s\"\n", dl_state_is_new ? "new" : "old", dl_state.dl_text.c_str());
-        if (dl_state.dl_plus_enabled) {
-            fprintf(
-                    stderr, "mot-encoder writing %s DL Plus tags (IT/IR: %d/%d): ",
-                    dl_state_is_new ? "new" : "old",
-                    dl_state.dl_plus_item_toggle ? 1 : 0,
-                    dl_state.dl_plus_item_running ? 1 : 0);
-            for (dl_plus_tags_t::const_iterator it = dl_state.dl_plus_tags.begin(); it != dl_state.dl_plus_tags.end(); it++) {
-                if (it != dl_state.dl_plus_tags.begin())
-                    fprintf(stderr, ", ");
-                fprintf(stderr, "%d (S/L: %d/%d)", it->content_type, it->start_marker, it->length_marker);
-            }
-            fprintf(stderr, "\n");
-        }
-    }
-
-    DATA_GROUP *remove_label_dg = NULL;
-    if (dl_state_is_new) {
-        if (remove_dls)
-            remove_label_dg = createDynamicLabelCommand(DLS_CMD_REMOVE_LABEL);
-
-        dls_toggle = !dls_toggle;   // indicate changed text
-
-        dl_state_prev = dl_state;
-    }
-
-    prepend_dl_dgs(dl_state, charset);
-    if (remove_label_dg)
-        pad_packetizer->queue.push_front(remove_label_dg);
-    pad_packetizer->WriteAllPADs(output_fd);
-}
-
-
-int dls_count(const std::string& text) {
-    size_t text_len = text.size();
-    return text_len / DLS_SEG_LEN_CHAR_MAX + (text_len % DLS_SEG_LEN_CHAR_MAX ? 1 : 0);
-}
-
-
-DATA_GROUP* dls_get(const std::string& text, uint8_t charset, int seg_index) {
-    bool first_seg = seg_index == 0;
-    bool last_seg  = seg_index == dls_count(text) - 1;
-
-    int seg_text_offset = seg_index * DLS_SEG_LEN_CHAR_MAX;
-    const char *seg_text_start = text.c_str() + seg_text_offset;
-    size_t seg_text_len = MIN(text.size() - seg_text_offset, DLS_SEG_LEN_CHAR_MAX);
-
-    DATA_GROUP* dg = new DATA_GROUP(DLS_SEG_LEN_PREFIX + seg_text_len, 2, 3);
-    uint8_vector_t &seg_data = dg->data;
-
-    // prefix: toggle? + first seg? + last seg? + (seg len - 1)
-    seg_data[0] =
-            (dls_toggle ? (1 << 7) : 0) +
-            (first_seg  ? (1 << 6) : 0) +
-            (last_seg   ? (1 << 5) : 0) +
-            (seg_text_len - 1);
-
-    // prefix: charset / seg index
-    seg_data[1] = (first_seg ? charset : seg_index) << 4;
-
-    // character field
-    memcpy(&seg_data[DLS_SEG_LEN_PREFIX], seg_text_start, seg_text_len);
-
-    // CRC
-    dg->AppendCRC();
-
-#if DEBUG
-    fprintf(stderr, "DL segment:");
-    for (int i = 0; i < seg_data.size(); i++)
-        fprintf(stderr, " %02x", seg_data[i]);
-    fprintf(stderr, "\n");
-#endif
-    return dg;
-}
-
-
-void prepend_dl_dgs(const DL_STATE& dl_state, uint8_t charset) {
-    // process all DL segments
-    int seg_count = dls_count(dl_state.dl_text);
-    std::vector<DATA_GROUP*> segs;
-    for (int seg_index = 0; seg_index < seg_count; seg_index++) {
-#if DEBUG
-        fprintf(stderr, "Segment number %d\n", seg_index + 1);
-#endif
-        segs.push_back(dls_get(dl_state.dl_text, charset, seg_index));
-    }
-
-    // if enabled, add DL Plus data group
-    if (dl_state.dl_plus_enabled)
-        segs.push_back(createDynamicLabelPlus(dl_state));
-
-    // prepend to packetizer
-    pad_packetizer->queue.insert(pad_packetizer->queue.begin(), segs.begin(), segs.end());
-
-#if DEBUG
-    fprintf(stderr, "PAD length: %d\n", padlen);
-    fprintf(stderr, "DLS text: %s\n", text.c_str());
-    fprintf(stderr, "Number of DL segments: %d\n", seg_count);
-#endif
-}
-
-int History::find(const fingerprint_t& fp) const
-{
-    size_t i;
-    for (i = 0; i < m_database.size(); i++) {
-        if (m_database[i] == fp) {
-            // return the id of fingerprint found
-            return m_database[i].fidx;
-        }
-    }
-
-    // return -1 when the database doesn't contain this fingerprint
-    return -1;
-}
-
-void History::add(fingerprint_t& fp)
-{
-    m_database.push_back(fp);
-
-    if (m_database.size() > m_hist_size) {
-        m_database.pop_front();
-    }
-}
-
-void History::disp_database()
-{
-    size_t id;
-    printf("HISTORY DATABASE:\n");
-    if (m_database.empty()) {
-        printf(" empty\n");
-    }
-    else {
-        for (id = 0; id < m_database.size(); id++) {
-            printf(" id %4zu: ", id);
-            m_database[id].disp();
-        }
-    }
-    printf("-----------------\n");
-}
-
-int History::get_fidx(const char* filepath)
-{
-    fingerprint_t fp;
-
-    fp.load_from_file(filepath);
-
-    int idx = find(fp);
-
-    if (idx < 0) {
-        idx = m_last_given_fidx++;
-        fp.fidx = idx;
-
-        if (m_last_given_fidx > MAXSLIDEID) {
-            m_last_given_fidx = 0;
-        }
-
-        add(fp);
-    }
-
-    return idx;
-}
-
diff --git a/src/utf8.h b/src/utf8.h
deleted file mode 100644
index 4e44514..0000000
--- a/src/utf8.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic

-

-/*

-Permission is hereby granted, free of charge, to any person or organization

-obtaining a copy of the software and accompanying documentation covered by

-this license (the "Software") to use, reproduce, display, distribute,

-execute, and transmit the Software, and to prepare derivative works of the

-Software, and to permit third-parties to whom the Software is furnished to

-do so, all subject to the following:

-

-The copyright notices in the Software and this entire statement, including

-the above license grant, this restriction and the following disclaimer,

-must be included in all copies of the Software, in whole or in part, and

-all derivative works of the Software, unless such copies or derivative

-works are solely in the form of machine-executable object code generated by

-a source language processor.

-

-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT

-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE

-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,

-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER

-DEALINGS IN THE SOFTWARE.

-*/

-

-

-#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731

-#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731

-

-#include "utf8/checked.h"

-#include "utf8/unchecked.h"

-

-#endif // header guard

diff --git a/src/utf8/checked.h b/src/utf8/checked.h
deleted file mode 100644
index 1331155..0000000
--- a/src/utf8/checked.h
+++ /dev/null
@@ -1,327 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "core.h"
-#include <stdexcept>
-
-namespace utf8
-{
-    // Base for the exceptions that may be thrown from the library
-    class exception : public ::std::exception {
-    };
-
-    // Exceptions that may be thrown from the library functions.
-    class invalid_code_point : public exception {
-        uint32_t cp;
-    public:
-        invalid_code_point(uint32_t cp) : cp(cp) {}
-        virtual const char* what() const throw() { return "Invalid code point"; }
-        uint32_t code_point() const {return cp;}
-    };
-
-    class invalid_utf8 : public exception {
-        uint8_t u8;
-    public:
-        invalid_utf8 (uint8_t u) : u8(u) {}
-        virtual const char* what() const throw() { return "Invalid UTF-8"; }
-        uint8_t utf8_octet() const {return u8;}
-    };
-
-    class invalid_utf16 : public exception {
-        uint16_t u16;
-    public:
-        invalid_utf16 (uint16_t u) : u16(u) {}
-        virtual const char* what() const throw() { return "Invalid UTF-16"; }
-        uint16_t utf16_word() const {return u16;}
-    };
-
-    class not_enough_room : public exception {
-    public:
-        virtual const char* what() const throw() { return "Not enough space"; }
-    };
-
-    /// The library API - functions intended to be called by the users
-
-    template <typename octet_iterator>
-    octet_iterator append(uint32_t cp, octet_iterator result)
-    {
-        if (!utf8::internal::is_code_point_valid(cp))
-            throw invalid_code_point(cp);
-
-        if (cp < 0x80)                        // one octet
-            *(result++) = static_cast<uint8_t>(cp);
-        else if (cp < 0x800) {                // two octets
-            *(result++) = static_cast<uint8_t>((cp >> 6)            | 0xc0);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        else if (cp < 0x10000) {              // three octets
-            *(result++) = static_cast<uint8_t>((cp >> 12)           | 0xe0);
-            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        else {                                // four octets
-            *(result++) = static_cast<uint8_t>((cp >> 18)           | 0xf0);
-            *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)  | 0x80);
-            *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f)   | 0x80);
-            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
-        }
-        return result;
-    }
-
-    template <typename octet_iterator, typename output_iterator>
-    output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
-    {
-        while (start != end) {
-            octet_iterator sequence_start = start;
-            internal::utf_error err_code = utf8::internal::validate_next(start, end);
-            switch (err_code) {
-                case internal::UTF8_OK :
-                    for (octet_iterator it = sequence_start; it != start; ++it)
-                        *out++ = *it;
-                    break;
-                case internal::NOT_ENOUGH_ROOM:
-                    throw not_enough_room();
-                case internal::INVALID_LEAD:
-                    out = utf8::append (replacement, out);
-                    ++start;
-                    break;
-                case internal::INCOMPLETE_SEQUENCE:
-                case internal::OVERLONG_SEQUENCE:
-                case internal::INVALID_CODE_POINT:
-                    out = utf8::append (replacement, out);
-                    ++start;
-                    // just one replacement mark for the sequence
-                    while (start != end && utf8::internal::is_trail(*start))
-                        ++start;
-                    break;
-            }
-        }
-        return out;
-    }
-
-    template <typename octet_iterator, typename output_iterator>
-    inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
-    {
-        static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
-        return utf8::replace_invalid(start, end, out, replacement_marker);
-    }
-
-    template <typename octet_iterator>
-    uint32_t next(octet_iterator& it, octet_iterator end)
-    {
-        uint32_t cp = 0;
-        internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
-        switch (err_code) {
-            case internal::UTF8_OK :
-                break;
-            case internal::NOT_ENOUGH_ROOM :
-                throw not_enough_room();
-            case internal::INVALID_LEAD :
-            case internal::INCOMPLETE_SEQUENCE :
-            case internal::OVERLONG_SEQUENCE :
-                throw invalid_utf8(*it);
-            case internal::INVALID_CODE_POINT :
-                throw invalid_code_point(cp);
-        }
-        return cp;
-    }
-
-    template <typename octet_iterator>
-    uint32_t peek_next(octet_iterator it, octet_iterator end)
-    {
-        return utf8::next(it, end);
-    }
-
-    template <typename octet_iterator>
-    uint32_t prior(octet_iterator& it, octet_iterator start)
-    {
-        // can't do much if it == start
-        if (it == start)
-            throw not_enough_room();
-
-        octet_iterator end = it;
-        // Go back until we hit either a lead octet or start
-        while (utf8::internal::is_trail(*(--it)))
-            if (it == start)
-                throw invalid_utf8(*it); // error - no lead byte in the sequence
-        return utf8::peek_next(it, end);
-    }
-
-    /// Deprecated in versions that include "prior"
-    template <typename octet_iterator>
-    uint32_t previous(octet_iterator& it, octet_iterator pass_start)
-    {
-        octet_iterator end = it;
-        while (utf8::internal::is_trail(*(--it)))
-            if (it == pass_start)
-                throw invalid_utf8(*it); // error - no lead byte in the sequence
-        octet_iterator temp = it;
-        return utf8::next(temp, end);
-    }
-
-    template <typename octet_iterator, typename distance_type>
-    void advance (octet_iterator& it, distance_type n, octet_iterator end)
-    {
-        for (distance_type i = 0; i < n; ++i)
-            utf8::next(it, end);
-    }
-
-    template <typename octet_iterator>
-    typename std::iterator_traits<octet_iterator>::difference_type
-    distance (octet_iterator first, octet_iterator last)
-    {
-        typename std::iterator_traits<octet_iterator>::difference_type dist;
-        for (dist = 0; first < last; ++dist)
-            utf8::next(first, last);
-        return dist;
-    }
-
-    template <typename u16bit_iterator, typename octet_iterator>
-    octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
-    {
-        while (start != end) {
-            uint32_t cp = utf8::internal::mask16(*start++);
-            // Take care of surrogate pairs first
-            if (utf8::internal::is_lead_surrogate(cp)) {
-                if (start != end) {
-                    uint32_t trail_surrogate = utf8::internal::mask16(*start++);
-                    if (utf8::internal::is_trail_surrogate(trail_surrogate))
-                        cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
-                    else
-                        throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
-                }
-                else
-                    throw invalid_utf16(static_cast<uint16_t>(cp));
-
-            }
-            // Lone trail surrogate
-            else if (utf8::internal::is_trail_surrogate(cp))
-                throw invalid_utf16(static_cast<uint16_t>(cp));
-
-            result = utf8::append(cp, result);
-        }
-        return result;
-    }
-
-    template <typename u16bit_iterator, typename octet_iterator>
-    u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
-    {
-        while (start != end) {
-            uint32_t cp = utf8::next(start, end);
-            if (cp > 0xffff) { //make a surrogate pair
-                *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
-                *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
-            }
-            else
-                *result++ = static_cast<uint16_t>(cp);
-        }
-        return result;
-    }
-
-    template <typename octet_iterator, typename u32bit_iterator>
-    octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
-    {
-        while (start != end)
-            result = utf8::append(*(start++), result);
-
-        return result;
-    }
-
-    template <typename octet_iterator, typename u32bit_iterator>
-    u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
-    {
-        while (start != end)
-            (*result++) = utf8::next(start, end);
-
-        return result;
-    }
-
-    // The iterator class
-    template <typename octet_iterator>
-    class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
-      octet_iterator it;
-      octet_iterator range_start;
-      octet_iterator range_end;
-      public:
-      iterator () {}
-      explicit iterator (const octet_iterator& octet_it,
-                         const octet_iterator& range_start,
-                         const octet_iterator& range_end) :
-               it(octet_it), range_start(range_start), range_end(range_end)
-      {
-          if (it < range_start || it > range_end)
-              throw std::out_of_range("Invalid utf-8 iterator position");
-      }
-      // the default "big three" are OK
-      octet_iterator base () const { return it; }
-      uint32_t operator * () const
-      {
-          octet_iterator temp = it;
-          return utf8::next(temp, range_end);
-      }
-      bool operator == (const iterator& rhs) const
-      {
-          if (range_start != rhs.range_start || range_end != rhs.range_end)
-              throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
-          return (it == rhs.it);
-      }
-      bool operator != (const iterator& rhs) const
-      {
-          return !(operator == (rhs));
-      }
-      iterator& operator ++ ()
-      {
-          utf8::next(it, range_end);
-          return *this;
-      }
-      iterator operator ++ (int)
-      {
-          iterator temp = *this;
-          utf8::next(it, range_end);
-          return temp;
-      }
-      iterator& operator -- ()
-      {
-          utf8::prior(it, range_start);
-          return *this;
-      }
-      iterator operator -- (int)
-      {
-          iterator temp = *this;
-          utf8::prior(it, range_start);
-          return temp;
-      }
-    }; // class iterator
-
-} // namespace utf8
-
-#endif //header guard
-
-
diff --git a/src/utf8/core.h b/src/utf8/core.h
deleted file mode 100644
index 693d388..0000000
--- a/src/utf8/core.h
+++ /dev/null
@@ -1,329 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include <iterator>
-
-namespace utf8
-{
-    // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
-    // You may need to change them to match your system.
-    // These typedefs have the same names as ones from cstdint, or boost/cstdint
-    typedef unsigned char   uint8_t;
-    typedef unsigned short  uint16_t;
-    typedef unsigned int    uint32_t;
-
-// Helper code - not intended to be directly called by the library users. May be changed at any time
-namespace internal
-{
-    // Unicode constants
-    // Leading (high) surrogates: 0xd800 - 0xdbff
-    // Trailing (low) surrogates: 0xdc00 - 0xdfff
-    const uint16_t LEAD_SURROGATE_MIN  = 0xd800u;
-    const uint16_t LEAD_SURROGATE_MAX  = 0xdbffu;
-    const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
-    const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
-    const uint16_t LEAD_OFFSET         = LEAD_SURROGATE_MIN - (0x10000 >> 10);
-    const uint32_t SURROGATE_OFFSET    = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
-
-    // Maximum valid value for a Unicode code point
-    const uint32_t CODE_POINT_MAX      = 0x0010ffffu;
-
-    template<typename octet_type>
-    inline uint8_t mask8(octet_type oc)
-    {
-        return static_cast<uint8_t>(0xff & oc);
-    }
-    template<typename u16_type>
-    inline uint16_t mask16(u16_type oc)
-    {
-        return static_cast<uint16_t>(0xffff & oc);
-    }
-    template<typename octet_type>
-    inline bool is_trail(octet_type oc)
-    {
-        return ((utf8::internal::mask8(oc) >> 6) == 0x2);
-    }
-
-    template <typename u16>
-    inline bool is_lead_surrogate(u16 cp)
-    {
-        return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
-    }
-
-    template <typename u16>
-    inline bool is_trail_surrogate(u16 cp)
-    {
-        return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
-    }
-
-    template <typename u16>
-    inline bool is_surrogate(u16 cp)
-    {
-        return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
-    }
-
-    template <typename u32>
-    inline bool is_code_point_valid(u32 cp)
-    {
-        return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
-    }
-
-    template <typename octet_iterator>
-    inline typename std::iterator_traits<octet_iterator>::difference_type
-    sequence_length(octet_iterator lead_it)
-    {
-        uint8_t lead = utf8::internal::mask8(*lead_it);
-        if (lead < 0x80)
-            return 1;
-        else if ((lead >> 5) == 0x6)
-            return 2;
-        else if ((lead >> 4) == 0xe)
-            return 3;
-        else if ((lead >> 3) == 0x1e)
-            return 4;
-        else
-            return 0;
-    }
-
-    template <typename octet_difference_type>
-    inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
-    {
-        if (cp < 0x80) {
-            if (length != 1) 
-                return true;
-        }
-        else if (cp < 0x800) {
-            if (length != 2) 
-                return true;
-        }
-        else if (cp < 0x10000) {
-            if (length != 3) 
-                return true;
-        }
-
-        return false;
-    }
-
-    enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
-
-    /// Helper for get_sequence_x
-    template <typename octet_iterator>
-    utf_error increase_safely(octet_iterator& it, octet_iterator end)
-    {
-        if (++it == end)
-            return NOT_ENOUGH_ROOM;
-
-        if (!utf8::internal::is_trail(*it))
-            return INCOMPLETE_SEQUENCE;
-        
-        return UTF8_OK;
-    }
-
-    #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}    
-
-    /// get_sequence_x functions decode utf-8 sequences of the length x
-    template <typename octet_iterator>
-    utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
-    {
-        if (it == end)
-            return NOT_ENOUGH_ROOM;
-
-        code_point = utf8::internal::mask8(*it);
-
-        return UTF8_OK;
-    }
-
-    template <typename octet_iterator>
-    utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
-    {
-        if (it == end) 
-            return NOT_ENOUGH_ROOM;
-        
-        code_point = utf8::internal::mask8(*it);
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
-
-        return UTF8_OK;
-    }
-
-    template <typename octet_iterator>
-    utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
-    {
-        if (it == end)
-            return NOT_ENOUGH_ROOM;
-            
-        code_point = utf8::internal::mask8(*it);
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point += (*it) & 0x3f;
-
-        return UTF8_OK;
-    }
-
-    template <typename octet_iterator>
-    utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
-    {
-        if (it == end)
-           return NOT_ENOUGH_ROOM;
-
-        code_point = utf8::internal::mask8(*it);
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
-
-        UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
-        code_point += (*it) & 0x3f;
-
-        return UTF8_OK;
-    }
-
-    #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
-
-    template <typename octet_iterator>
-    utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
-    {
-        // Save the original value of it so we can go back in case of failure
-        // Of course, it does not make much sense with i.e. stream iterators
-        octet_iterator original_it = it;
-
-        uint32_t cp = 0;
-        // Determine the sequence length based on the lead octet
-        typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
-        const octet_difference_type length = utf8::internal::sequence_length(it);
-
-        // Get trail octets and calculate the code point
-        utf_error err = UTF8_OK;
-        switch (length) {
-            case 0: 
-                return INVALID_LEAD;
-            case 1:
-                err = utf8::internal::get_sequence_1(it, end, cp);
-                break;
-            case 2:
-                err = utf8::internal::get_sequence_2(it, end, cp);
-            break;
-            case 3:
-                err = utf8::internal::get_sequence_3(it, end, cp);
-            break;
-            case 4:
-                err = utf8::internal::get_sequence_4(it, end, cp);
-            break;
-        }
-
-        if (err == UTF8_OK) {
-            // Decoding succeeded. Now, security checks...
-            if (utf8::internal::is_code_point_valid(cp)) {
-                if (!utf8::internal::is_overlong_sequence(cp, length)){
-                    // Passed! Return here.
-                    code_point = cp;
-                    ++it;
-                    return UTF8_OK;
-                }
-                else
-                    err = OVERLONG_SEQUENCE;
-            }
-            else 
-                err = INVALID_CODE_POINT;
-        }
-
-        // Failure branch - restore the original value of the iterator
-        it = original_it;
-        return err;
-    }
-
-    template <typename octet_iterator>
-    inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
-        uint32_t ignored;
-        return utf8::internal::validate_next(it, end, ignored);
-    }
-
-} // namespace internal
-
-    /// The library API - functions intended to be called by the users
-
-    // Byte order mark
-    const uint8_t bom[] = {0xef, 0xbb, 0xbf};
-
-    template <typename octet_iterator>
-    octet_iterator find_invalid(octet_iterator start, octet_iterator end)
-    {
-        octet_iterator result = start;
-        while (result != end) {
-            utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
-            if (err_code != internal::UTF8_OK)
-                return result;
-        }
-        return result;
-    }
-
-    template <typename octet_iterator>
-    inline bool is_valid(octet_iterator start, octet_iterator end)
-    {
-        return (utf8::find_invalid(start, end) == end);
-    }
-
-    template <typename octet_iterator>
-    inline bool starts_with_bom (octet_iterator it, octet_iterator end)
-    {
-        return (
-            ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
-            ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
-            ((it != end) && (utf8::internal::mask8(*it))   == bom[2])
-           );
-    }
-	
-    //Deprecated in release 2.3 
-    template <typename octet_iterator>
-    inline bool is_bom (octet_iterator it)
-    {
-        return (
-            (utf8::internal::mask8(*it++)) == bom[0] &&
-            (utf8::internal::mask8(*it++)) == bom[1] &&
-            (utf8::internal::mask8(*it))   == bom[2]
-           );
-    }
-} // namespace utf8
-
-#endif // header guard
-
-
diff --git a/src/utf8/unchecked.h b/src/utf8/unchecked.h
deleted file mode 100644
index cb24271..0000000
--- a/src/utf8/unchecked.h
+++ /dev/null
@@ -1,228 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "core.h"
-
-namespace utf8
-{
-    namespace unchecked 
-    {
-        template <typename octet_iterator>
-        octet_iterator append(uint32_t cp, octet_iterator result)
-        {
-            if (cp < 0x80)                        // one octet
-                *(result++) = static_cast<uint8_t>(cp);  
-            else if (cp < 0x800) {                // two octets
-                *(result++) = static_cast<uint8_t>((cp >> 6)          | 0xc0);
-                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
-            }
-            else if (cp < 0x10000) {              // three octets
-                *(result++) = static_cast<uint8_t>((cp >> 12)         | 0xe0);
-                *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
-                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
-            }
-            else {                                // four octets
-                *(result++) = static_cast<uint8_t>((cp >> 18)         | 0xf0);
-                *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
-                *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
-                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
-            }
-            return result;
-        }
-
-        template <typename octet_iterator>
-        uint32_t next(octet_iterator& it)
-        {
-            uint32_t cp = utf8::internal::mask8(*it);
-            typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
-            switch (length) {
-                case 1:
-                    break;
-                case 2:
-                    it++;
-                    cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
-                    break;
-                case 3:
-                    ++it; 
-                    cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
-                    ++it;
-                    cp += (*it) & 0x3f;
-                    break;
-                case 4:
-                    ++it;
-                    cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);                
-                    ++it;
-                    cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
-                    ++it;
-                    cp += (*it) & 0x3f; 
-                    break;
-            }
-            ++it;
-            return cp;        
-        }
-
-        template <typename octet_iterator>
-        uint32_t peek_next(octet_iterator it)
-        {
-            return utf8::unchecked::next(it);    
-        }
-
-        template <typename octet_iterator>
-        uint32_t prior(octet_iterator& it)
-        {
-            while (utf8::internal::is_trail(*(--it))) ;
-            octet_iterator temp = it;
-            return utf8::unchecked::next(temp);
-        }
-
-        // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
-        template <typename octet_iterator>
-        inline uint32_t previous(octet_iterator& it)
-        {
-            return utf8::unchecked::prior(it);
-        }
-
-        template <typename octet_iterator, typename distance_type>
-        void advance (octet_iterator& it, distance_type n)
-        {
-            for (distance_type i = 0; i < n; ++i)
-                utf8::unchecked::next(it);
-        }
-
-        template <typename octet_iterator>
-        typename std::iterator_traits<octet_iterator>::difference_type
-        distance (octet_iterator first, octet_iterator last)
-        {
-            typename std::iterator_traits<octet_iterator>::difference_type dist;
-            for (dist = 0; first < last; ++dist) 
-                utf8::unchecked::next(first);
-            return dist;
-        }
-
-        template <typename u16bit_iterator, typename octet_iterator>
-        octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
-        {       
-            while (start != end) {
-                uint32_t cp = utf8::internal::mask16(*start++);
-            // Take care of surrogate pairs first
-                if (utf8::internal::is_lead_surrogate(cp)) {
-                    uint32_t trail_surrogate = utf8::internal::mask16(*start++);
-                    cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
-                }
-                result = utf8::unchecked::append(cp, result);
-            }
-            return result;         
-        }
-
-        template <typename u16bit_iterator, typename octet_iterator>
-        u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
-        {
-            while (start < end) {
-                uint32_t cp = utf8::unchecked::next(start);
-                if (cp > 0xffff) { //make a surrogate pair
-                    *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
-                    *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
-                }
-                else
-                    *result++ = static_cast<uint16_t>(cp);
-            }
-            return result;
-        }
-
-        template <typename octet_iterator, typename u32bit_iterator>
-        octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
-        {
-            while (start != end)
-                result = utf8::unchecked::append(*(start++), result);
-
-            return result;
-        }
-
-        template <typename octet_iterator, typename u32bit_iterator>
-        u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
-        {
-            while (start < end)
-                (*result++) = utf8::unchecked::next(start);
-
-            return result;
-        }
-
-        // The iterator class
-        template <typename octet_iterator>
-          class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> { 
-            octet_iterator it;
-            public:
-            iterator () {}
-            explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
-            // the default "big three" are OK
-            octet_iterator base () const { return it; }
-            uint32_t operator * () const
-            {
-                octet_iterator temp = it;
-                return utf8::unchecked::next(temp);
-            }
-            bool operator == (const iterator& rhs) const 
-            { 
-                return (it == rhs.it);
-            }
-            bool operator != (const iterator& rhs) const
-            {
-                return !(operator == (rhs));
-            }
-            iterator& operator ++ () 
-            {
-                ::std::advance(it, utf8::internal::sequence_length(it));
-                return *this;
-            }
-            iterator operator ++ (int)
-            {
-                iterator temp = *this;
-                ::std::advance(it, utf8::internal::sequence_length(it));
-                return temp;
-            }  
-            iterator& operator -- ()
-            {
-                utf8::unchecked::prior(it);
-                return *this;
-            }
-            iterator operator -- (int)
-            {
-                iterator temp = *this;
-                utf8::unchecked::prior(it);
-                return temp;
-            }
-          }; // class iterator
-
-    } // namespace utf8::unchecked
-} // namespace utf8 
-
-
-#endif // header guard
-