aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile.am9
-rw-r--r--configure.ac16
-rw-r--r--src/charset-test.cpp57
-rw-r--r--src/charset.h115
-rw-r--r--src/mot-encoder.cpp1831
-rw-r--r--src/utf8.h34
-rw-r--r--src/utf8/checked.h327
-rw-r--r--src/utf8/core.h329
-rw-r--r--src/utf8/unchecked.h228
9 files changed, 1 insertions, 2945 deletions
diff --git a/Makefile.am b/Makefile.am
index b1138aa..31eccfc 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -94,14 +94,7 @@ odr_audioencoder_SOURCES = src/odr-audioencoder.cpp \
src/utf8/core.h \
src/common.h
-mot_encoder_CXXFLAGS = $(GITVERSION_FLAGS) @MAGICKWAND_CFLAGS@ -Icontrib -Wall -Wextra
-mot_encoder_LDADD = @MAGICKWAND_LDADD@
-mot_encoder_SOURCES = src/mot-encoder.cpp \
- contrib/lib_crc.h \
- contrib/lib_crc.c
-
-bin_PROGRAMS = odr-audioencoder$(EXEEXT) \
- mot-encoder$(EXEEXT)
+bin_PROGRAMS = odr-audioencoder$(EXEEXT)
noinst_HEADERS = src/wavreader.h
diff --git a/configure.ac b/configure.ac
index 4c92bd7..39a0ee7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -72,18 +72,6 @@ AM_CONDITIONAL([HAVE_ALSA], [ test "x$enable_alsa" = "xyes" ])
AC_CHECK_LIB(zmq, zmq_init, , AC_MSG_ERROR(ZeroMQ libzmq is required))
AC_CHECK_LIB(fdk-aac, aacEncOpen, , AC_MSG_ERROR(The FDK-AAC library is required))
-if pkg-config MagickWand; then
- MAGICKWAND_CFLAGS=`pkg-config MagickWand --cflags`
- MAGICKWAND_LDADD=`pkg-config MagickWand --libs`
- AC_SUBST(MAGICKWAND_CFLAGS)
- AC_SUBST(MAGICKWAND_LDADD)
-else
- AC_MSG_WARN(MagickWand not found)
-fi
-
-AS_IF([ pkg-config MagickWand ],
- AC_DEFINE(HAVE_MAGICKWAND, [1], [Define if MagickWand is available]))
-
dnl soname version to use
dnl goes by ‘current[:revision[:age]]’ with the soname ending up as
@@ -114,9 +102,5 @@ do
[disabled="$disabled $feature"])
done
-AS_IF([ pkg-config MagickWand ],
- [enabled="$enabled magickwand"],
- [disabled="$disabled magickwand"])
-
echo " Enabled: $enabled"
echo " Disabled: $disabled"
diff --git a/src/charset-test.cpp b/src/charset-test.cpp
deleted file mode 100644
index d65fcc7..0000000
--- a/src/charset-test.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- Copyright (C) 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-
- charset.cpp
- This is test code for the character set conversion. It does not
- get compiled by default.
- Please use
- g++ -Wall -Wextra -std=c++11 -o charset-test charset-test.cpp
- to compile this, and create a input.txt file with test data.
-
- Authors:
- Matthias P. Braendli <matthias@mpb.li>
-*/
-
-#include "charset.h"
-#include <iostream>
-#include <fstream>
-
-#include <cstdio>
-
-using namespace std;
-
-int main(int argc, char** argv)
-{
- string test_file_path("input.txt");
-
- ifstream fs8(test_file_path);
- if (!fs8.is_open()) {
- cerr << "Could not open " << test_file_path << endl;
- return 1;
- }
-
- CharsetConverter conv;
-
- string line;
- // Play with all the lines in the file
- while (getline(fs8, line)) {
- cout << conv.convert(line) << endl;
- }
-
- return 0;
-}
-
-
diff --git a/src/charset.h b/src/charset.h
deleted file mode 100644
index 8d1e1a2..0000000
--- a/src/charset.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- Copyright (C) 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-/*!
- \file charset.h
- \brief Define the EBU charset according to ETSI TS 101 756v1.8.1 for DLS encoding
-
- \author Matthias P. Braendli <matthias@mpb.li>
- \author Lindsay Cornell
-*/
-
-#ifndef __CHARSET_H_
-#define __CHARSET_H_
-
-#include "utf8.h"
-#include <string>
-#include <vector>
-#include <algorithm>
-
-/**********************************************/
-/************* BIG FAT WARNING ****************/
-/**********************************************/
-/**** Make sure this file is always saved ****/
-/**** encoded in UTF-8, otherwise you will ****/
-/**** mess up the table below ! ****/
-/**********************************************/
-/********* END OF BIG FAT WARNING *************/
-/**********************************************/
-
-
-#define CHARSET_TABLE_OFFSET 1 // NUL at index 0 cannot be represented
-#define CHARSET_TABLE_ENTRIES (256 - CHARSET_TABLE_OFFSET)
-const char* utf8_encoded_EBU_Latin[CHARSET_TABLE_ENTRIES] = {
- "Ę", "Į", "Ų", "Ă", "Ė", "Ď", "Ș", "Ț", "Ċ", "\n","\v","Ġ", "Ĺ", "Ż", "Ń",
-"ą", "ę", "į", "ų", "ă", "ė", "ď", "ș", "ț", "ċ", "Ň", "Ě", "ġ", "ĺ", "ż", "\u0082",
-" ", "!", "\"","#", "ł", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/",
-"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?",
-"@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O",
-"P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "Ů", "]", "Ł", "_",
-"Ą", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
-"p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "«", "ů", "»", "Ľ", "Ħ",
-"á", "à", "é", "è", "í", "ì", "ó", "ò", "ú", "ù", "Ñ", "Ç", "Ş", "ß", "¡", "Ÿ",
-"â", "ä", "ê", "ë", "î", "ï", "ô", "ö", "û", "ü", "ñ", "ç", "ş", "ğ", "ı", "ÿ",
-"Ķ", "Ņ", "©", "Ģ", "Ğ", "ě", "ň", "ő", "Ő", "€", "£", "$", "Ā", "Ē", "Ī", "Ū",
-"ķ", "ņ", "Ļ", "ģ", "ļ", "İ", "ń", "ű", "Ű", "¿", "ľ", "°", "ā", "ē", "ī", "ū",
-"Á", "À", "É", "È", "Í", "Ì", "Ó", "Ò", "Ú", "Ù", "Ř", "Č", "Š", "Ž", "Ð", "Ŀ",
-"Â", "Ä", "Ê", "Ë", "Î", "Ï", "Ô", "Ö", "Û", "Ü", "ř", "č", "š", "ž", "đ", "ŀ",
-"Ã", "Å", "Æ", "Œ", "ŷ", "Ý", "Õ", "Ø", "Þ", "Ŋ", "Ŕ", "Ć", "Ś", "Ź", "Ť", "ð",
-"ã", "å", "æ", "œ", "ŵ", "ý", "õ", "ø", "þ", "ŋ", "ŕ", "ć", "ś", "ź", "ť", "ħ"};
-
-class CharsetConverter
-{
- public:
- CharsetConverter() {
- /*! Build the converstion table that contains the known code points,
- * at the indices corresponding to the EBU Latin table
- */
- using namespace std;
- for (size_t i = 0; i < CHARSET_TABLE_ENTRIES; i++) {
- string table_entry(utf8_encoded_EBU_Latin[i]);
- string::iterator it = table_entry.begin();
- uint32_t code_point = utf8::next(it, table_entry.end());
- m_conversion_table.push_back(code_point);
- }
- }
-
- /*! Convert a UTF-8 encoded text line into an EBU Latin encoded byte stream
- */
- std::string convert(std::string line_utf8) {
- using namespace std;
-
- // check for invalid utf-8, we only convert up to the first error
- string::iterator end_it = utf8::find_invalid(line_utf8.begin(), line_utf8.end());
-
- // Convert it to utf-32
- vector<uint32_t> utf32line;
- utf8::utf8to32(line_utf8.begin(), end_it, back_inserter(utf32line));
-
- string encoded_line(utf32line.size(), '0');
-
- // Try to convert each codepoint
- for (size_t i = 0; i < utf32line.size(); i++) {
- vector<uint32_t>::iterator iter = find(m_conversion_table.begin(),
- m_conversion_table.end(), utf32line[i]);
- if (iter != m_conversion_table.end()) {
- size_t index = std::distance(m_conversion_table.begin(), iter);
-
- encoded_line[i] = (char)(index + CHARSET_TABLE_OFFSET);
- }
- else {
- encoded_line[i] = ' ';
- }
- }
- return encoded_line;
- }
-
- private:
-
- std::vector<uint32_t> m_conversion_table;
-};
-
-#endif
diff --git a/src/mot-encoder.cpp b/src/mot-encoder.cpp
deleted file mode 100644
index 4cb6688..0000000
--- a/src/mot-encoder.cpp
+++ /dev/null
@@ -1,1831 +0,0 @@
-/*
- Copyright (C) 2014 CSP Innovazione nelle ICT s.c.a r.l. (http://rd.csp.it/)
-
- Copyright (C) 2014, 2015 Matthias P. Braendli (http://opendigitalradio.org)
-
- Copyright (C) 2015, 2016 Stefan Pöschel (http://opendigitalradio.org)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-*/
-/*!
- \file mot-encoder.cpp
- \brief Generete PAD data for MOT Slideshow and DLS
-
- \author Sergio Sagliocco <sergio.sagliocco@csp.it>
- \author Matthias P. Braendli <matthias@mpb.li>
- \author Stefan Pöschel <odr@basicmaster.de>
-*/
-
-#include <cstdio>
-#include <stdlib.h>
-#include <stdint.h>
-#include <unistd.h>
-#include <cstring>
-#include <string>
-#include <sstream>
-#include <iostream>
-#include <fstream>
-#include <algorithm>
-#include <vector>
-#include <deque>
-#include <list>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <getopt.h>
-#include "config.h"
-#include "charset.h"
-
-
-#if HAVE_MAGICKWAND
-# include <wand/magick_wand.h>
-#endif
-
-#define DEBUG 0
-
-#define SLEEPDELAY_DEFAULT 10 //seconds
-
-extern "C" {
-#include "lib_crc.h"
-}
-
-#define MIN(a,b) (((a)<(b))?(a):(b))
-#define MAX(a,b) (((a)>(b))?(a):(b))
-
-#define XSTR(x) #x
-#define STR(x) XSTR(x)
-
-#define MAXSEGLEN 8189 // Bytes (EN 301 234 v2.1.1, ch. 5.1.1)
-#define MAXDLS 128 // chars
-#define MAXSLIDESIZE 51200 // Bytes (TS 101 499 v3.1.1, ch. 9.1.2)
-
-// Roll-over value for fidx
-#define MAXSLIDEID 9999
-
-// How many slides to keep in history
-#define MAXHISTORYLEN 50
-
-// Do not allow the image compressor to go below
-// JPEG quality 40
-#define MINQUALITY 40
-
-// Charsets from TS 101 756
-#define CHARSET_COMPLETE_EBU_LATIN 0 //!< Complete EBU Latin based repertoire
-#define CHARSET_EBU_LATIN_CY_GR 1 //!< EBU Latin based common core, Cyrillic, Greek
-#define CHARSET_EBU_LATIN_AR_HE_CY_GR 2 //!< EBU Latin based core, Arabic, Hebrew, Cyrillic and Greek
-#define CHARSET_ISO_LATIN_ALPHABET_2 3 //!< ISO Latin Alphabet No 2
-#define CHARSET_UCS2_BE 6 //!< ISO/IEC 10646 using UCS-2 transformation format, big endian byte order
-#define CHARSET_UTF8 15 //!< ISO Latin Alphabet No 2
-
-
-typedef std::vector<uint8_t> uint8_vector_t;
-static int verbose = 0;
-
-
-struct MSCDG {
- // MSC Data Group Header (extension field not supported)
- unsigned char extflag; // 1 bit
- unsigned char crcflag; // 1 bit
- unsigned char segflag; // 1 bit
- unsigned char accflag; // 1 bit
- unsigned char dgtype; // 4 bits
- unsigned char cindex; // 4 bits
- unsigned char rindex; // 4 bits
- /// Session header - Segment field
- unsigned char last; // 1 bit
- unsigned short int segnum; // 16 bits
- // Session header - User access field
- unsigned char rfa; // 3 bits
- unsigned char tidflag; // 1 bit
- unsigned char lenid; // 4 bits - Fixed to value 2 in this implemntation
- unsigned short int tid; // 16 bits
- // MSC data group data field
- // Mot Segmentation header
- unsigned char rcount; // 3 bits
- unsigned short int seglen; // 13 bits
- // Mot segment
- unsigned char* segdata;
- // MSC data group CRC
- unsigned short int crc; // 16 bits
-};
-
-/*! Between collection of slides and transmission, the slide data is saved
- * in this structure.
- */
-struct slide_metadata_t {
- // complete path to slide
- std::string filepath;
-
- // index, values from 0 to MAXSLIDEID, rolls over
- int fidx;
-
- // This is used to define the order in which several discovered
- // slides are transmitted
- bool operator<(const slide_metadata_t& other) const {
- return this->fidx < other.fidx;
- }
-};
-
-/*! A simple fingerprint for each slide transmitted.
- * Allows us to reuse the same fidx if the same slide
- * is transmitted more than once.
- */
-struct fingerprint_t {
- // file name
- std::string s_name;
- // file size, in bytes
- off_t s_size;
- // time of last modification
- unsigned long s_mtime;
-
- // assigned fidx, -1 means invalid
- int fidx;
-
- /*! The comparison is not done on fidx, only
- * on the file-specific data
- */
- bool operator==(const fingerprint_t& other) const {
- return (((s_name == other.s_name &&
- s_size == other.s_size) &&
- s_mtime == other.s_mtime));
- }
-
- void disp(void) {
- printf("%s_%ld_%lu:%d\n", s_name.c_str(), s_size, s_mtime, fidx);
- }
-
- void load_from_file(const char* filepath)
- {
- struct stat file_attribue;
- const char * final_slash;
-
- stat(filepath, &file_attribue);
- final_slash = strrchr(filepath, '/');
-
- // load filename, size and mtime
- // Save only the basename of the filepath
- this->s_name.assign((final_slash == NULL) ? filepath : final_slash + 1);
- this->s_size = file_attribue.st_size;
- this->s_mtime = file_attribue.st_mtime;
-
- this->fidx = -1;
- }
-};
-
-/*! We keep track of transmitted files so that we can retransmit
- * identical slides with the same index, in case the receivers cache
- * them.
- *
- * \c MAXHISTORYLEN defines for how how many slides we want to keep this
- * history.
- */
-class History {
- public:
- History(size_t hist_size) :
- m_hist_size(hist_size),
- m_last_given_fidx(0) {}
- void disp_database();
- // controller of id base on database
- int get_fidx(const char* filepath);
-
- private:
- std::deque<fingerprint_t> m_database;
-
- size_t m_hist_size;
-
- int m_last_given_fidx;
-
- // find the fingerprint fp in database.
- // returns the fidx when found,
- // or -1 if not found
- int find(const fingerprint_t& fp) const;
-
- // add a new fingerprint into database
- // returns its fidx
- void add(fingerprint_t& fp);
-};
-
-
-int encodeFile(int output_fd, const std::string& fname, int fidx, bool raw_slides);
-
-uint8_vector_t createMotHeader(
- size_t blobsize,
- int fidx,
- bool jfif_not_png);
-
-void createMscDG(MSCDG* msc, unsigned short int dgtype, int *cindex, unsigned short int segnum,
- unsigned short int lastseg, unsigned short int tid, unsigned char* data,
- unsigned short int datalen);
-
-struct DATA_GROUP;
-DATA_GROUP* packMscDG(MSCDG* msc);
-
-struct DL_STATE;
-void prepend_dl_dgs(const DL_STATE& dl_state, uint8_t charset);
-void writeDLS(int output_fd, const std::string& dls_file, uint8_t charset, bool raw_dls, bool remove_dls);
-
-// PAD related
-#define CRC_LEN 2
-
-struct DATA_GROUP {
- uint8_vector_t data;
- int apptype_start;
- int apptype_cont;
- size_t written;
-
- DATA_GROUP(size_t len, int apptype_start, int apptype_cont) {
- this->data.resize(len);
- this->apptype_start = apptype_start;
- this->apptype_cont = apptype_cont;
- written = 0;
- }
-
- void AppendCRC() {
- uint16_t crc = 0xFFFF;
- for (size_t i = 0; i < data.size(); i++)
- crc = update_crc_ccitt(crc, data[i]);
- crc = ~crc;
-#if DEBUG
- fprintf(stderr, "crc=%04x ~crc=%04x\n", crc, ~crc);
-#endif
-
- data.push_back((crc & 0xFF00) >> 8);
- data.push_back((crc & 0x00FF));
- }
-
- size_t Available() {
- return data.size() - written;
- }
-
- int Write(uint8_t *write_data, size_t len, int *cont_apptype) {
- size_t written_now = std::min(len, Available());
-
- // fill up remaining bytes with zero padding
- memcpy(write_data, &data[written], written_now);
- memset(write_data + written_now, 0x00, len - written_now);
-
- // set app type depending on progress
- int apptype = written > 0 ? apptype_cont : apptype_start;
-
- written += written_now;
-
- // prevent continuation of a different DG having the same type
- if (cont_apptype)
- *cont_apptype = Available() > 0 ? apptype_cont : -1;
-
- return apptype;
- }
-};
-
-#define SHORT_PAD 6 // F-PAD + 1x CI + 1x 3 bytes data sub-field
-#define VARSIZE_PAD_MIN 8 // F-PAD + 1x CI + end marker + 1x 4 bytes data sub-field
-#define VARSIZE_PAD_MAX 196 // F-PAD + 4x CI + 4x 48 bytes data sub-field
-#define ALLOWED_PADLEN "6 (short X-PAD), 8 to 196 (variable size X-PAD)"
-
-
-
-// MOT Slideshow related
-static int cindex_header = 0;
-static int cindex_body = 0;
-
-
-class MOTHeader {
-private:
- size_t header_size;
- uint8_vector_t data;
-
- void IncrementHeaderSize(size_t size);
- void AddParamHeader(int pli, int param_id) {data.push_back((pli << 6) | (param_id & 0x3F));}
-public:
- MOTHeader(size_t body_size, int content_type, int content_subtype);
-
- void AddExtension(int param_id);
- void AddExtension8Bit(int param_id, uint8_t data_field);
- void AddExtension32Bit(int param_id, uint32_t data_field);
- void AddExtensionVarSize(int param_id, const uint8_t* data_field, size_t data_field_len);
- const uint8_vector_t GetData() {return data;}
-};
-
-MOTHeader::MOTHeader(size_t body_size, int content_type, int content_subtype)
-: header_size(0), data(uint8_vector_t(7, 0x00)) {
- // init header core
-
- // body size
- data[0] = (body_size >> 20) & 0xFF;
- data[1] = (body_size >> 12) & 0xFF;
- data[2] = (body_size >> 4) & 0xFF;
- data[3] = (body_size << 4) & 0xF0;
-
- // header size
- IncrementHeaderSize(data.size());
-
- // content type
- data[5] |= (content_type << 1) & 0x7E;
-
- // content subtype
- data[5] |= (content_subtype >> 8) & 0x01;
- data[6] |= content_subtype & 0xFF;
-}
-
-void MOTHeader::IncrementHeaderSize(size_t size) {
- header_size += size;
-
- data[3] &= 0xF0;
- data[3] |= (header_size >> 9) & 0x0F;
-
- data[4] = (header_size >> 1) & 0xFF;
-
- data[5] &= 0x7F;
- data[5] |= (header_size << 7) & 0x80;
-}
-
-void MOTHeader::AddExtension(int param_id) {
- AddParamHeader(0b00, param_id);
-
- IncrementHeaderSize(1);
-}
-
-void MOTHeader::AddExtension8Bit(int param_id, uint8_t data_field) {
- AddParamHeader(0b01, param_id);
- data.push_back(data_field);
-
- IncrementHeaderSize(2);
-}
-
-void MOTHeader::AddExtension32Bit(int param_id, uint32_t data_field) {
- AddParamHeader(0b10, param_id);
- data.push_back((data_field >> 24) & 0xFF);
- data.push_back((data_field >> 16) & 0xFF);
- data.push_back((data_field >> 8) & 0xFF);
- data.push_back( data_field & 0xFF);
-
- IncrementHeaderSize(5);
-}
-
-void MOTHeader::AddExtensionVarSize(int param_id, const uint8_t* data_field, size_t data_field_len) {
- AddParamHeader(0b11, param_id);
-
- // longer field lens use 15 instead of 7 bits
- bool ext = data_field_len > 127;
- if (ext) {
- data.push_back(0x80 | ((data_field_len >> 8) & 0x7F));
- data.push_back(data_field_len & 0xFF);
- } else {
- data.push_back(data_field_len & 0x7F);
- }
-
- for (size_t i = 0; i < data_field_len; i++)
- data.push_back(data_field[i]);
-
- IncrementHeaderSize(1 + (ext ? 2 : 1) + data_field_len);
-}
-
-
-
-// DLS related
-#define FPAD_LEN 2
-#define DLS_SEG_LEN_PREFIX 2
-#define DLS_SEG_LEN_CHAR_MAX 16
-#define DLS_CMD_REMOVE_LABEL 0b0001
-#define DLS_CMD_DL_PLUS 0b0010
-#define DL_PLUS_CMD_TAGS 0b0000
-
-#define DL_PARAMS_OPEN "##### parameters { #####"
-#define DL_PARAMS_CLOSE "##### parameters } #####"
-
-CharsetConverter charset_converter;
-
-struct DL_PLUS_TAG {
- int content_type;
- int start_marker;
- int length_marker;
-
- DL_PLUS_TAG() :
- content_type(0), // = DUMMY
- start_marker(0),
- length_marker(0)
- {}
-
- DL_PLUS_TAG(int content_type, int start_marker, int length_marker) :
- content_type(content_type),
- start_marker(start_marker),
- length_marker(length_marker)
- {}
-
- bool operator==(const DL_PLUS_TAG& other) const {
- return
- content_type == other.content_type &&
- start_marker == other.start_marker &&
- length_marker == other.length_marker;
- }
- bool operator!=(const DL_PLUS_TAG& other) const {
- return !(*this == other);
- }
-};
-
-typedef std::vector<DL_PLUS_TAG> dl_plus_tags_t;
-
-struct DL_STATE {
- std::string dl_text;
-
- bool dl_plus_enabled;
- bool dl_plus_item_toggle;
- bool dl_plus_item_running;
- dl_plus_tags_t dl_plus_tags;
-
- DL_STATE() :
- dl_plus_enabled(false),
- dl_plus_item_toggle(false),
- dl_plus_item_running(false)
- {}
-
- bool operator==(const DL_STATE& other) const {
- if (dl_text != other.dl_text)
- return false;
- if (dl_plus_enabled != other.dl_plus_enabled)
- return false;
- if (dl_plus_enabled) {
- if (dl_plus_item_toggle != other.dl_plus_item_toggle)
- return false;
- if (dl_plus_item_running != other.dl_plus_item_running)
- return false;
- if (dl_plus_tags != other.dl_plus_tags)
- return false;
- }
- return true;
- }
- bool operator!=(const DL_STATE& other) const {
- return !(*this == other);
- }
-};
-
-
-typedef uint8_vector_t pad_t;
-static bool dls_toggle = false;
-static DL_STATE dl_state_prev;
-
-
-class PADPacketizer {
-private:
- static const size_t subfield_lens[];
-
- const size_t xpad_size_max;
- const bool short_xpad;
- const size_t max_cis;
-
- size_t xpad_size;
- uint8_t subfields[4*48];
- size_t subfields_size;
-
- // PAD w/ CI list
- int ci_type[4];
- size_t ci_len_index[4];
- size_t used_cis;
-
- // PAD w/o CI list
- int last_ci_type;
- size_t last_ci_size;
-
- size_t AddCINeededBytes();
- void AddCI(int apptype, int len_index);
-
- int OptimalSubFieldSizeIndex(size_t available_bytes);
- int WriteDGToSubField(DATA_GROUP* dg, size_t len);
-
- bool AppendDG(DATA_GROUP* dg);
- void AppendDGWithCI(DATA_GROUP* dg);
- void AppendDGWithoutCI(DATA_GROUP* dg);
-
- void ResetPAD();
- pad_t* FlushPAD();
-public:
- std::deque<DATA_GROUP*> queue;
-
- PADPacketizer(size_t pad_size);
- ~PADPacketizer();
-
- pad_t* GetPAD();
-
- // will be removed, when pull (instead of push) approach is implemented!
- void WriteAllPADs(int output_fd);
-};
-
-
-const size_t PADPacketizer::subfield_lens[] = {4, 6, 8, 12, 16, 24, 32, 48};
-
-PADPacketizer::PADPacketizer(size_t pad_size) :
- xpad_size_max(pad_size - FPAD_LEN),
- short_xpad(pad_size == SHORT_PAD),
- max_cis(short_xpad ? 1 : 4),
- last_ci_type(-1)
-{
- ResetPAD();
-}
-
-PADPacketizer::~PADPacketizer() {
- while (!queue.empty()) {
- delete queue.front();
- queue.pop_front();
- }
-}
-
-
-pad_t* PADPacketizer::GetPAD() {
- bool pad_flushable = false;
-
- // process DG queue
- while (!pad_flushable && !queue.empty()) {
- DATA_GROUP* dg = queue.front();
-
- // repeatedly append DG
- while (!pad_flushable && dg->Available() > 0)
- pad_flushable = AppendDG(dg);
-
- if (dg->Available() == 0) {
- delete dg;
- queue.pop_front();
- }
- }
-
- // (possibly empty) PAD
- return FlushPAD();
-}
-
-void PADPacketizer::WriteAllPADs(int output_fd) {
- for (;;) {
- pad_t* pad = GetPAD();
-
- // if only F-PAD present, abort
- if (pad->back() == FPAD_LEN) {
- delete pad;
- break;
- }
-
- if (write(output_fd, &(*pad)[0], pad->size()) != (signed) pad->size())
- fprintf(stderr, "mot-encoder Error: Could not write PAD\n");
-
- delete pad;
- }
-}
-
-
-size_t PADPacketizer::AddCINeededBytes() {
- // returns the amount of additional bytes needed for the next CI
-
- // special cases: end marker added/replaced
- if (!short_xpad && used_cis == 0)
- return 2;
- if (!short_xpad && used_cis == (max_cis - 1))
- return 0;
- return 1;
-}
-
-void PADPacketizer::AddCI(int apptype, int len_index) {
- ci_type[used_cis] = apptype;
- ci_len_index[used_cis] = len_index;
-
- xpad_size += AddCINeededBytes();
- used_cis++;
-}
-
-
-int PADPacketizer::OptimalSubFieldSizeIndex(size_t available_bytes) {
- /*! Return the index of the optimal sub-field size by stepwise search (regards only Variable Size X-PAD):
- * - find the smallest sub-field able to hold (at least) all available bytes
- * - find the biggest regarding sub-field we have space for (which definitely exists - otherwise previously the PAD would have been flushed)
- * - if the wasted space is at least as big as the smallest possible sub-field, use a sub-field one size smaller
- */
- int len_index = 0;
-
- while ((len_index + 1) < 8 && subfield_lens[len_index] < available_bytes)
- len_index++;
- while ((len_index - 1) >= 0 && (subfield_lens[len_index] + AddCINeededBytes()) > (xpad_size_max - xpad_size))
- len_index--;
- if ((len_index - 1) >= 0 && ((int) subfield_lens[len_index] - (int) available_bytes) >= (int) subfield_lens[0])
- len_index--;
-
- return len_index;
-}
-
-int PADPacketizer::WriteDGToSubField(DATA_GROUP* dg, size_t len) {
- int apptype = dg->Write(&subfields[subfields_size], len, &last_ci_type);
- subfields_size += len;
- xpad_size += len;
- return apptype;
-}
-
-
-bool PADPacketizer::AppendDG(DATA_GROUP* dg) {
- /*! use X-PAD w/o CIs instead of X-PAD w/ CIs, if we can save some bytes or at least do not waste additional bytes
- *
- * Omit CI list in case:
- * 1. no pending data sub-fields
- * 2. last CI type valid
- * 3. last CI type matching current (continuity) CI type
- * 4a. short X-PAD; OR
- * 4ba. size of the last X-PAD being at least as big as the available X-PAD payload in case all CIs are used AND
- * 4bb. the amount of available DG bytes being at least as big as the size of the last X-PAD in case all CIs are used
- */
- if (
- used_cis == 0 &&
- last_ci_type != -1 &&
- last_ci_type == dg->apptype_cont &&
- (short_xpad ||
- (last_ci_size >= (xpad_size_max - max_cis) &&
- dg->Available() >= (last_ci_size - max_cis)))
- ) {
- AppendDGWithoutCI(dg);
- return true;
- } else {
- AppendDGWithCI(dg);
-
- // if no further sub-fields could be added, PAD must be flushed
- if (used_cis == max_cis || subfield_lens[0] + AddCINeededBytes() > (xpad_size_max - xpad_size))
- return true;
- }
- return false;
-}
-
-
-void PADPacketizer::AppendDGWithCI(DATA_GROUP* dg) {
- int len_index = short_xpad ? 0 : OptimalSubFieldSizeIndex(dg->Available());
- size_t len_size = short_xpad ? 3 : subfield_lens[len_index];
-
- int apptype = WriteDGToSubField(dg, len_size);
- AddCI(apptype, len_index);
-
-#if DEBUG
- fprintf(stderr, "PADPacketizer: added sub-field w/ CI - type: %2d, size: %2zu\n", apptype, len_size);
-#endif
-}
-
-void PADPacketizer::AppendDGWithoutCI(DATA_GROUP* dg) {
-#if DEBUG
- int old_last_ci_type = last_ci_type;
-#endif
-
- WriteDGToSubField(dg, last_ci_size);
-
-#if DEBUG
- fprintf(stderr, "PADPacketizer: added sub-field w/o CI - type: %2d, size: %2zu\n", old_last_ci_type, last_ci_size);
-#endif
-}
-
-void PADPacketizer::ResetPAD() {
- xpad_size = 0;
- subfields_size = 0;
- used_cis = 0;
-}
-
-pad_t* PADPacketizer::FlushPAD() {
- pad_t* result = new pad_t(xpad_size_max + FPAD_LEN + 1);
- pad_t &pad = *result;
-
- size_t pad_offset = xpad_size_max;
-
- if (subfields_size > 0) {
- if (used_cis > 0) {
- // X-PAD: CIs
- for (size_t i = 0; i < used_cis; i++)
- pad[--pad_offset] = (short_xpad ? 0 : ci_len_index[i]) << 5 | ci_type[i];
-
- // X-PAD: end marker (if needed)
- if (used_cis < max_cis)
- pad[--pad_offset] = 0x00;
- }
-
- // X-PAD: sub-fields (reversed on-the-fly)
- for (size_t off = 0; off < subfields_size; off++)
- pad[--pad_offset] = subfields[off];
- } else {
- // no X-PAD
- last_ci_type = -1;
- }
-
- // zero padding
- memset(&pad[0], 0x00, pad_offset);
-
- // F-PAD
- pad[xpad_size_max + 0] = subfields_size > 0 ? (short_xpad ? 0x10 : 0x20) : 0x00;
- pad[xpad_size_max + 1] = subfields_size > 0 ? (used_cis > 0 ? 0x02 : 0x00) : 0x00;
-
- // used PAD len
- pad[xpad_size_max + FPAD_LEN] = xpad_size + FPAD_LEN;
-
- last_ci_size = xpad_size;
- ResetPAD();
- return result;
-}
-
-
-static PADPacketizer *pad_packetizer;
-
-
-
-
-
-
-
-void usage(char* name)
-{
- fprintf(stderr, "DAB MOT encoder %s for slideshow and DLS\n\n"
- "By CSP Innovazione nelle ICT s.c.a r.l. (http://rd.csp.it/) and\n"
- "Opendigitalradio.org\n\n"
- "Reads image data from the specified directory, DLS text from a file,\n"
- "and outputs PAD data to the given FIFO.\n"
- " http://opendigitalradio.org\n\n",
-#if defined(GITVERSION)
- GITVERSION
-#else
- PACKAGE_VERSION
-#endif
- );
- fprintf(stderr, "Usage: %s [OPTIONS...]\n", name);
- fprintf(stderr, " -d, --dir=DIRNAME Directory to read images from.\n"
- " -e, --erase Erase slides from DIRNAME once they have\n"
- " been encoded.\n"
- " -s, --sleep=DELAY Wait DELAY seconds between each slide\n"
- " Default: " STR(SLEEPDELAY_DEFAULT) "\n"
- " -o, --output=FILENAME Fifo to write PAD data into.\n"
- " Default: /tmp/pad.fifo\n"
- " -t, --dls=FILENAME Fifo or file to read DLS text from.\n"
- " -p, --pad=LENGTH Set the pad length.\n"
- " Possible values: " ALLOWED_PADLEN "\n"
- " Default: 58\n"
- " -c, --charset=ID ID of the character set encoding used for DLS text input.\n"
- " ID = 0: Complete EBU Latin based repertoire\n"
- " ID = 6: ISO/IEC 10646 using UCS-2 BE\n"
- " ID = 15: ISO/IEC 10646 using UTF-8\n"
- " Default: 15\n"
- " -r, --remove-dls Always insert a DLS Remove Label command when replacing a DLS text.\n"
- " -C, --raw-dls Do not convert DLS texts to Complete EBU Latin based repertoire\n"
- " character set encoding.\n"
- " -R, --raw-slides Do not process slides. Integrity checks and resizing\n"
- " slides is skipped. Use this if you know what you are doing !\n"
- " It is useful only when -d is used\n"
- " -v, --verbose Print more information to the console\n"
- );
-}
-
-#define no_argument 0
-#define required_argument 1
-#define optional_argument 2
-int main(int argc, char *argv[])
-{
- int ret;
- struct dirent *pDirent;
- int padlen = 58;
- bool erase_after_tx = false;
- int sleepdelay = SLEEPDELAY_DEFAULT;
- bool raw_slides = false;
- int charset = CHARSET_UTF8;
- bool raw_dls = false;
- bool remove_dls = false;
-
- const char* sls_dir = NULL;
- const char* output = "/tmp/pad.fifo";
- std::string dls_file;
-
- const struct option longopts[] = {
- {"charset", required_argument, 0, 'c'},
- {"raw-dls", no_argument, 0, 'C'},
- {"remove-dls", no_argument, 0, 'r'},
- {"dir", required_argument, 0, 'd'},
- {"erase", no_argument, 0, 'e'},
- {"output", required_argument, 0, 'o'},
- {"dls", required_argument, 0, 't'},
- {"pad", required_argument, 0, 'p'},
- {"sleep", required_argument, 0, 's'},
- {"raw-slides", no_argument, 0, 'R'},
- {"help", no_argument, 0, 'h'},
- {"verbose", no_argument, 0, 'v'},
- {0,0,0,0},
- };
-
- int ch=0;
- int index;
- while(ch != -1) {
- ch = getopt_long(argc, argv, "eChRrc:d:o:p:s:t:v", longopts, &index);
- switch (ch) {
- case 'c':
- charset = atoi(optarg);
- break;
- case 'C':
- raw_dls = true;
- break;
- case 'r':
- remove_dls = true;
- break;
- case 'd':
- sls_dir = optarg;
- break;
- case 'e':
- erase_after_tx = true;
- break;
- case 'o':
- output = optarg;
- break;
- case 's':
- sleepdelay = atoi(optarg);
- break;
- case 't':
- dls_file = optarg;
- break;
- case 'p':
- padlen = atoi(optarg);
- break;
- case 'R':
- raw_slides = true;
- break;
- case 'v':
- verbose++;
- break;
- case '?':
- case 'h':
- usage(argv[0]);
- return 0;
- }
- }
-
- if (padlen != SHORT_PAD && (padlen < VARSIZE_PAD_MIN || padlen > VARSIZE_PAD_MAX)) {
- fprintf(stderr, "mot-encoder Error: pad length %d invalid: Possible values: "
- ALLOWED_PADLEN "\n",
- padlen);
- return 2;
- }
-
- if (sls_dir && not dls_file.empty()) {
- fprintf(stderr, "mot-encoder encoding Slideshow from '%s' and DLS from '%s' to '%s'\n",
- sls_dir, dls_file.c_str(), output);
- }
- else if (sls_dir) {
- fprintf(stderr, "mot-encoder encoding Slideshow from '%s' to '%s'. No DLS.\n",
- sls_dir, output);
- }
- else if (not dls_file.empty()) {
- fprintf(stderr, "mot-encoder encoding DLS from '%s' to '%s'. No Slideshow.\n",
- dls_file.c_str(), output);
- }
- else {
- fprintf(stderr, "mot-encoder Error: No DLS nor slideshow to encode !\n");
- usage(argv[0]);
- return 1;
- }
-
- const char* user_charset;
- switch (charset) {
- case CHARSET_COMPLETE_EBU_LATIN:
- user_charset = "Complete EBU Latin";
- break;
- case CHARSET_EBU_LATIN_CY_GR:
- user_charset = "EBU Latin core, Cyrillic, Greek";
- break;
- case CHARSET_EBU_LATIN_AR_HE_CY_GR:
- user_charset = "EBU Latin core, Arabic, Hebrew, Cyrillic, Greek";
- break;
- case CHARSET_ISO_LATIN_ALPHABET_2:
- user_charset = "ISO Latin Alphabet 2";
- break;
- case CHARSET_UCS2_BE:
- user_charset = "UCS-2 BE";
- break;
- case CHARSET_UTF8:
- user_charset = "UTF-8";
- break;
- default:
- user_charset = "Invalid";
- charset = -1;
- break;
- }
-
- if (charset == -1) {
- fprintf(stderr, "mot-encoder Error: Invalid charset!\n");
- usage(argv[0]);
- return 1;
- }
- else {
- fprintf(stderr, "mot-encoder using charset %s (%d)\n",
- user_charset, charset);
- }
-
- if (not raw_dls) {
- switch (charset) {
- case CHARSET_COMPLETE_EBU_LATIN:
- // no conversion needed
- break;
- case CHARSET_UTF8:
- fprintf(stderr, "mot-encoder converting DLS texts to Complete EBU Latin\n");
- break;
- default:
- fprintf(stderr, "mot-encoder Error: DLS conversion to EBU is currently only supported for UTF-8 input!\n");
- return 1;
- }
- }
-
- int output_fd = open(output, O_WRONLY);
- if (output_fd == -1) {
- perror("mot-encoder Error: failed to open output");
- return 3;
- }
-
-#if HAVE_MAGICKWAND
- MagickWandGenesis();
-#endif
-
- pad_packetizer = new PADPacketizer(padlen);
-
- std::list<slide_metadata_t> slides_to_transmit;
- History slides_history(MAXHISTORYLEN);
-
- while(1) {
- if (sls_dir) { // slide + possibly DLS
- DIR *pDir = opendir(sls_dir);
- if (pDir == NULL) {
- fprintf(stderr, "mot-encoder Error: cannot open directory '%s'\n", sls_dir);
- return 1;
- }
-
- // Add new slides to transmit to list
- while ((pDirent = readdir(pDir)) != NULL) {
- if (pDirent->d_name[0] != '.') {
- char imagepath[256];
- sprintf(imagepath, "%s/%s", sls_dir, pDirent->d_name);
-
- slide_metadata_t md;
- md.filepath = imagepath;
- md.fidx = slides_history.get_fidx(imagepath);
- slides_to_transmit.push_back(md);
-
- if (verbose) {
- fprintf(stderr, "mot-encoder found slide '%s', fidx %d\n", imagepath, md.fidx);
- }
- }
- }
-
- closedir(pDir);
-
-#if DEBUG
- slides_history.disp_database();
-#endif
-
- // if ATM no slides, transmit at least DLS
- if (slides_to_transmit.empty()) {
- if (not dls_file.empty())
- writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
- sleep(sleepdelay);
- } else {
- // Sort the list in fidx order
- slides_to_transmit.sort();
-
- // Encode the slides
- for (std::list<slide_metadata_t>::const_iterator it = slides_to_transmit.cbegin();
- it != slides_to_transmit.cend();
- ++it) {
-
- ret = encodeFile(output_fd, it->filepath, it->fidx, raw_slides);
- if (ret != 1)
- fprintf(stderr, "mot-encoder Error: cannot encode file '%s'\n", it->filepath.c_str());
-
- if (erase_after_tx) {
- if (unlink(it->filepath.c_str()) == -1) {
- fprintf(stderr, "mot-encoder Error: erasing file '%s' failed: ", it->filepath.c_str());
- perror("");
- }
- }
-
- // Always retransmit DLS after each slide, we want it to be updated frequently
- if (not dls_file.empty())
- writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
- sleep(sleepdelay);
- }
-
- slides_to_transmit.resize(0);
- }
- } else { // only DLS
- // Always retransmit DLS, we want it to be updated frequently
- writeDLS(output_fd, dls_file, charset, raw_dls, remove_dls);
-
- sleep(sleepdelay);
- }
- }
-
- delete pad_packetizer;
-
- return 1;
-}
-
-
-DATA_GROUP* createDataGroupLengthIndicator(size_t len) {
- DATA_GROUP* dg = new DATA_GROUP(2, 1, 1); // continuation never used (except for comparison at short X-PAD)
- uint8_vector_t &data = dg->data;
-
- // Data Group length
- data[0] = (len & 0x3F00) >> 8;
- data[1] = (len & 0x00FF);
-
- // CRC
- dg->AppendCRC();
-
- return dg;
-}
-
-
-void warnOnSmallerImage(size_t height, size_t width, const std::string& fname) {
- if (height < 240 || width < 320)
- fprintf(stderr, "mot-encoder Warning: Image '%s' smaller than recommended size (%zu x %zu < 320 x 240 px)\n", fname.c_str(), width, height);
-}
-
-
-/*! Scales the image down if needed,
- * so that it is 320x240 pixels.
- * Automatically reduces the quality to make sure the
- * blobsize is not too large.
- *
- * \return the blobsize
- */
-#if HAVE_MAGICKWAND
-size_t resizeImage(MagickWand* m_wand, unsigned char** blob, const std::string& fname, bool* jfif_not_png)
-{
- unsigned char* blob_png;
- unsigned char* blob_jpg;
- size_t blobsize_png;
- size_t blobsize_jpg;
-
- size_t height = MagickGetImageHeight(m_wand);
- size_t width = MagickGetImageWidth(m_wand);
-
- while (height > 240 || width > 320) {
- if (height/240.0 > width/320.0) {
- width = width * 240.0 / height;
- height = 240;
- }
- else {
- height = height * 320.0 / width;
- width = 320;
- }
- MagickResizeImage(m_wand, width, height, LanczosFilter, 1);
- }
-
- height = MagickGetImageHeight(m_wand);
- width = MagickGetImageWidth(m_wand);
-
- // try PNG (zlib level 9 / possibly adaptive filtering)
- MagickSetImageFormat(m_wand, "png");
- MagickSetImageCompressionQuality(m_wand, 95);
- blob_png = MagickGetImageBlob(m_wand, &blobsize_png);
-
- // try JPG
- MagickSetImageFormat(m_wand, "jpg");
-
- blob_jpg = NULL;
- int quality_jpg = 100;
- do {
- free(blob_jpg);
- quality_jpg -= 5;
-
- MagickSetImageCompressionQuality(m_wand, quality_jpg);
- blob_jpg = MagickGetImageBlob(m_wand, &blobsize_jpg);
- } while (blobsize_jpg > MAXSLIDESIZE && quality_jpg > MINQUALITY);
-
-
- // check for max size
- if (blobsize_png > MAXSLIDESIZE && blobsize_jpg > MAXSLIDESIZE) {
- fprintf(stderr, "mot-encoder: Image Size too large after compression: %zu bytes (PNG), %zu bytes (JPEG)\n",
- blobsize_png, blobsize_jpg);
- free(blob_png);
- free(blob_jpg);
- return 0;
- }
-
- // choose the smaller one (at least one does not exceed the max size)
- *jfif_not_png = blobsize_jpg < blobsize_png;
-
- if (verbose) {
- if (*jfif_not_png)
- fprintf(stderr, "mot-encoder resized image to %zu x %zu. Size after compression %zu bytes (JPEG, q=%d; PNG was %zu bytes)\n",
- width, height, blobsize_jpg, quality_jpg, blobsize_png);
- else
- fprintf(stderr, "mot-encoder resized image to %zu x %zu. Size after compression %zu bytes (PNG; JPEG was %zu bytes)\n",
- width, height, blobsize_png, blobsize_jpg);
- }
-
- // warn if resized image smaller than default dimension
- warnOnSmallerImage(height, width, fname);
-
- free(*jfif_not_png ? blob_png : blob_jpg);
- *blob = *jfif_not_png ? blob_jpg : blob_png;
- return *jfif_not_png ? blobsize_jpg : blobsize_png;
-}
-#endif
-
-int encodeFile(int output_fd, const std::string& fname, int fidx, bool raw_slides)
-{
- int ret = 0;
- int nseg, lastseglen, i, last, curseglen;
-#if HAVE_MAGICKWAND
- MagickWand *m_wand = NULL;
- MagickBooleanType err;
-#endif
- size_t blobsize, height, width;
- bool jpeg_progr;
- unsigned char *blob = NULL;
- unsigned char *curseg = NULL;
- MSCDG msc;
- DATA_GROUP* dgli;
- DATA_GROUP* mscdg;
-
- size_t orig_quality;
- char* orig_format = NULL;
- /*! We handle JPEG differently, because we want to avoid recompressing the
- * image if it is suitable as is
- */
- bool orig_is_jpeg = false;
-
- /*! If the original is a PNG, we transmit it as is, if the resolution is correct
- * and the file is not too large. Otherwise it gets resized and sent as JPEG.
- */
- bool orig_is_png = false;
-
- /*! By default, we do resize the image to 320x240, with a quality such that
- * the blobsize is at most MAXSLIDESIZE.
- *
- * For JPEG input files that are already at the right resolution and at the
- * right blobsize, we disable this to avoid quality loss due to recompression
- *
- * As device support of this feature is optional, we furthermore require JPEG input
- * files to not have progressive coding.
- */
- bool resize_required = true;
-
- bool jfif_not_png = true;
-
- if (!raw_slides) {
-#if HAVE_MAGICKWAND
-
- m_wand = NewMagickWand();
-
- err = MagickReadImage(m_wand, fname.c_str());
- if (err == MagickFalse) {
- fprintf(stderr, "mot-encoder Error: Unable to load image '%s'\n",
- fname.c_str());
-
- goto encodefile_out;
- }
-
- height = MagickGetImageHeight(m_wand);
- width = MagickGetImageWidth(m_wand);
- orig_format = MagickGetImageFormat(m_wand);
- jpeg_progr = MagickGetImageInterlaceScheme(m_wand) == JPEGInterlace;
-
- // By default assume that the image has full quality and can be reduced
- orig_quality = 100;
-
- // strip unneeded information (profiles, meta data)
- MagickStripImage(m_wand);
-
- if (orig_format) {
- if (strcmp(orig_format, "JPEG") == 0) {
- orig_quality = MagickGetImageCompressionQuality(m_wand);
- orig_is_jpeg = true;
-
- if (verbose) {
- fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
- " Original size: %zu x %zu. (%s, q=%zu, progr=%s)\n",
- fname.c_str(), fidx, width, height, orig_format, orig_quality, jpeg_progr ? "y" : "n");
- }
- }
- else if (strcmp(orig_format, "PNG") == 0) {
- orig_is_png = true;
- jfif_not_png = false;
-
- if (verbose) {
- fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
- " Original size: %zu x %zu. (%s)\n",
- fname.c_str(), fidx, width, height, orig_format);
- }
- }
- else if (verbose) {
- fprintf(stderr, "mot-encoder image: '%s' (id=%d)."
- " Original size: %zu x %zu. (%s)\n",
- fname.c_str(), fidx, width, height, orig_format);
- }
-
- free(orig_format);
- }
- else {
- fprintf(stderr, "mot-encoder Warning: Unable to detect image format of '%s'\n",
- fname.c_str());
-
- fprintf(stderr, "mot-encoder image: '%s' (id=%d). Original size: %zu x %zu.\n",
- fname.c_str(), fidx, width, height);
- }
-
- if ((orig_is_jpeg || orig_is_png) && height <= 240 && width <= 320 && not jpeg_progr) {
- // Don't recompress the image and check if the blobsize is suitable
- blob = MagickGetImageBlob(m_wand, &blobsize);
-
- if (blobsize <= MAXSLIDESIZE) {
- if (verbose) {
- fprintf(stderr, "mot-encoder image: '%s' (id=%d). No resize needed: %zu Bytes\n",
- fname.c_str(), fidx, blobsize);
- }
- resize_required = false;
- }
- }
-
- if (resize_required) {
- blobsize = resizeImage(m_wand, &blob, fname, &jfif_not_png);
- }
- else {
- // warn if unresized image smaller than default dimension
- warnOnSmallerImage(height, width, fname);
- }
-
-#else
- fprintf(stderr, "mot-encoder has not been compiled with MagickWand, only RAW slides are supported!\n");
- ret = -1;
- goto encodefile_out;
-#endif
- }
- else { // Use RAW data, it might not even be a jpg !
- // read file
- FILE* pFile = fopen(fname.c_str(), "rb");
- if (pFile == NULL) {
- fprintf(stderr, "mot-encoder Error: Unable to load file '%s'\n",
- fname.c_str());
- goto encodefile_out;
- }
-
- // obtain file size:
- fseek(pFile, 0, SEEK_END);
- blobsize = ftell(pFile);
- rewind(pFile);
-
- if (blobsize > MAXSLIDESIZE) {
- fprintf(stderr, "mot-encoder Warning: blob in raw-slide '%s' too large\n",
- fname.c_str());
- }
-
- // allocate memory to contain the whole file:
- blob = (unsigned char*)malloc(sizeof(char) * blobsize);
- if (blob == NULL) {
- fprintf(stderr, "mot-encoder Error: Memory allocation error\n");
- goto encodefile_out;
- }
-
- // copy the file into the buffer:
- if (fread(blob, blobsize, 1, pFile) != 1) {
- fprintf(stderr, "mot-encoder Error: Could not read file\n");
- goto encodefile_out;
- }
-
- size_t last_dot = fname.rfind(".");
-
- // default:
- jfif_not_png = true; // This is how we did it in the past.
- // It's wrong anyway, so we're at least compatible
-
- if (last_dot != std::string::npos) {
- std::string file_extension = fname.substr(last_dot, std::string::npos);
-
- std::transform(file_extension.begin(), file_extension.end(), file_extension.begin(), ::tolower);
-
- if (file_extension == ".png") {
- jfif_not_png = false;
- }
- }
-
- if (pFile != NULL) {
- fclose(pFile);
- }
- }
-
- if (blobsize) {
- nseg = blobsize / MAXSEGLEN;
- lastseglen = blobsize % MAXSEGLEN;
- if (lastseglen != 0) {
- nseg++;
- }
-
- uint8_vector_t mothdr = createMotHeader(blobsize, fidx, jfif_not_png);
- // Create the MSC Data Group C-Structure
- createMscDG(&msc, 3, &cindex_header, 0, 1, fidx, &mothdr[0], mothdr.size());
- // Generate the MSC DG frame (Figure 9 en 300 401)
- mscdg = packMscDG(&msc);
- dgli = createDataGroupLengthIndicator(mscdg->data.size());
-
- pad_packetizer->queue.push_back(dgli);
- pad_packetizer->queue.push_back(mscdg);
-
- for (i = 0; i < nseg; i++) {
- curseg = blob + i * MAXSEGLEN;
- if (i == nseg-1) {
- curseglen = lastseglen;
- last = 1;
- }
- else {
- curseglen = MAXSEGLEN;
- last = 0;
- }
-
- createMscDG(&msc, 4, &cindex_body, i, last, fidx, curseg, curseglen);
- mscdg = packMscDG(&msc);
- dgli = createDataGroupLengthIndicator(mscdg->data.size());
-
- pad_packetizer->queue.push_back(dgli);
- pad_packetizer->queue.push_back(mscdg);
- }
-
- pad_packetizer->WriteAllPADs(output_fd);
-
- ret = 1;
- }
-
-encodefile_out:
-#if HAVE_MAGICKWAND
- if (m_wand) {
- m_wand = DestroyMagickWand(m_wand);
- }
-#endif
-
- if (blob) {
- free(blob);
- }
- return ret;
-}
-
-
-uint8_vector_t createMotHeader(size_t blobsize, int fidx, bool jfif_not_png)
-{
- // prepare ContentName
- uint8_t cntemp[10]; // = 1 + 8 + 1 = charset + name + terminator
- cntemp[0] = 0x0 << 4; // charset: 0 (Complete EBU Latin based) - doesn't really matter here
- snprintf((char*) (cntemp + 1), sizeof(cntemp) - 1, "%04d.%s", fidx, jfif_not_png ? "jpg" : "png");
- if (verbose)
- fprintf(stderr, "mot-encoder writing image as '%s'\n", cntemp + 1);
-
- // MOT header - content type: image, content subtype: JFIF / PNG
- MOTHeader header(blobsize, 0x02, jfif_not_png ? 0x001 : 0x003);
-
- // TriggerTime: NOW
- header.AddExtension32Bit(0x05, 0x00000000);
-
- // ContentName: XXXX.jpg / XXXX.png
- header.AddExtensionVarSize(0x0C, cntemp, sizeof(cntemp) - 1); // omit terminator
-
- return header.GetData();
-}
-
-
-void createMscDG(MSCDG* msc, unsigned short int dgtype,
- int *cindex, unsigned short int segnum, unsigned short int lastseg,
- unsigned short int tid, unsigned char* data,
- unsigned short int datalen)
-{
- msc->extflag = 0;
- msc->crcflag = 1;
- msc->segflag = 1;
- msc->accflag = 1;
- msc->dgtype = dgtype;
- msc->cindex = *cindex;
- msc->rindex = 0;
- msc->last = lastseg;
- msc->segnum = segnum;
- msc->rfa = 0;
- msc->tidflag = 1;
- msc->lenid = 2;
- msc->tid = tid;
- msc->segdata = data;
- msc->rcount = 0;
- msc->seglen = datalen;
-
- *cindex = (*cindex + 1) % 16; // increment continuity index
-}
-
-
-DATA_GROUP* packMscDG(MSCDG* msc)
-{
- DATA_GROUP* dg = new DATA_GROUP(9 + msc->seglen, 12, 13);
- uint8_vector_t &b = dg->data;
-
- // headers
- b[0] = (msc->extflag<<7) | (msc->crcflag<<6) | (msc->segflag<<5) |
- (msc->accflag<<4) | msc->dgtype;
-
- b[1] = (msc->cindex<<4) | msc->rindex;
- b[2] = (msc->last<<7) | ((msc->segnum & 0x7F00) >> 8);
- b[3] = msc->segnum & 0x00FF;
- b[4] = 0;
- b[4] = (msc->rfa << 5) | (msc->tidflag << 4) | msc->lenid;
- b[5] = (msc->tid & 0xFF00) >> 8;
- b[6] = msc->tid & 0x00FF;
- b[7] = (msc->rcount << 5) | ((msc->seglen & 0x1F00)>>8);
- b[8] = msc->seglen & 0x00FF;
-
- // data field
- memcpy(&b[9], msc->segdata, msc->seglen);
-
- // CRC
- dg->AppendCRC();
-
- return dg;
-}
-
-
-DATA_GROUP* createDynamicLabelCommand(uint8_t command) {
- DATA_GROUP* dg = new DATA_GROUP(2, 2, 3);
- uint8_vector_t &seg_data = dg->data;
-
- // prefix: toggle? + first seg + last seg + command flag + command
- seg_data[0] =
- (dls_toggle ? (1 << 7) : 0) +
- (1 << 6) +
- (1 << 5) +
- (1 << 4) +
- command;
-
- // prefix: charset (though irrelevant here)
- seg_data[1] = CHARSET_COMPLETE_EBU_LATIN;
-
- // CRC
- dg->AppendCRC();
-
- return dg;
-}
-
-DATA_GROUP* createDynamicLabelPlus(const DL_STATE& dl_state) {
- size_t tags_size = dl_state.dl_plus_tags.size();
- size_t len_dl_plus_cmd_field = 1 + 3 * tags_size;
- DATA_GROUP* dg = new DATA_GROUP(2 + len_dl_plus_cmd_field, 2, 3);
- uint8_vector_t &seg_data = dg->data;
-
- // prefix: toggle? + first seg + last seg + command flag + command
- seg_data[0] =
- (dls_toggle ? (1 << 7) : 0) +
- (1 << 6) +
- (1 << 5) +
- (1 << 4) +
- DLS_CMD_DL_PLUS;
-
- // prefix: link bit + length
- seg_data[1] =
- (dls_toggle ? (1 << 7) : 0) +
- (len_dl_plus_cmd_field - 1); // -1 !
-
- // DL Plus tags command: CId + IT + IR + NT
- seg_data[2] =
- (DL_PLUS_CMD_TAGS << 4) +
- (dl_state.dl_plus_item_toggle ? (1 << 3) : 0) +
- (dl_state.dl_plus_item_running ? (1 << 2) : 0) +
- (tags_size - 1); // -1 !
-
- for (size_t i = 0; i < tags_size; i++) {
- // DL Plus tags command: Content Type + Start Marker + Length Marker
- seg_data[3 + 3 * i] = dl_state.dl_plus_tags[i].content_type & 0x7F;
- seg_data[4 + 3 * i] = dl_state.dl_plus_tags[i].start_marker & 0x7F;
- seg_data[5 + 3 * i] = dl_state.dl_plus_tags[i].length_marker & 0x7F;
- }
-
- // CRC
- dg->AppendCRC();
-
- return dg;
-}
-
-
-std::vector<std::string> split_string(const std::string &s, const char delimiter) {
- std::vector<std::string> result;
- std::stringstream ss(s);
- std::string part;
-
- while (std::getline(ss, part, delimiter))
- result.push_back(part);
- return result;
-}
-
-bool parse_dl_param_bool(const std::string &key, const std::string &value, bool &target) {
- if (value == "0") {
- target = 0;
- return true;
- }
- if (value == "1") {
- target = 1;
- return true;
- }
- fprintf(stderr, "mot-encoder Warning: DL parameter '%s' has unsupported value '%s' - ignored\n", key.c_str(), value.c_str());
- return false;
-}
-
-bool parse_dl_param_int_dl_plus_tag(const std::string &key, const std::string &value, int &target) {
- int value_int = atoi(value.c_str());
- if (value_int >= 0x00 && value_int <= 0x7F) {
- target = value_int;
- return true;
- }
- fprintf(stderr, "mot-encoder Warning: DL Plus tag parameter '%s' %d out of range - ignored\n", key.c_str(), value_int);
- return false;
-}
-
-void parse_dl_params(std::ifstream &dls_fstream, DL_STATE &dl_state) {
- std::string line;
- while (std::getline(dls_fstream, line)) {
- // return on params close
- if (line == DL_PARAMS_CLOSE)
- return;
-
- // ignore empty lines and comments
- if (line.empty() || line[0] == '#')
- continue;
-
- // parse key/value pair
- size_t separator_pos = line.find('=');
- if (separator_pos == std::string::npos) {
- fprintf(stderr, "mot-encoder Warning: DL parameter line '%s' without separator - ignored\n", line.c_str());
- continue;
- }
- std::string key = line.substr(0, separator_pos);
- std::string value = line.substr(separator_pos + 1);
-#if DEBUG
- fprintf(stderr, "parse_dl_params: key: '%s', value: '%s'\n", key.c_str(), value.c_str());
-#endif
-
- if (key == "DL_PLUS") {
- parse_dl_param_bool(key, value, dl_state.dl_plus_enabled);
- continue;
- }
- if (key == "DL_PLUS_ITEM_TOGGLE") {
- parse_dl_param_bool(key, value, dl_state.dl_plus_item_toggle);
- continue;
- }
- if (key == "DL_PLUS_ITEM_RUNNING") {
- parse_dl_param_bool(key, value, dl_state.dl_plus_item_running);
- continue;
- }
- if (key == "DL_PLUS_TAG") {
- if (dl_state.dl_plus_tags.size() == 4) {
- fprintf(stderr, "mot-encoder Warning: DL Plus tag ignored, as already four tags present\n");
- continue;
- }
-
- // split value
- std::vector<std::string> params = split_string(value, ' ');
- if (params.size() != 3) {
- fprintf(stderr, "mot-encoder Warning: DL Plus tag value '%s' does not have three parts - ignored\n", value.c_str());
- continue;
- }
-
- int content_type, start_marker, length_marker;
- if (parse_dl_param_int_dl_plus_tag("content_type", params[0], content_type) &
- parse_dl_param_int_dl_plus_tag("start_marker", params[1], start_marker) &
- parse_dl_param_int_dl_plus_tag("length_marker", params[2], length_marker))
- dl_state.dl_plus_tags.push_back(DL_PLUS_TAG(content_type, start_marker, length_marker));
- continue;
- }
-
- fprintf(stderr, "mot-encoder Warning: DL parameter '%s' unknown - ignored\n", key.c_str());
- }
-
- fprintf(stderr, "mot-encoder Warning: no param closing tag, so the DLS text will be empty\n");
-}
-
-
-void writeDLS(int output_fd, const std::string& dls_file, uint8_t charset, bool raw_dls, bool remove_dls)
-{
- std::ifstream dls_fstream(dls_file);
- if (!dls_fstream.is_open()) {
- std::cerr << "Could not open " << dls_file << std::endl;
- return;
- }
-
- DL_STATE dl_state;
-
- std::vector<std::string> dls_lines;
-
- std::string line;
- // Read and convert lines one by one because the converter doesn't understand
- // line endings
- while (std::getline(dls_fstream, line)) {
- if (line.empty())
- continue;
- if (line == DL_PARAMS_OPEN) {
- parse_dl_params(dls_fstream, dl_state);
- } else {
- if (not raw_dls && charset == CHARSET_UTF8) {
- dls_lines.push_back(charset_converter.convert(line));
- }
- else {
- dls_lines.push_back(line);
- }
- // TODO handle the other charsets accordingly
- }
- }
-
- std::stringstream ss;
- for (size_t i = 0; i < dls_lines.size(); i++) {
- if (i != 0) {
- if (charset == CHARSET_UCS2_BE)
- ss << '\0' << '\n';
- else
- ss << '\n';
- }
-
- // UCS-2 BE: if from file the first byte of \0\n remains, remove it
- if (charset == CHARSET_UCS2_BE && dls_lines[i].size() % 2) {
- dls_lines[i].resize(dls_lines[i].size() - 1);
- }
-
- ss << dls_lines[i];
- }
-
- dl_state.dl_text = ss.str();
- if (dl_state.dl_text.size() > MAXDLS)
- dl_state.dl_text.resize(MAXDLS);
-
-
- // if DL Plus enabled, but no DL Plus tags were added, add the required DUMMY tag
- if (dl_state.dl_plus_enabled && dl_state.dl_plus_tags.empty())
- dl_state.dl_plus_tags.push_back(DL_PLUS_TAG());
-
- if (not raw_dls)
- charset = CHARSET_COMPLETE_EBU_LATIN;
-
-
- // toggle the toggle bit only on new DL state
- bool dl_state_is_new = dl_state != dl_state_prev;
- if (verbose) {
- fprintf(stderr, "mot-encoder writing %s DLS text \"%s\"\n", dl_state_is_new ? "new" : "old", dl_state.dl_text.c_str());
- if (dl_state.dl_plus_enabled) {
- fprintf(
- stderr, "mot-encoder writing %s DL Plus tags (IT/IR: %d/%d): ",
- dl_state_is_new ? "new" : "old",
- dl_state.dl_plus_item_toggle ? 1 : 0,
- dl_state.dl_plus_item_running ? 1 : 0);
- for (dl_plus_tags_t::const_iterator it = dl_state.dl_plus_tags.begin(); it != dl_state.dl_plus_tags.end(); it++) {
- if (it != dl_state.dl_plus_tags.begin())
- fprintf(stderr, ", ");
- fprintf(stderr, "%d (S/L: %d/%d)", it->content_type, it->start_marker, it->length_marker);
- }
- fprintf(stderr, "\n");
- }
- }
-
- DATA_GROUP *remove_label_dg = NULL;
- if (dl_state_is_new) {
- if (remove_dls)
- remove_label_dg = createDynamicLabelCommand(DLS_CMD_REMOVE_LABEL);
-
- dls_toggle = !dls_toggle; // indicate changed text
-
- dl_state_prev = dl_state;
- }
-
- prepend_dl_dgs(dl_state, charset);
- if (remove_label_dg)
- pad_packetizer->queue.push_front(remove_label_dg);
- pad_packetizer->WriteAllPADs(output_fd);
-}
-
-
-int dls_count(const std::string& text) {
- size_t text_len = text.size();
- return text_len / DLS_SEG_LEN_CHAR_MAX + (text_len % DLS_SEG_LEN_CHAR_MAX ? 1 : 0);
-}
-
-
-DATA_GROUP* dls_get(const std::string& text, uint8_t charset, int seg_index) {
- bool first_seg = seg_index == 0;
- bool last_seg = seg_index == dls_count(text) - 1;
-
- int seg_text_offset = seg_index * DLS_SEG_LEN_CHAR_MAX;
- const char *seg_text_start = text.c_str() + seg_text_offset;
- size_t seg_text_len = MIN(text.size() - seg_text_offset, DLS_SEG_LEN_CHAR_MAX);
-
- DATA_GROUP* dg = new DATA_GROUP(DLS_SEG_LEN_PREFIX + seg_text_len, 2, 3);
- uint8_vector_t &seg_data = dg->data;
-
- // prefix: toggle? + first seg? + last seg? + (seg len - 1)
- seg_data[0] =
- (dls_toggle ? (1 << 7) : 0) +
- (first_seg ? (1 << 6) : 0) +
- (last_seg ? (1 << 5) : 0) +
- (seg_text_len - 1);
-
- // prefix: charset / seg index
- seg_data[1] = (first_seg ? charset : seg_index) << 4;
-
- // character field
- memcpy(&seg_data[DLS_SEG_LEN_PREFIX], seg_text_start, seg_text_len);
-
- // CRC
- dg->AppendCRC();
-
-#if DEBUG
- fprintf(stderr, "DL segment:");
- for (int i = 0; i < seg_data.size(); i++)
- fprintf(stderr, " %02x", seg_data[i]);
- fprintf(stderr, "\n");
-#endif
- return dg;
-}
-
-
-void prepend_dl_dgs(const DL_STATE& dl_state, uint8_t charset) {
- // process all DL segments
- int seg_count = dls_count(dl_state.dl_text);
- std::vector<DATA_GROUP*> segs;
- for (int seg_index = 0; seg_index < seg_count; seg_index++) {
-#if DEBUG
- fprintf(stderr, "Segment number %d\n", seg_index + 1);
-#endif
- segs.push_back(dls_get(dl_state.dl_text, charset, seg_index));
- }
-
- // if enabled, add DL Plus data group
- if (dl_state.dl_plus_enabled)
- segs.push_back(createDynamicLabelPlus(dl_state));
-
- // prepend to packetizer
- pad_packetizer->queue.insert(pad_packetizer->queue.begin(), segs.begin(), segs.end());
-
-#if DEBUG
- fprintf(stderr, "PAD length: %d\n", padlen);
- fprintf(stderr, "DLS text: %s\n", text.c_str());
- fprintf(stderr, "Number of DL segments: %d\n", seg_count);
-#endif
-}
-
-int History::find(const fingerprint_t& fp) const
-{
- size_t i;
- for (i = 0; i < m_database.size(); i++) {
- if (m_database[i] == fp) {
- // return the id of fingerprint found
- return m_database[i].fidx;
- }
- }
-
- // return -1 when the database doesn't contain this fingerprint
- return -1;
-}
-
-void History::add(fingerprint_t& fp)
-{
- m_database.push_back(fp);
-
- if (m_database.size() > m_hist_size) {
- m_database.pop_front();
- }
-}
-
-void History::disp_database()
-{
- size_t id;
- printf("HISTORY DATABASE:\n");
- if (m_database.empty()) {
- printf(" empty\n");
- }
- else {
- for (id = 0; id < m_database.size(); id++) {
- printf(" id %4zu: ", id);
- m_database[id].disp();
- }
- }
- printf("-----------------\n");
-}
-
-int History::get_fidx(const char* filepath)
-{
- fingerprint_t fp;
-
- fp.load_from_file(filepath);
-
- int idx = find(fp);
-
- if (idx < 0) {
- idx = m_last_given_fidx++;
- fp.fidx = idx;
-
- if (m_last_given_fidx > MAXSLIDEID) {
- m_last_given_fidx = 0;
- }
-
- add(fp);
- }
-
- return idx;
-}
-
diff --git a/src/utf8.h b/src/utf8.h
deleted file mode 100644
index 4e44514..0000000
--- a/src/utf8.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "utf8/checked.h"
-#include "utf8/unchecked.h"
-
-#endif // header guard
diff --git a/src/utf8/checked.h b/src/utf8/checked.h
deleted file mode 100644
index 1331155..0000000
--- a/src/utf8/checked.h
+++ /dev/null
@@ -1,327 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "core.h"
-#include <stdexcept>
-
-namespace utf8
-{
- // Base for the exceptions that may be thrown from the library
- class exception : public ::std::exception {
- };
-
- // Exceptions that may be thrown from the library functions.
- class invalid_code_point : public exception {
- uint32_t cp;
- public:
- invalid_code_point(uint32_t cp) : cp(cp) {}
- virtual const char* what() const throw() { return "Invalid code point"; }
- uint32_t code_point() const {return cp;}
- };
-
- class invalid_utf8 : public exception {
- uint8_t u8;
- public:
- invalid_utf8 (uint8_t u) : u8(u) {}
- virtual const char* what() const throw() { return "Invalid UTF-8"; }
- uint8_t utf8_octet() const {return u8;}
- };
-
- class invalid_utf16 : public exception {
- uint16_t u16;
- public:
- invalid_utf16 (uint16_t u) : u16(u) {}
- virtual const char* what() const throw() { return "Invalid UTF-16"; }
- uint16_t utf16_word() const {return u16;}
- };
-
- class not_enough_room : public exception {
- public:
- virtual const char* what() const throw() { return "Not enough space"; }
- };
-
- /// The library API - functions intended to be called by the users
-
- template <typename octet_iterator>
- octet_iterator append(uint32_t cp, octet_iterator result)
- {
- if (!utf8::internal::is_code_point_valid(cp))
- throw invalid_code_point(cp);
-
- if (cp < 0x80) // one octet
- *(result++) = static_cast<uint8_t>(cp);
- else if (cp < 0x800) { // two octets
- *(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- else if (cp < 0x10000) { // three octets
- *(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
- *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- else { // four octets
- *(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
- *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80);
- *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- return result;
- }
-
- template <typename octet_iterator, typename output_iterator>
- output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
- {
- while (start != end) {
- octet_iterator sequence_start = start;
- internal::utf_error err_code = utf8::internal::validate_next(start, end);
- switch (err_code) {
- case internal::UTF8_OK :
- for (octet_iterator it = sequence_start; it != start; ++it)
- *out++ = *it;
- break;
- case internal::NOT_ENOUGH_ROOM:
- throw not_enough_room();
- case internal::INVALID_LEAD:
- out = utf8::append (replacement, out);
- ++start;
- break;
- case internal::INCOMPLETE_SEQUENCE:
- case internal::OVERLONG_SEQUENCE:
- case internal::INVALID_CODE_POINT:
- out = utf8::append (replacement, out);
- ++start;
- // just one replacement mark for the sequence
- while (start != end && utf8::internal::is_trail(*start))
- ++start;
- break;
- }
- }
- return out;
- }
-
- template <typename octet_iterator, typename output_iterator>
- inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
- {
- static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
- return utf8::replace_invalid(start, end, out, replacement_marker);
- }
-
- template <typename octet_iterator>
- uint32_t next(octet_iterator& it, octet_iterator end)
- {
- uint32_t cp = 0;
- internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
- switch (err_code) {
- case internal::UTF8_OK :
- break;
- case internal::NOT_ENOUGH_ROOM :
- throw not_enough_room();
- case internal::INVALID_LEAD :
- case internal::INCOMPLETE_SEQUENCE :
- case internal::OVERLONG_SEQUENCE :
- throw invalid_utf8(*it);
- case internal::INVALID_CODE_POINT :
- throw invalid_code_point(cp);
- }
- return cp;
- }
-
- template <typename octet_iterator>
- uint32_t peek_next(octet_iterator it, octet_iterator end)
- {
- return utf8::next(it, end);
- }
-
- template <typename octet_iterator>
- uint32_t prior(octet_iterator& it, octet_iterator start)
- {
- // can't do much if it == start
- if (it == start)
- throw not_enough_room();
-
- octet_iterator end = it;
- // Go back until we hit either a lead octet or start
- while (utf8::internal::is_trail(*(--it)))
- if (it == start)
- throw invalid_utf8(*it); // error - no lead byte in the sequence
- return utf8::peek_next(it, end);
- }
-
- /// Deprecated in versions that include "prior"
- template <typename octet_iterator>
- uint32_t previous(octet_iterator& it, octet_iterator pass_start)
- {
- octet_iterator end = it;
- while (utf8::internal::is_trail(*(--it)))
- if (it == pass_start)
- throw invalid_utf8(*it); // error - no lead byte in the sequence
- octet_iterator temp = it;
- return utf8::next(temp, end);
- }
-
- template <typename octet_iterator, typename distance_type>
- void advance (octet_iterator& it, distance_type n, octet_iterator end)
- {
- for (distance_type i = 0; i < n; ++i)
- utf8::next(it, end);
- }
-
- template <typename octet_iterator>
- typename std::iterator_traits<octet_iterator>::difference_type
- distance (octet_iterator first, octet_iterator last)
- {
- typename std::iterator_traits<octet_iterator>::difference_type dist;
- for (dist = 0; first < last; ++dist)
- utf8::next(first, last);
- return dist;
- }
-
- template <typename u16bit_iterator, typename octet_iterator>
- octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
- {
- while (start != end) {
- uint32_t cp = utf8::internal::mask16(*start++);
- // Take care of surrogate pairs first
- if (utf8::internal::is_lead_surrogate(cp)) {
- if (start != end) {
- uint32_t trail_surrogate = utf8::internal::mask16(*start++);
- if (utf8::internal::is_trail_surrogate(trail_surrogate))
- cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
- else
- throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
- }
- else
- throw invalid_utf16(static_cast<uint16_t>(cp));
-
- }
- // Lone trail surrogate
- else if (utf8::internal::is_trail_surrogate(cp))
- throw invalid_utf16(static_cast<uint16_t>(cp));
-
- result = utf8::append(cp, result);
- }
- return result;
- }
-
- template <typename u16bit_iterator, typename octet_iterator>
- u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
- {
- while (start != end) {
- uint32_t cp = utf8::next(start, end);
- if (cp > 0xffff) { //make a surrogate pair
- *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
- *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
- }
- else
- *result++ = static_cast<uint16_t>(cp);
- }
- return result;
- }
-
- template <typename octet_iterator, typename u32bit_iterator>
- octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
- {
- while (start != end)
- result = utf8::append(*(start++), result);
-
- return result;
- }
-
- template <typename octet_iterator, typename u32bit_iterator>
- u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
- {
- while (start != end)
- (*result++) = utf8::next(start, end);
-
- return result;
- }
-
- // The iterator class
- template <typename octet_iterator>
- class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
- octet_iterator it;
- octet_iterator range_start;
- octet_iterator range_end;
- public:
- iterator () {}
- explicit iterator (const octet_iterator& octet_it,
- const octet_iterator& range_start,
- const octet_iterator& range_end) :
- it(octet_it), range_start(range_start), range_end(range_end)
- {
- if (it < range_start || it > range_end)
- throw std::out_of_range("Invalid utf-8 iterator position");
- }
- // the default "big three" are OK
- octet_iterator base () const { return it; }
- uint32_t operator * () const
- {
- octet_iterator temp = it;
- return utf8::next(temp, range_end);
- }
- bool operator == (const iterator& rhs) const
- {
- if (range_start != rhs.range_start || range_end != rhs.range_end)
- throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
- return (it == rhs.it);
- }
- bool operator != (const iterator& rhs) const
- {
- return !(operator == (rhs));
- }
- iterator& operator ++ ()
- {
- utf8::next(it, range_end);
- return *this;
- }
- iterator operator ++ (int)
- {
- iterator temp = *this;
- utf8::next(it, range_end);
- return temp;
- }
- iterator& operator -- ()
- {
- utf8::prior(it, range_start);
- return *this;
- }
- iterator operator -- (int)
- {
- iterator temp = *this;
- utf8::prior(it, range_start);
- return temp;
- }
- }; // class iterator
-
-} // namespace utf8
-
-#endif //header guard
-
-
diff --git a/src/utf8/core.h b/src/utf8/core.h
deleted file mode 100644
index 693d388..0000000
--- a/src/utf8/core.h
+++ /dev/null
@@ -1,329 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include <iterator>
-
-namespace utf8
-{
- // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
- // You may need to change them to match your system.
- // These typedefs have the same names as ones from cstdint, or boost/cstdint
- typedef unsigned char uint8_t;
- typedef unsigned short uint16_t;
- typedef unsigned int uint32_t;
-
-// Helper code - not intended to be directly called by the library users. May be changed at any time
-namespace internal
-{
- // Unicode constants
- // Leading (high) surrogates: 0xd800 - 0xdbff
- // Trailing (low) surrogates: 0xdc00 - 0xdfff
- const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
- const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
- const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
- const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
- const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
- const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
-
- // Maximum valid value for a Unicode code point
- const uint32_t CODE_POINT_MAX = 0x0010ffffu;
-
- template<typename octet_type>
- inline uint8_t mask8(octet_type oc)
- {
- return static_cast<uint8_t>(0xff & oc);
- }
- template<typename u16_type>
- inline uint16_t mask16(u16_type oc)
- {
- return static_cast<uint16_t>(0xffff & oc);
- }
- template<typename octet_type>
- inline bool is_trail(octet_type oc)
- {
- return ((utf8::internal::mask8(oc) >> 6) == 0x2);
- }
-
- template <typename u16>
- inline bool is_lead_surrogate(u16 cp)
- {
- return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
- }
-
- template <typename u16>
- inline bool is_trail_surrogate(u16 cp)
- {
- return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
- }
-
- template <typename u16>
- inline bool is_surrogate(u16 cp)
- {
- return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
- }
-
- template <typename u32>
- inline bool is_code_point_valid(u32 cp)
- {
- return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
- }
-
- template <typename octet_iterator>
- inline typename std::iterator_traits<octet_iterator>::difference_type
- sequence_length(octet_iterator lead_it)
- {
- uint8_t lead = utf8::internal::mask8(*lead_it);
- if (lead < 0x80)
- return 1;
- else if ((lead >> 5) == 0x6)
- return 2;
- else if ((lead >> 4) == 0xe)
- return 3;
- else if ((lead >> 3) == 0x1e)
- return 4;
- else
- return 0;
- }
-
- template <typename octet_difference_type>
- inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
- {
- if (cp < 0x80) {
- if (length != 1)
- return true;
- }
- else if (cp < 0x800) {
- if (length != 2)
- return true;
- }
- else if (cp < 0x10000) {
- if (length != 3)
- return true;
- }
-
- return false;
- }
-
- enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
-
- /// Helper for get_sequence_x
- template <typename octet_iterator>
- utf_error increase_safely(octet_iterator& it, octet_iterator end)
- {
- if (++it == end)
- return NOT_ENOUGH_ROOM;
-
- if (!utf8::internal::is_trail(*it))
- return INCOMPLETE_SEQUENCE;
-
- return UTF8_OK;
- }
-
- #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
-
- /// get_sequence_x functions decode utf-8 sequences of the length x
- template <typename octet_iterator>
- utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
- {
- if (it == end)
- return NOT_ENOUGH_ROOM;
-
- code_point = utf8::internal::mask8(*it);
-
- return UTF8_OK;
- }
-
- template <typename octet_iterator>
- utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
- {
- if (it == end)
- return NOT_ENOUGH_ROOM;
-
- code_point = utf8::internal::mask8(*it);
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
-
- return UTF8_OK;
- }
-
- template <typename octet_iterator>
- utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
- {
- if (it == end)
- return NOT_ENOUGH_ROOM;
-
- code_point = utf8::internal::mask8(*it);
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point += (*it) & 0x3f;
-
- return UTF8_OK;
- }
-
- template <typename octet_iterator>
- utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
- {
- if (it == end)
- return NOT_ENOUGH_ROOM;
-
- code_point = utf8::internal::mask8(*it);
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
-
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
-
- code_point += (*it) & 0x3f;
-
- return UTF8_OK;
- }
-
- #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
-
- template <typename octet_iterator>
- utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
- {
- // Save the original value of it so we can go back in case of failure
- // Of course, it does not make much sense with i.e. stream iterators
- octet_iterator original_it = it;
-
- uint32_t cp = 0;
- // Determine the sequence length based on the lead octet
- typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
- const octet_difference_type length = utf8::internal::sequence_length(it);
-
- // Get trail octets and calculate the code point
- utf_error err = UTF8_OK;
- switch (length) {
- case 0:
- return INVALID_LEAD;
- case 1:
- err = utf8::internal::get_sequence_1(it, end, cp);
- break;
- case 2:
- err = utf8::internal::get_sequence_2(it, end, cp);
- break;
- case 3:
- err = utf8::internal::get_sequence_3(it, end, cp);
- break;
- case 4:
- err = utf8::internal::get_sequence_4(it, end, cp);
- break;
- }
-
- if (err == UTF8_OK) {
- // Decoding succeeded. Now, security checks...
- if (utf8::internal::is_code_point_valid(cp)) {
- if (!utf8::internal::is_overlong_sequence(cp, length)){
- // Passed! Return here.
- code_point = cp;
- ++it;
- return UTF8_OK;
- }
- else
- err = OVERLONG_SEQUENCE;
- }
- else
- err = INVALID_CODE_POINT;
- }
-
- // Failure branch - restore the original value of the iterator
- it = original_it;
- return err;
- }
-
- template <typename octet_iterator>
- inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
- uint32_t ignored;
- return utf8::internal::validate_next(it, end, ignored);
- }
-
-} // namespace internal
-
- /// The library API - functions intended to be called by the users
-
- // Byte order mark
- const uint8_t bom[] = {0xef, 0xbb, 0xbf};
-
- template <typename octet_iterator>
- octet_iterator find_invalid(octet_iterator start, octet_iterator end)
- {
- octet_iterator result = start;
- while (result != end) {
- utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
- if (err_code != internal::UTF8_OK)
- return result;
- }
- return result;
- }
-
- template <typename octet_iterator>
- inline bool is_valid(octet_iterator start, octet_iterator end)
- {
- return (utf8::find_invalid(start, end) == end);
- }
-
- template <typename octet_iterator>
- inline bool starts_with_bom (octet_iterator it, octet_iterator end)
- {
- return (
- ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
- ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
- ((it != end) && (utf8::internal::mask8(*it)) == bom[2])
- );
- }
-
- //Deprecated in release 2.3
- template <typename octet_iterator>
- inline bool is_bom (octet_iterator it)
- {
- return (
- (utf8::internal::mask8(*it++)) == bom[0] &&
- (utf8::internal::mask8(*it++)) == bom[1] &&
- (utf8::internal::mask8(*it)) == bom[2]
- );
- }
-} // namespace utf8
-
-#endif // header guard
-
-
diff --git a/src/utf8/unchecked.h b/src/utf8/unchecked.h
deleted file mode 100644
index cb24271..0000000
--- a/src/utf8/unchecked.h
+++ /dev/null
@@ -1,228 +0,0 @@
-// Copyright 2006 Nemanja Trifunovic
-
-/*
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-
-#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
-
-#include "core.h"
-
-namespace utf8
-{
- namespace unchecked
- {
- template <typename octet_iterator>
- octet_iterator append(uint32_t cp, octet_iterator result)
- {
- if (cp < 0x80) // one octet
- *(result++) = static_cast<uint8_t>(cp);
- else if (cp < 0x800) { // two octets
- *(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- else if (cp < 0x10000) { // three octets
- *(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
- *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- else { // four octets
- *(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
- *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
- *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
- *(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
- }
- return result;
- }
-
- template <typename octet_iterator>
- uint32_t next(octet_iterator& it)
- {
- uint32_t cp = utf8::internal::mask8(*it);
- typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
- switch (length) {
- case 1:
- break;
- case 2:
- it++;
- cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
- break;
- case 3:
- ++it;
- cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
- ++it;
- cp += (*it) & 0x3f;
- break;
- case 4:
- ++it;
- cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
- ++it;
- cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
- ++it;
- cp += (*it) & 0x3f;
- break;
- }
- ++it;
- return cp;
- }
-
- template <typename octet_iterator>
- uint32_t peek_next(octet_iterator it)
- {
- return utf8::unchecked::next(it);
- }
-
- template <typename octet_iterator>
- uint32_t prior(octet_iterator& it)
- {
- while (utf8::internal::is_trail(*(--it))) ;
- octet_iterator temp = it;
- return utf8::unchecked::next(temp);
- }
-
- // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
- template <typename octet_iterator>
- inline uint32_t previous(octet_iterator& it)
- {
- return utf8::unchecked::prior(it);
- }
-
- template <typename octet_iterator, typename distance_type>
- void advance (octet_iterator& it, distance_type n)
- {
- for (distance_type i = 0; i < n; ++i)
- utf8::unchecked::next(it);
- }
-
- template <typename octet_iterator>
- typename std::iterator_traits<octet_iterator>::difference_type
- distance (octet_iterator first, octet_iterator last)
- {
- typename std::iterator_traits<octet_iterator>::difference_type dist;
- for (dist = 0; first < last; ++dist)
- utf8::unchecked::next(first);
- return dist;
- }
-
- template <typename u16bit_iterator, typename octet_iterator>
- octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
- {
- while (start != end) {
- uint32_t cp = utf8::internal::mask16(*start++);
- // Take care of surrogate pairs first
- if (utf8::internal::is_lead_surrogate(cp)) {
- uint32_t trail_surrogate = utf8::internal::mask16(*start++);
- cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
- }
- result = utf8::unchecked::append(cp, result);
- }
- return result;
- }
-
- template <typename u16bit_iterator, typename octet_iterator>
- u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
- {
- while (start < end) {
- uint32_t cp = utf8::unchecked::next(start);
- if (cp > 0xffff) { //make a surrogate pair
- *result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
- *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
- }
- else
- *result++ = static_cast<uint16_t>(cp);
- }
- return result;
- }
-
- template <typename octet_iterator, typename u32bit_iterator>
- octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
- {
- while (start != end)
- result = utf8::unchecked::append(*(start++), result);
-
- return result;
- }
-
- template <typename octet_iterator, typename u32bit_iterator>
- u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
- {
- while (start < end)
- (*result++) = utf8::unchecked::next(start);
-
- return result;
- }
-
- // The iterator class
- template <typename octet_iterator>
- class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
- octet_iterator it;
- public:
- iterator () {}
- explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
- // the default "big three" are OK
- octet_iterator base () const { return it; }
- uint32_t operator * () const
- {
- octet_iterator temp = it;
- return utf8::unchecked::next(temp);
- }
- bool operator == (const iterator& rhs) const
- {
- return (it == rhs.it);
- }
- bool operator != (const iterator& rhs) const
- {
- return !(operator == (rhs));
- }
- iterator& operator ++ ()
- {
- ::std::advance(it, utf8::internal::sequence_length(it));
- return *this;
- }
- iterator operator ++ (int)
- {
- iterator temp = *this;
- ::std::advance(it, utf8::internal::sequence_length(it));
- return temp;
- }
- iterator& operator -- ()
- {
- utf8::unchecked::prior(it);
- return *this;
- }
- iterator operator -- (int)
- {
- iterator temp = *this;
- utf8::unchecked::prior(it);
- return temp;
- }
- }; // class iterator
-
- } // namespace utf8::unchecked
-} // namespace utf8
-
-
-#endif // header guard
-
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-29 09:22:44 +0000