From 9525224ac5f026ed2610902970cfc493ecdcc29a Mon Sep 17 00:00:00 2001 From: "Matthias P. Braendli" Date: Fri, 8 Aug 2014 15:27:24 +0200 Subject: Add scripts and example site --- README.md | 100 ++++++++++++++ encode-gst.sh | 48 +++++++ encode-jack.sh | 274 +++++++++++++++++++++++++++++++++++++ encode-mpg123.sh | 45 ++++++ examplesite/configuration.sh | 14 ++ examplesite/dls/radio1-default.dls | 1 + examplesite/filtertaps.txt | 46 +++++++ examplesite/mail-warning.txt | 0 examplesite/mod.ini | 46 +++++++ examplesite/multiplex.mux | 76 ++++++++++ icy-info.py | 93 +++++++++++++ kill-all-encoders.sh | 6 + launch-all-encoders.sh | 19 +++ radio.sh | 52 +++++++ start-mux-mod.sh | 34 +++++ 15 files changed, 854 insertions(+) create mode 100644 README.md create mode 100755 encode-gst.sh create mode 100755 encode-jack.sh create mode 100755 encode-mpg123.sh create mode 100644 examplesite/configuration.sh create mode 100644 examplesite/dls/radio1-default.dls create mode 100644 examplesite/filtertaps.txt create mode 100644 examplesite/mail-warning.txt create mode 100644 examplesite/mod.ini create mode 100644 examplesite/multiplex.mux create mode 100755 icy-info.py create mode 100755 kill-all-encoders.sh create mode 100755 launch-all-encoders.sh create mode 100755 radio.sh create mode 100755 start-mux-mod.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..22134ea --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +ODR-mmbTools scripts for 24/7 operation +======================================= + +This repository contains a set of scripts that can be used for +running the ODR-mmbTools in a production environment, where +failure resilience is important. + +Concept +------- + +The scripts themselves, all located in this folder, are independent +of the configuration of the transmission. All configuration is +in a separate folder called 'site'. 'examplesite' contains a sample +configuration you can adapt. + +These scripts assume that all programme streams to be included in +the multiplex are mp3 webstreams. For each programme, a URL has to +be specified. + +Prerequisites +------------- +You need to have a working ODR-DabMux and ODR-DabMod configuration to +use these scripts. Also several audio-related tools are necessary +(mplayer, gstreamer, etc.) + +The ICY-Text to DLS script requires python. + +Folder structure +---------------- + +The root folder contains the scripts described below. + +The site/ folder contains the site configuration. +The site/dls folder contains the files needed for communication between +the scripts for the insertion of DLS, and the site/slide contains slides +to be transmitted. + +site/configuration.sh lists all the programmes to be transmitted. + +site/multiplex.mux contains the ODR-DabMux configuration. site/mod.ini is used +as configuration for ODR-DabMod. + +site/filtertaps.txt contains the ODR-DabMod FIRFilter taps. + +site/mail-warning.txt is either empty, if no warning mails should be sent by the +script, or a valid email address to send the warning to. + +The site/dls folder must exist, and can contain default DLS text files for the +different programmes. + +The site/slide folder is optional, and must contain one folder for each programme, +named after its ID, into which you can place slides to be transmitted. + +How to get started +------------------ + +The examplesite/ folder should be taken as a template to create a new site, it +contains all needed files. + + * Create a copy of it, and call it *site*. + * Modify and adapt all configuration files presented above. + * Start a GNU Screen session. + * Launch JACK (see below) + * Start ODR-DabMux and ODR-DabMod using *start-mux-mod.sh* + * Start one encoder using *radio.sh ID* or all encoders with + *launch-all-encoders.sh* + +Scripts +------- + +*start-mux-mod.sh* starts ODR-DabMux and ODR-DabMod with the configuration +given in site/multiplex.mux and site/mod.ini + +*radio.sh* starts one mplayer and fdk-aac-dabplus encoder for the radio +programme whose identification is set in site/configuration.sh. This script +will automatically restart the encoder if there is a failure. + +The *encode-XYZ.sh* scripts contain all the logic for failure resilience, +and use different players to read the stream. + +*encode-jack.sh* is the best supported script of all these. It extracts +ICY-Text and launches the mot-encoder to insert DLS and optionally slides. + It needs a running JACK daemon to work, which should be started with + + jackd -d dummy -r 32000 + +*launch-all-encoders.sh* and *kill-all-encoders.sh* can start and stop all +configured encoders, and are meant to be used inside a GNU Screen session. For +each encoder, a new GNU Screen window is created. + +*icy-info.py* is a helper script that converts the webstream ICY Text into DLS +for the mot-encoder. Do not run it directly, it is used by *encode-jack.sh*. + + +Additional remarks +------------------ + +For a rate of 48kHz, the RATE parameters in the *encode-XYZ.sh* script must be +changed. + diff --git a/encode-gst.sh b/encode-gst.sh new file mode 100755 index 0000000..255b4be --- /dev/null +++ b/encode-gst.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Encode one programme using gstreamer. +# +# Status: Experimental + +URL=$1 +ID=$2 +DST=$3 + +QUEUEDELAY=400000 #400ms + +BITRATE=80 +RATE=32000 + +if [[ "$DST" == "" ]] +then + echo "Usage $0 url id destination" + exit 1 +fi + +while true +do + + gst-launch -q \ + uridecodebin uri=$URL ! \ + queue "max-size-time=$QUEUEDELAY" ! \ + audioresample quality=8 ! \ + audioconvert ! \ + audio/x-raw-int, "rate=$RATE,format=S16LE,channels=2" ! \ + filesink location="/dev/stdout" | \ + dabplus-enc -i /dev/stdin -b $BITRATE -r $RATE -f raw -a -o $DST + + R=$? + + NOW=$(date) + + mail -s "Encoder $ID restart $URL" matthias+odrge1@mpb.li << EOF +The encoder id:$ID +encoding $URL -> $DST with gstreamer was restarted at +$NOW + +The return code was $R + +EOF + + sleep 5 +done diff --git a/encode-jack.sh b/encode-jack.sh new file mode 100755 index 0000000..7a73589 --- /dev/null +++ b/encode-jack.sh @@ -0,0 +1,274 @@ +#!/bin/bash +# +# Encode programme using mplayer, connect through JACK +# to dabplus-enc +# +# Read webstream from URL using mplayer +# Launch dabplus-enc encoder +# connect both through JACK +# monitor processes, and restart if necessary +# Optionally send an email when restart happens +# +# Extract ICY Text from stream and use it for DLS + +printerr() { + echo -e "\033[01;31m$1\033[0m" +} + +printmsg() { + echo -e "\033[01;32m$1\033[0m" +} + +set -u + +# check number of arguments +if [[ "$#" < 3 ]] ; then + echo "Usage $0 url jack-id destination [volume]" + echo "The volume setting is optional" + exit 1 +fi + +if [[ "$#" > 2 ]] ; then + URL=$1 + ID=$2 + DST=$3 +fi + +if [[ "$#" == 4 ]] ; then + VOL=$4 +else + VOL="" +fi + +BITRATE=80 +RATE=32000 + +DLSDIR=site/dls +SLIDEDIR=site/slide + +encoderalive=0 +mplayerpid=0 +encoderpid=0 +motencoderpid=0 +running=1 + +mplayer_ok=0 +encoder_ok=0 + +# The trap for Ctrl-C +sigint_trap() { + printerr "Got Ctrl-C, killing mplayer and encoder" + running=0 + + if [[ "$mplayerpid" != "0" ]] ; then + kill -TERM $mplayerpid + sleep 2 + kill -KILL $mplayerpid + fi + + if [[ "$encoderpid" != "0" ]] ; then + kill -TERM $encoderpid + sleep 2 + kill -KILL $encoderpid + fi + + if [[ "$motencoderpid" != "0" ]] ; then + kill -TERM $motencoderpid + sleep 2 + kill -KILL $motencoderpid + fi + + printmsg "Goodbye" + exit +} + +trap sigint_trap SIGTERM +trap sigint_trap SIGINT + +while [[ "$running" == "1" ]] +do + if [[ "$mplayerpid" == "0" ]] ; then + if [[ "$VOL" == "" ]] ; then + mplayer -quiet -af resample=$RATE:0:2 -ao jack:name=$ID $URL | \ + ./icy-info.py $DLSDIR/${ID}.dls $DLSDIR/${ID}-default.dls & + mplayerpid=$! + else + mplayer -quiet -af resample=$RATE:0:2 -af volume=$VOL -ao jack:name=$ID $URL | \ + ./icy-info.py $DLSDIR/${ID}.dls $DLSDIR/${ID}-default.dls & + mplayerpid=$! + fi + + printmsg "Started mplayer with pid $mplayerpid" + + # give some time to mplayer to set up and + # wait until port becomes visible + timeout=10 + + while [[ "$mplayer_ok" == "0" ]] + do + printmsg "Waiting for mplayer to connect to jack ($timeout)" + sleep 1 + mplayer_ok=$(jack_lsp $ID:out_0 | wc -l) + + timeout=$(( $timeout - 1 )) + + if [[ "$timeout" == "0" ]] ; then + printerr "mplayer doesn't connect to jack !" + kill $mplayerpid + break + fi + done + else + printmsg "No need to start mplayer: $mplayerpid" + fi + + if [[ "$mplayer_ok" == "1" && "$encoder_ok" == "0" ]] ; then + dabplus-enc -j ${ID}enc -l \ + -p 34 -P $DLSDIR/${ID}.pad \ + -b $BITRATE -r $RATE -f raw -a -o $DST & + encoderpid=$! + + # give some time to the encoder to set up and + # wait until port becomes visible + timeout=10 + + encoder_connected=0 + + while [[ "$encoder_connected" == "0" ]] + do + printmsg "Waiting for encoder to connect to jack ($timeout)" + sleep 1 + encoder_connected=$(jack_lsp ${ID}enc:input0 | wc -l) + + timeout=$(( $timeout - 1)) + + if [[ "$timeout" == "0" ]] ; then + printerr "encoder doesn't connect to jack !" + kill $encoderpid + break + fi + done + + if [[ "$encoder_connected" == "1" ]] ; then + jack_connect ${ID}:out_0 ${ID}enc:input0 && \ + jack_connect ${ID}:out_1 ${ID}enc:input1 + connect_ret=$? + + if [[ "$connect_ret" == "0" ]] ; then + encoder_ok=1 + else + encoder_ok=0 + fi + + if [[ "$encoder_ok" == "1" ]] ; then + printmsg "Started encoder with pid $encoderpid" + else + if [[ "$encoderpid" != "0" ]] ; then + kill -TERM $encoderpid + fi + fi + fi + fi + + if [[ "$encoder_ok" == "1" && "$motencoderpid" == "0" ]] ; then + # Check if the slides folder exists, and start mot-encoder accordingly + if [[ -d "$SLIDEDIR/$ID" ]] ; then + mot-encoder -o $DLSDIR/${ID}.pad -t $DLSDIR/${ID}.dls -p 34 -v \ + -e -d $SLIDEDIR/${ID} & + motencoderpid=$! + else + mot-encoder -o $DLSDIR/${ID}.pad -t $DLSDIR/${ID}.dls -p 34 -v & + motencoderpid=$! + fi + + printmsg "Started mot-encoder with pid $encoderpid" + fi + + + sleep 5 + + checkloop=1 + while [[ "$checkloop" == "1" ]] + do + sleep 2 + + kill -s 0 $mplayerpid + if [[ "$?" != "0" ]] ; then + # mplayer died + # we must kill jack-stdout, because we cannot reconnect it + # to a new mplayer, since we do not know the jack-stdout name. + # And it has no cmdline option to set one, Rrrrongntudtjuuu! + if [[ "$encoderpid" != "0" ]] ; then + kill -TERM $encoderpid + fi + + if [[ "$motencoderpid" != "0" ]] ; then + kill -TERM $motencoderpid + fi + + # mark as dead + mplayerpid=0 + mplayer_ok=0 + encoderpid=0 + encoder_ok=0 + motencoderpid=0 + + checkloop=0 + + printerr "Mplayer died" + fi + + if [[ "$encoderpid" != "0" ]] ; then + kill -s 0 $encoderpid + if [[ "$?" != "0" ]] ; then + # the encoder died, + # no need to kill the mplayer, we can reconnect to it + + if [[ "$motencoderpid" != "0" ]] ; then + kill -TERM $motencoderpid + fi + + motencoderpid=0 + encoderpid=0 + encoder_ok=0 + + checkloop=0 + + printerr "Encoder died" + fi + fi + + if [[ "$motencoderpid" != "0" ]] ; then + kill -s 0 $motencoderpid + if [[ "$?" != "0" ]] ; then + # mot-encoder died + # let's try restarting it + + motencoderpid=0 + + checkloop=0 + + printerr "mot-encoder died" + fi + fi + done + + MAILTO=$(cat site/mail-warning.txt) + + if [[ "$MAILTO" != "" ]] ; then + NOW=$(date) + + mail -s "Encoder $ID restart $URL" $MAILTO << EOF +The encoder id:$ID +encoding $URL -> $DST using mplayer and jack was restarted at +$NOW + +mplayer ok? $mplayer_ok + +EOF + + fi + sleep 5 + +done + diff --git a/encode-mpg123.sh b/encode-mpg123.sh new file mode 100755 index 0000000..17c4d7f --- /dev/null +++ b/encode-mpg123.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Encode programme using mpg123 +# +# Status: Experimental + +URL=$1 +ID=$2 +DST=$4 + +BITRATE=80 +RATE=32000 + +if [[ "$DST" == "" ]] +then + echo "Usage $0 url id destination" + exit 1 +fi + +while true +do + + mpg123 -r $RATE -s $URL | \ + dabplus-enc -i /dev/stdin -b $BITRATE -r $RATE -f raw -a -o $DST + + R=$? + + MAILTO=$(cat site/mail-warning.txt) + + if [[ "$MAILTO" != "" ]] ; then + NOW=$(date) + + mail -s "Encoder $ID restart $URL" $MAILTO << EOF +The encoder id:$ID +encoding $URL -> $DST with mpg123 was restarted at +$NOW + +The return code was $R + +EOF + fi + + sleep 5 +done + diff --git a/examplesite/configuration.sh b/examplesite/configuration.sh new file mode 100644 index 0000000..9ca0942 --- /dev/null +++ b/examplesite/configuration.sh @@ -0,0 +1,14 @@ +# Configuration file for the encoder scripts + +all_radios=( + "radio1" + "radio2" ) + +# for each radio, write here the full encoder command. +# encode-jack needs: +# URL ID dabmux-URL [amplitude correction] +radios[radio1]="./encode-jack.sh http://radio1streamurl.example.com radio1 tcp://localhost:9000" + +# Attenuate radio2 by 3dB +radios[radio2]="./encode-jack.sh http://radio2streamurl.example.com radio2 tcp://localhost:9001 -3" + diff --git a/examplesite/dls/radio1-default.dls b/examplesite/dls/radio1-default.dls new file mode 100644 index 0000000..b79382b --- /dev/null +++ b/examplesite/dls/radio1-default.dls @@ -0,0 +1 @@ +Radio1 - The Best Example DLS Text Ever diff --git a/examplesite/filtertaps.txt b/examplesite/filtertaps.txt new file mode 100644 index 0000000..cd0b28d --- /dev/null +++ b/examplesite/filtertaps.txt @@ -0,0 +1,46 @@ +45 +-0.00110450468492 +0.00120703084394 +-0.000840645749122 +-0.000187368263141 +0.00184351124335 +-0.00355578539893 +0.00419321097434 +-0.00254214904271 +-0.00183473504148 +0.00781436730176 +-0.0125957569107 +0.0126200336963 +-0.00537294941023 +-0.00866683479398 +0.0249746385962 +-0.0356550291181 +0.0319730602205 +-0.00795613788068 +-0.0363943465054 +0.0938014090061 +-0.151176810265 +0.193567320704 +0.791776955128 +0.193567320704 +-0.151176810265 +0.0938014090061 +-0.0363943465054 +-0.00795613788068 +0.0319730602205 +-0.0356550291181 +0.0249746385962 +-0.00866683479398 +-0.00537294941023 +0.0126200336963 +-0.0125957569107 +0.00781436730176 +-0.00183473504148 +-0.00254214904271 +0.00419321097434 +-0.00355578539893 +0.00184351124335 +-0.000187368263141 +-0.000840645749122 +0.00120703084394 +-0.00110450468492 diff --git a/examplesite/mail-warning.txt b/examplesite/mail-warning.txt new file mode 100644 index 0000000..e69de29 diff --git a/examplesite/mod.ini b/examplesite/mod.ini new file mode 100644 index 0000000..c3d2d37 --- /dev/null +++ b/examplesite/mod.ini @@ -0,0 +1,46 @@ +[remotecontrol] +telnet=1 +telnetport=2121 + +[log] +syslog=1 +filelog=1 +filename=/dev/stderr + +[input] +transport=file +source=/dev/stdin +loop=0 + +[modulator] +gainmode=2 +mode=1 +dac_clk_rate=0 +digital_gain=0.5 +rate=2048000 + +[firfilter] +enabled=0 +filtertapsfile=site/filtertaps.txt + +[output] +output=uhd + +[fileoutput] +filename=/dev/null + +[uhdoutput] +device= +type=b200 +master_clock_rate=32768000 +channel=10D +txgain=40 +refclk_source=internal +pps_source=none +behaviour_refclk_lock_lost=crash + +[delaymanagement] +synchronous=0 +management=dynamic +fixedoffset=0.000 +dynamicoffsetfile=site/modulatoroffset diff --git a/examplesite/multiplex.mux b/examplesite/multiplex.mux new file mode 100644 index 0000000..3fcda00 --- /dev/null +++ b/examplesite/multiplex.mux @@ -0,0 +1,76 @@ +; Example configuration file for site. +; Please see ODR-DabMux repository for more details +; +general { + dabmode 1 + nbframes 0 + syslog 1 + writescca false + tist false + statsserverport 12720 +} + +remotecontrol { + telnetport 12721 +} + +ensemble { + id 0xabcd + ecc 0x01 + label "ODR-mmbTools" + shortlabel "mmbTools" + international-table 1 + local-time-offset auto +} + +services { + srv_radio1 { + label "GLOBAL FM+" + id 0x6543 + } + srv_radio2 { + label "radio2" + id 0x6542 + } +} + +subchannels { + sub_radio1 { + type dabplus + inputfile "tcp://*:9000" + zmq-buffer 80 + zmq-prebuffering 40 + bitrate 80 + id 1 + protection 3 + } + sub_radio2 { + type dabplus + inputfile "tcp://*:9001" + zmq-buffer 80 + zmq-prebuffering 40 + bitrate 80 + id 2 + protection 3 + } +} + +components { + comp_radio1 { + label "radio1" + shortlabel "radio1" + service srv_radio1 + subchannel sub_radio1 + } + comp_radio2 { + label "radio2" + shortlabel "radio2" + service srv_radio2 + subchannel sub_radio2 + } +} + +outputs { + stdout "fifo:///dev/stdout?type=raw" +} + diff --git a/icy-info.py b/icy-info.py new file mode 100755 index 0000000..14ed558 --- /dev/null +++ b/icy-info.py @@ -0,0 +1,93 @@ +#!/usr/bin/python2 +# +# This script parses the mplayer standard output and +# extracts ICY info for the mot-encoder. +# +# Usage: +# mplayer | icy-info.py file.dls file-with-default.dls +# +# the file-with-default.dls contains DLS text to be sent when there +# is no ICY info + +import re +import select +import sys +import time + +re_icy = re.compile(r"""ICY Info: StreamTitle='([^']*)'.*""") + +if len(sys.argv) < 3: + print("Please specify dls output file, and file containing default text") + sys.exit(1) + +dls_file = sys.argv[1] + +default_textfile = sys.argv[2] + +def new_dlstext(text): + if text.strip() == "": + try: + fd = open(default_textfile, "r") + text = fd.read().strip() + fd.close() + except Exception as e: + print("Could not read default text from {}: {}".format(default_textfile, e)) + + print("New Text: {}".format(text)) + + fd = open(dls_file, "w") + fd.write(text) + fd.close() + +wait_timeout = 5 +nodls_timeout = 0 + + +while True: + # readline is blocking, therefore we cannot send a default text + # after some timeout + new_data = sys.stdin.readline() + if not new_data: + break + + match = re_icy.match(new_data) + + if match: + artist_title = match.groups()[0] + new_dlstext(artist_title) + else: + print("{}".format(new_data.strip())) + +if False: + # The select call creates a one ICY delay, and it's not clear why... + while True: + rfds, wfds, efds = select.select( [sys.stdin], [], [], wait_timeout) + + if rfds: + # new data available on stdin + print("SELECT !") + new_data = sys.stdin.readline() + print("DATA ! {}".format(new_data)) + + if not new_data: + break + + match = re_icy.match(new_data) + + if match: + artist_title = match.groups()[0] + new_dlstext(artist_title) + else: + print("{}".format(new_data.strip())) + + else: + # timeout reading stdin + nodls_timeout += 1 + + if nodls_timeout == 100: + new_dlstext("") + nodls_timeout = 0 + + time.sleep(.1) + + diff --git a/kill-all-encoders.sh b/kill-all-encoders.sh new file mode 100755 index 0000000..0ced123 --- /dev/null +++ b/kill-all-encoders.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# kill all processes that contain _encode- + +pkill _encode- + diff --git a/launch-all-encoders.sh b/launch-all-encoders.sh new file mode 100755 index 0000000..1f7e56d --- /dev/null +++ b/launch-all-encoders.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# launch each encoder in its own screen window + +if [ -f site/configuration.sh ] +then + + source site/configuration.sh + + for radio in ${all_radios[*]} + do + screen -t $radio ./radio $radio + sleep 0.4 + done + +else + echo "Error: No site configuration available!" + exit 1 +fi + diff --git a/radio.sh b/radio.sh new file mode 100755 index 0000000..d475ce4 --- /dev/null +++ b/radio.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Start the encoder for the radio + +# Declare radios to be an associative array +declare -A radios + +source site/configuration.sh + +printerr() { + echo -e "\033[01;31m$1\033[0m" +} + +printmsg() { + echo -e "\033[01;32m$1\033[0m" +} + +script_pid=0 +sigint_trap() { + printerr "Got Ctrl-C, killing radio encoder script" + if [[ "$script_pid" != "0" ]] ; then + kill $script_pid + script_pid=0 + wait + fi +} + +set -e + +# check number of arguments +if [[ "$#" < 1 ]] ; then + echo "Usage $0 radio-id" + echo "You must specify which radio to start" + exit 1 +fi + +RADIO=$1 + +if [ ${radios[$RADIO]+_} ] ; then + COMMAND=${radios[$RADIO]} + + trap sigint_trap SIGINT + + # execute script + $COMMAND & + script_pid=$! + wait +else + echo "Radio $RADIO not defined in configuration" + exit 1 +fi + diff --git a/start-mux-mod.sh b/start-mux-mod.sh new file mode 100755 index 0000000..c7f2489 --- /dev/null +++ b/start-mux-mod.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Launch the multiplexer and the modulator + +if [[ -e site/multiplex.mux && -e site/mod.ini && site/mail-warning.txt ]] +then + + while true + do + odr-dabmux -e site/multiplex.mux | sudo odr-dabmod -C site/mod.ini + R=$? + + MAILTO=$(cat site/mail-warning.txt) + + if [[ "$MAILTO" != "" ]] ; then + NOW=$(date) + mail -s "MUX and MOD restart" $MAILTO << EOF +The mux and mod were restarted at +$NOW + +The return code was $R + +EOF + + fi + + sleep 15 + done +else + echo "Incomplete site configuration !" + echo "Check that the site/ folder exists" + exit 1 +fi + -- cgit v1.2.3