Add syslog_munin tool

4 files changed, 251 insertions, 0 deletions

diff --git a/syslog_munin/README.md b/syslog_munin/README.md
new file mode 100644
index 0000000..cc0580b
--- /dev/null
+++ b/syslog_munin/README.md
@@ -0,0 +1,21 @@
+Overview
+--------
+
+This folder contains a helper tool that
+monitors syslog and counts warnings and errors
+for munin to plot.
+
+Setup
+-----
+
+ # Configure your syslog daemon to send messages from
+   local0 (the facility the ODR-mmbTools log to)
+   to localhost:51400 over UDP
+ # Use supervisord or similar to run the service *syslog_munin.py*
+ # Copy *syslog_munin_plugin.sh* to */etc/munin/plugins/*
+ # Wait for munin to start graphing
+
+Todo
+----
+
+ * Define alert thresholds in the config descriptor.
diff --git a/syslog_munin/syslog_munin.py b/syslog_munin/syslog_munin.py
new file mode 100755
index 0000000..ee9e8f7
--- /dev/null
+++ b/syslog_munin/syslog_munin.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+#
+# This syslog server receives UDP based syslog entries and
+# serves them in the format munin can understand
+#
+# Read an EDI dump file and transmit over UDP
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2017 Matthias P. Braendli
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+#    The above copyright notice and this permission notice shall be included in
+#    all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#    THE SOFTWARE.
+
+HOST, syslogport, muninport = "127.0.0.1", 51400, 51401
+
+import SocketServer
+import Queue
+import re
+import threading
+
+LOG_FACILITY = {
+    0: 'kernel messages',
+    1: 'user-level messages',
+    2: 'mail system',
+    3: 'system daemons',
+    4: 'security/authorization messages',
+    5: 'messages generated internally by syslogd',
+    6: 'line printer subsystem',
+    7: 'network news subsystem',
+    8: 'UUCP subsystem',
+    9: 'clock daemon',
+    10: 'security/authorization messages',
+    11: 'FTP daemon',
+    12: 'NTP subsystem',
+    13: 'log audit',
+    14: 'log alert',
+    15: 'clock daemon',
+    16: 'local use 0 (local0)',
+    17: 'local use 1 (local1)',
+    18: 'local use 2 (local2)',
+    19: 'local use 3 (local3)',
+    20: 'local use 4 (local4)',
+    21: 'local use 5 (local5)',
+    22: 'local use 6 (local6)',
+    23: 'local use 7 (local7)'
+}
+
+LOG_LEVEL = {
+    0: 'Emergency',
+    1: 'Alert',
+    2: 'Critical',
+    3: 'Error',
+    4: 'Warning',
+    5: 'Notice',
+    6: 'Informational',
+    7: 'Debug'
+}
+
+def split_priority_from_message(msg):
+    '''
+    https://www.fir3net.com/UNIX/Linux/how-to-determine-the-syslog-facility-
+    using-tcpdump.html
+    Each Syslog message contains a priority value. The priority value is
+    enclosed within the characters < >. The priority value can be
+    between 0 and 191 and consists of a Facility value and a Level value.
+    Facility being the type of message, such as a kernel or mail message.
+    And level being a severity level of the message.
+
+    To calculate the priority value the following formula is used :
+    Priority = Facility * 8 + Level
+
+    So to determine the facility value of a syslog message we divide the
+    priority value by 8. The remainder is the level value.
+    '''
+    match = re.search(r"\b(?=\w)(\d*)\b(?!\w)>(.*)", msg, re.MULTILINE)
+    if match:
+        result = match.group(1)
+        return int(result), str(match.group(2))
+    else:
+        raise
+
+def get_facility_from_priority(priority):
+    return int(priority/8)
+
+def get_level_from_priority(priority):
+    return priority % 8
+
+class ThreadedUDPServer(SocketServer.ThreadingMixIn, SocketServer.UDPServer):
+        pass
+
+class SyslogUDPHandler(SocketServer.BaseRequestHandler):
+    def handle(self):
+        data = bytes.decode(self.request[0].strip())
+        socket = self.request[1]
+        syslog_message = str(data)
+
+        priority, text = split_priority_from_message(syslog_message)
+        facility = get_facility_from_priority(priority)
+        lvl = get_level_from_priority(priority)
+
+        logentry = dict()
+        logentry['raw']      = syslog_message
+        logentry['level']    = LOG_LEVEL[lvl]
+        logentry['facility'] = LOG_FACILITY[facility]
+        logentry['text']     = text
+        print("Push {}".format(logentry['text']))
+        logentries.put(logentry)
+
+
+munin_config = """
+multigraph mmbtools_log
+graph_title mmbTools errors and warnings
+graph_order high low
+graph_args --base 1000
+graph_vlabel number of entries in log during last ${{graph_period}}
+graph_category mmbtools
+graph_info This graph shows number of error and warning messages generated by the mmbTools
+errors.info Errors
+errors.label Errors
+errors.min 0
+errors.type ABSOLUTE
+warnings.info Warnings
+warnings.label Warnings
+warnings.min 0
+warnings.type ABSOLUTE
+"""
+
+class MuninHandler(SocketServer.BaseRequestHandler):
+    def handle(self):
+        global error_count, warning_count
+
+        self.data = self.request.recv(128).strip()
+        print self.data
+        if self.data == 'config':
+            self.request.sendall(munin_config)
+        elif self.data == 'values':
+            self.update_log_counts()
+
+            values = ["multigraph mmbtools_log"]
+            values += ["errors.value {}".format(error_count)]
+            values += ["warnings.value {}".format(warning_count)]
+
+            error_count = 0
+            warning_count = 0
+            self.request.sendall("\n".join(values) + "\n")
+
+    def update_log_counts(self):
+        global error_count, warning_count
+        try:
+            while True:
+                entry = logentries.get_nowait()
+                print("Pop {}".format(entry['text']))
+
+                if entry['level'] in ['Emergency', 'Alert', 'Critical', 'Error']:
+                   error_count += 1
+                elif entry['level'] == 'Warning':
+                   warning_count += 1
+        except Queue.Empty:
+            print("No data in queue")
+
+
+if __name__ == "__main__":
+    try:
+        print("Startup")
+
+        logentries = Queue.Queue(10000)
+        error_count = 0
+        warning_count = 0
+        muninserver = None
+        logudp = None
+
+        print("Init Syslog handler")
+        logudp = ThreadedUDPServer((HOST, syslogport), SyslogUDPHandler)
+        print("Create Syslog thread")
+        syslogthread = threading.Thread(target=logudp.serve_forever, kwargs={'poll_interval': 0.1})
+
+        print("Init Munin handler")
+        muninserver = SocketServer.TCPServer((HOST, muninport), MuninHandler)
+
+        print("Starting handlers")
+
+        syslogthread.start()
+        muninserver.serve_forever()
+
+    except KeyboardInterrupt:
+        print("Ctrl-C received")
+    finally:
+        if muninserver is not None:
+            muninserver.shutdown()
+        if logudp is not None:
+            logudp.shutdown()
+        print("Waiting for thread to finish")
+        syslogthread.join()
+
+    print("Quitting")
diff --git a/syslog_munin/syslog_munin.supervisor.conf b/syslog_munin/syslog_munin.supervisor.conf
new file mode 100644
index 0000000..dc43bdd
--- /dev/null
+++ b/syslog_munin/syslog_munin.supervisor.conf
@@ -0,0 +1,7 @@
+[program:syslog_munin]
+command=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.py
+directory=/home/bram/dab/mmbtools-aux/syslog_munin
+autostart=true
+autorestart=true
+stderr_logfile=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.err.log
+stdout_logfile=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.out.log
diff --git a/syslog_munin/syslog_munin_plugin.sh b/syslog_munin/syslog_munin_plugin.sh
new file mode 100755
index 0000000..97e0951
--- /dev/null
+++ b/syslog_munin/syslog_munin_plugin.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# This is the munin plugin that connects to
+# ./syslog_munin.py and asks for config and/or
+# values
+
+if [ "$1" == "" ]; then
+    echo "argument expected"
+    exit 1
+else
+    echo $1 | nc 127.0.0.1 51401
+fi