diff options
| -rw-r--r-- | syslog_munin/README.md | 21 | ||||
| -rwxr-xr-x | syslog_munin/syslog_munin.py | 211 | ||||
| -rw-r--r-- | syslog_munin/syslog_munin.supervisor.conf | 7 | ||||
| -rwxr-xr-x | syslog_munin/syslog_munin_plugin.sh | 12 |
4 files changed, 251 insertions, 0 deletions
diff --git a/syslog_munin/README.md b/syslog_munin/README.md new file mode 100644 index 0000000..cc0580b --- /dev/null +++ b/syslog_munin/README.md @@ -0,0 +1,21 @@ +Overview +-------- + +This folder contains a helper tool that +monitors syslog and counts warnings and errors +for munin to plot. + +Setup +----- + + # Configure your syslog daemon to send messages from + local0 (the facility the ODR-mmbTools log to) + to localhost:51400 over UDP + # Use supervisord or similar to run the service *syslog_munin.py* + # Copy *syslog_munin_plugin.sh* to */etc/munin/plugins/* + # Wait for munin to start graphing + +Todo +---- + + * Define alert thresholds in the config descriptor. diff --git a/syslog_munin/syslog_munin.py b/syslog_munin/syslog_munin.py new file mode 100755 index 0000000..ee9e8f7 --- /dev/null +++ b/syslog_munin/syslog_munin.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# This syslog server receives UDP based syslog entries and +# serves them in the format munin can understand +# +# Read an EDI dump file and transmit over UDP +# +# The MIT License (MIT) +# +# Copyright (c) 2017 Matthias P. Braendli +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +HOST, syslogport, muninport = "127.0.0.1", 51400, 51401 + +import SocketServer +import Queue +import re +import threading + +LOG_FACILITY = { + 0: 'kernel messages', + 1: 'user-level messages', + 2: 'mail system', + 3: 'system daemons', + 4: 'security/authorization messages', + 5: 'messages generated internally by syslogd', + 6: 'line printer subsystem', + 7: 'network news subsystem', + 8: 'UUCP subsystem', + 9: 'clock daemon', + 10: 'security/authorization messages', + 11: 'FTP daemon', + 12: 'NTP subsystem', + 13: 'log audit', + 14: 'log alert', + 15: 'clock daemon', + 16: 'local use 0 (local0)', + 17: 'local use 1 (local1)', + 18: 'local use 2 (local2)', + 19: 'local use 3 (local3)', + 20: 'local use 4 (local4)', + 21: 'local use 5 (local5)', + 22: 'local use 6 (local6)', + 23: 'local use 7 (local7)' +} + +LOG_LEVEL = { + 0: 'Emergency', + 1: 'Alert', + 2: 'Critical', + 3: 'Error', + 4: 'Warning', + 5: 'Notice', + 6: 'Informational', + 7: 'Debug' +} + +def split_priority_from_message(msg): + ''' + https://www.fir3net.com/UNIX/Linux/how-to-determine-the-syslog-facility- + using-tcpdump.html + Each Syslog message contains a priority value. The priority value is + enclosed within the characters < >. The priority value can be + between 0 and 191 and consists of a Facility value and a Level value. + Facility being the type of message, such as a kernel or mail message. + And level being a severity level of the message. + + To calculate the priority value the following formula is used : + Priority = Facility * 8 + Level + + So to determine the facility value of a syslog message we divide the + priority value by 8. The remainder is the level value. + ''' + match = re.search(r"\b(?=\w)(\d*)\b(?!\w)>(.*)", msg, re.MULTILINE) + if match: + result = match.group(1) + return int(result), str(match.group(2)) + else: + raise + +def get_facility_from_priority(priority): + return int(priority/8) + +def get_level_from_priority(priority): + return priority % 8 + +class ThreadedUDPServer(SocketServer.ThreadingMixIn, SocketServer.UDPServer): + pass + +class SyslogUDPHandler(SocketServer.BaseRequestHandler): + def handle(self): + data = bytes.decode(self.request[0].strip()) + socket = self.request[1] + syslog_message = str(data) + + priority, text = split_priority_from_message(syslog_message) + facility = get_facility_from_priority(priority) + lvl = get_level_from_priority(priority) + + logentry = dict() + logentry['raw'] = syslog_message + logentry['level'] = LOG_LEVEL[lvl] + logentry['facility'] = LOG_FACILITY[facility] + logentry['text'] = text + print("Push {}".format(logentry['text'])) + logentries.put(logentry) + + +munin_config = """ +multigraph mmbtools_log +graph_title mmbTools errors and warnings +graph_order high low +graph_args --base 1000 +graph_vlabel number of entries in log during last ${{graph_period}} +graph_category mmbtools +graph_info This graph shows number of error and warning messages generated by the mmbTools +errors.info Errors +errors.label Errors +errors.min 0 +errors.type ABSOLUTE +warnings.info Warnings +warnings.label Warnings +warnings.min 0 +warnings.type ABSOLUTE +""" + +class MuninHandler(SocketServer.BaseRequestHandler): + def handle(self): + global error_count, warning_count + + self.data = self.request.recv(128).strip() + print self.data + if self.data == 'config': + self.request.sendall(munin_config) + elif self.data == 'values': + self.update_log_counts() + + values = ["multigraph mmbtools_log"] + values += ["errors.value {}".format(error_count)] + values += ["warnings.value {}".format(warning_count)] + + error_count = 0 + warning_count = 0 + self.request.sendall("\n".join(values) + "\n") + + def update_log_counts(self): + global error_count, warning_count + try: + while True: + entry = logentries.get_nowait() + print("Pop {}".format(entry['text'])) + + if entry['level'] in ['Emergency', 'Alert', 'Critical', 'Error']: + error_count += 1 + elif entry['level'] == 'Warning': + warning_count += 1 + except Queue.Empty: + print("No data in queue") + + +if __name__ == "__main__": + try: + print("Startup") + + logentries = Queue.Queue(10000) + error_count = 0 + warning_count = 0 + muninserver = None + logudp = None + + print("Init Syslog handler") + logudp = ThreadedUDPServer((HOST, syslogport), SyslogUDPHandler) + print("Create Syslog thread") + syslogthread = threading.Thread(target=logudp.serve_forever, kwargs={'poll_interval': 0.1}) + + print("Init Munin handler") + muninserver = SocketServer.TCPServer((HOST, muninport), MuninHandler) + + print("Starting handlers") + + syslogthread.start() + muninserver.serve_forever() + + except KeyboardInterrupt: + print("Ctrl-C received") + finally: + if muninserver is not None: + muninserver.shutdown() + if logudp is not None: + logudp.shutdown() + print("Waiting for thread to finish") + syslogthread.join() + + print("Quitting") diff --git a/syslog_munin/syslog_munin.supervisor.conf b/syslog_munin/syslog_munin.supervisor.conf new file mode 100644 index 0000000..dc43bdd --- /dev/null +++ b/syslog_munin/syslog_munin.supervisor.conf @@ -0,0 +1,7 @@ +[program:syslog_munin] +command=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.py +directory=/home/bram/dab/mmbtools-aux/syslog_munin +autostart=true +autorestart=true +stderr_logfile=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.err.log +stdout_logfile=/home/bram/dab/mmbtools-aux/syslog_munin/syslog_munin.out.log diff --git a/syslog_munin/syslog_munin_plugin.sh b/syslog_munin/syslog_munin_plugin.sh new file mode 100755 index 0000000..97e0951 --- /dev/null +++ b/syslog_munin/syslog_munin_plugin.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# +# This is the munin plugin that connects to +# ./syslog_munin.py and asks for config and/or +# values + +if [ "$1" == "" ]; then + echo "argument expected" + exit 1 +else + echo $1 | nc 127.0.0.1 51401 +fi |