My original script for monitoring had a few issues. While it worked really well for the actual push notifications, it didn’t seem to be able to survive log files being rotated.
I initially settled for a solution where I just scheduled a restart of my service every night after the logfiles rotated, but i hate loose ends, so I rewrote the script, this time using inotify instead of just calling “tail -F” in a subprocess.
The new script requires pyinotify, which is a python interface for inotify.
pyinotify is available as a dpkg package, python-pyinotify, which can be installed like this
sudo apt-get install python-pyinotify
If you prefer, pyinotify can also be installed from pip, instructions are on the pyinotify site.
The new script looks like this, and is available at Github
#!/usr/bin/env python
import re
import os
import urllib
import httplib
import logging
import pyinotify
from threading import Thread, Event
from socket import gethostname
try:
from queue import Queue
except ImportError:
from Queue import Queue
class PushoverNotification(object):
_token = "PUSHOVER_API_TOKEN"
_user = "PUSHOVER_USER_TOKEN"
@property
def token(self):
return self._token
@token.setter
def token(self, value):
self._token = value
@property
def user(self):
return self._user
@user.setter
def user(self, value):
self._user = value
def __init__(self):
pass
def send_notification(self, message, token=None, user=None):
if token is None:
token = self.token
if user is None:
user = self.user
conn = httplib.HTTPSConnection("api.pushover.net:443")
conn.request("POST", "/1/messages.json",
urllib.urlencode({
"token": token,
"user": user,
"message": message,
}), {"Content-type": "application/x-www-form-urlencoded"})
resp = conn.getresponse()
logging.debug("sent notification for user=%s, message=%s" % (user, message))
if resp.status != 200:
logging.error(resp.to_s())
raise Exception("Error : %d " % resp.status)
# the event handlers:
class PTmp(pyinotify.ProcessEvent):
def __init__(self, filename, filehandle, patterns, queue, prefix=""):
self._filename = filename
self._filehandle = filehandle
self._patterns = patterns
self._queue = queue
self._prefix = prefix
def scan_line(self, line):
for m in self._patterns:
match = m.match(line)
if match is not None:
logging.debug("match on '%s'" % line)
self._queue.put("%s:%s" % (self._prefix, line))
break
def process_IN_MODIFY(self, event):
if self._filename not in os.path.join(event.path, event.name):
return
else:
self.scan_line(self._filehandle.readline())
def process_IN_MOVE_SELF(self, event):
logging.debug("The file moved! Continuing to read from that, until a new one is created..")
def process_IN_CREATE(self, event):
if self._filename in os.path.join(event.path, event.name):
self._filehandle.close
self._filehandle = open(self._filename, 'r')
# catch up, in case lines were written during the time we were re-opening:
logging.debug("My file was created! I'm now catching up with lines in the newly created file.")
for line in self._filehandle.readlines():
logging.debug("read %s" % line)
self.scan_line(line)
# then skip to the end, and wait for more IN_MODIFY events
self._filehandle.seek(0, 2)
return
class LogWatcher(object):
_patterns = list()
class WatcherThread(Thread):
def __init__(self, queue, watchManager, filename, patterns, prefix=""):
super(LogWatcher.WatcherThread, self).__init__()
self._queue = queue
self._filename = filename
self._patterns = patterns
self._prefix = prefix
self._watchManager = watchManager
self._stop = Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def run(self):
dirmask = pyinotify.IN_MODIFY | pyinotify.IN_DELETE | pyinotify.IN_MOVE_SELF | pyinotify.IN_CREATE
index = self._filename.rfind('/')
self._watchManager.add_watch(self._filename[:index], dirmask)
fh = open(self._filename, 'r')
fh.seek(0, 2)
notifier = pyinotify.Notifier(self._watchManager, PTmp(self._filename, fh, self._patterns, self._queue, self._prefix))
while not self.stopped():
try:
notifier.process_events()
if notifier.check_events():
notifier.read_events()
except KeyboardInterrupt:
break
# cleanup: stop the inotify, and close the file handle:
notifier.stop()
fh.close()
def __init__(self):
self._queue = Queue()
self._threads = []
def stop(self):
for t in self._threads:
t.stop()
t.join()
def monitor_file(self, filename, watchManager, patterns, prefix=""):
monitored_patterns = list()
for pat in patterns:
if isinstance(pat, str):
p = re.compile(pat)
if p is not None:
monitored_patterns.append(p)
else:
raise Exception("Error compiling pattern:\"%s\"" % pat)
else:
monitored_patterns.append(pat)
t = LogWatcher.WatcherThread(self._queue, watchManager, filename, monitored_patterns, prefix)
t.start()
self._threads.append(t)
def iterate_matches(self):
try:
r = self._queue.get()
logging.debug("got %s from queue" % r)
self._queue.task_done()
return r
except KeyboardInterrupt:
pass
def main():
try:
logging.basicConfig(level=logging.WARNING, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%y-%m-%d %H:%M')
p = PushoverNotification()
l = LogWatcher()
pattern = re.compile("^.*sshd\\[[0-9]+\\]: [^ ]+ session opened for user ([^ ]+).*$")
wm = pyinotify.WatchManager()
l.monitor_file('/var/log/auth.log', wm, (pattern,), "%s:" % gethostname())
while True:
try:
user = l.iterate_matches()
if user is not None:
p.send_notification("%s" % user)
except KeyboardInterrupt:
break
except Exception as e:
logging.exception(e)
finally:
l.stop()
if __name__ == '__main__':
main()