diff --git a/.travis.yml b/.travis.yml index 8cfeeff1..4d312575 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,6 @@ script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: +# Coverage config file must be .coveragerc for coveralls + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cp -v .travis_coveragerc .coveragerc; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi diff --git a/.travis_coveragerc b/.travis_coveragerc index ac4a15d5..49fc3134 100644 --- a/.travis_coveragerc +++ b/.travis_coveragerc @@ -4,3 +4,4 @@ branch = True omit = /usr/* /home/travis/virtualenv/* + fail2ban/server/filtersystemd.py diff --git a/MANIFEST b/MANIFEST index 8ad73b5c..0722b46e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -27,6 +27,7 @@ fail2ban/server/filter.py fail2ban/server/filterpyinotify.py fail2ban/server/filtergamin.py fail2ban/server/filterpoll.py +fail2ban/server/filtersystemd.py fail2ban/server/iso8601.py fail2ban/server/server.py fail2ban/server/actions.py diff --git a/README.md b/README.md index 7eb380a3..9a34feca 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Optional: - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify) - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) +- [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) To install, just do: diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index a67b5347..ec2dc875 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -30,12 +30,18 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" -import getopt, sys, time, logging, os, locale +import getopt, sys, time, logging, os, locale, shlex from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError +try: + from systemd import journal + from fail2ban.server.filtersystemd import FilterSystemd +except ImportError: + journal = None + from fail2ban.version import version from fail2ban.client.configparserinc import SafeConfigParserWithIncludes from fail2ban.server.filter import Filter @@ -69,6 +75,7 @@ def get_opt_parser(): LOG: string a string representing a log line filename path to a log file (/var/log/auth.log) + "systemd-journal" search systemd journal (systemd-python required) REGEX: string a string representing a 'failregex' @@ -85,6 +92,9 @@ IGNOREREGEX: help="File encoding. Default: system locale"), Option("-L", "--maxlines", type=int, default=0, help="maxlines for multi-line regex"), + Option("-m", "--journalmatch", + help="journalctl style matches overriding filter file. " + "\"systemd-journal\" only"), Option("-v", "--verbose", action='store_true', help="Be verbose in output"), @@ -167,6 +177,8 @@ class Fail2banRegex(object): self._print_all_missed = opts.print_all_missed self._print_all_ignored = opts.print_all_ignored self._maxlines_set = False # so we allow to override maxlines in cmdline + self._journalmatch = None + if opts.encoding: self.encoding = opts.encoding else: @@ -179,6 +191,8 @@ class Fail2banRegex(object): if opts.maxlines: self.setMaxLines(opts.maxlines) + if opts.journalmatch is not None: + self.setJournalMatch(opts.journalmatch.split()) def setMaxLines(self, v): @@ -186,6 +200,9 @@ class Fail2banRegex(object): self._filter.setMaxLines(int(v)) self._maxlines_set = True + def setJournalMatch(self, v): + if self._journalmatch is None: + self._journalmatch = v def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) @@ -222,6 +239,14 @@ class Fail2banRegex(object): print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ "read from %(value)s" % locals() return False + # Read out and set possible value for journalmatch + try: + journalmatch = reader.get("Init", "journalmatch") + except (NoSectionError, NoOptionError): + # No [Init].journalmatch found. + pass + else: + self.setJournalMatch(shlex.split(journalmatch)) else: print "Use %11s line : %s" % (regex, shortstr(value)) regex_values = [RegexStat(value)] @@ -425,6 +450,32 @@ if __name__ == "__main__": except IOError, e: print e sys.exit(-1) + elif cmd_log == "systemd-journal": + if not journal: + print "Error: systemd library not found. Exiting..." + sys.exit(-1) + myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + journalmatch = fail2banRegex._journalmatch + if journalmatch: + try: + for element in journalmatch: + if element == "+": + myjournal.add_disjunction() + else: + myjournal.add_match(element) + except ValueError: + print "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) + sys.exit(-1) + print "Use journal match : %s" % " ".join(journalmatch) + test_lines = [] + while True: + try: + entry = myjournal.get_next() + except OSError: + continue + if not entry: + break + test_lines.append(FilterSystemd.formatJournalEntry(entry)) else: print "Use single line : %s" % shortstr(cmd_log) test_lines = [ cmd_log ] diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index dd4c35ba..f4ba4a23 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -24,3 +24,11 @@ failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; # Values: TEXT # ignoreregex = + +[Init] + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backends +# Values: TEXT +# +journalmatch = _SYSTEMD_UNIT=dovecot.service diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f92c3619..7bed801f 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -21,3 +21,11 @@ failregex = reject: RCPT from (.*)\[\]: 554 # Values: TEXT # ignoreregex = + +[Init] + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backends +# Values: TEXT +# +journalmatch = _SYSTEMD_UNIT=postfix.service diff --git a/config/filter.d/recidive.conf b/config/filter.d/recidive.conf index bbb48008..a8778b6d 100644 --- a/config/filter.d/recidive.conf +++ b/config/filter.d/recidive.conf @@ -36,3 +36,11 @@ failregex = fail2ban.actions:\s+WARNING\s+\[(?:.*)\]\s+Ban\s+ # # Ignore our own bans, to keep our counts exact. ignoreregex = fail2ban.actions:\s+WARNING\s+\[%(_jailname)s\]\s+Ban\s+ + +[Init] + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backends +# Values: TEXT +# +journalmatch = _SYSTEMD_UNIT=fail2ban.service diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf index 58698ced..ce2a290d 100644 --- a/config/filter.d/sshd-ddos.conf +++ b/config/filter.d/sshd-ddos.conf @@ -34,3 +34,11 @@ failregex = ^%(__prefix_line)sDid not receive identification string from \ # Values: TEXT # ignoreregex = + +[Init] + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backend +# Values: TEXT +# +journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 0d865afc..881bdaf6 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -48,3 +48,9 @@ ignoreregex = # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 10 + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backend +# Values: TEXT +# +journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd diff --git a/config/jail.conf b/config/jail.conf index af982f3c..070ec6fd 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -42,7 +42,7 @@ findtime = 600 maxretry = 5 # "backend" specifies the backend used to get files modification. -# Available options are "pyinotify", "gamin", "polling" and "auto". +# Available options are "pyinotify", "gamin", "polling", "systemd" and "auto". # This option can be overridden in each jail as well. # # pyinotify: requires pyinotify (a file alteration monitor) to be installed. @@ -50,6 +50,9 @@ maxretry = 5 # gamin: requires Gamin (a file alteration monitor) to be installed. # If Gamin is not installed, Fail2ban will use auto. # polling: uses a polling algorithm which does not require external libraries. +# systemd: uses systemd python library to access the systemd journal. +# Specifying "logpath" is not valid for this backend. +# See "journalmatch" in the jails associated filter config # auto: will try to use the following backends, in order: # pyinotify, gamin, polling. backend = auto diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 542611d4..0c5e31fb 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -113,6 +113,12 @@ class Beautifier: elif inC[2] == "logencoding": msg = "Current log encoding is set to:\n" msg = msg + response + elif inC[2] in ("journalmatch", "addjournalmatch", "deljournalmatch"): + if len(response) == 0: + msg = "No journal match filter set" + else: + msg = "Current match filter:\n" + msg += ' + '.join(" ".join(res) for res in response) elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"): if len(response) == 0: msg = "No IP address/network is ignored" diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index b9146f44..e5fe8f4f 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os +import logging, os, shlex from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. @@ -56,5 +56,11 @@ class FilterReader(DefinitionInitConfigReader): if self._initOpts: if 'maxlines' in self._initOpts: stream.append(["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) + # Do not send a command if the match is empty. + if self._initOpts.get("journalmatch", '') != '': + for match in self._initOpts["journalmatch"].split("\n"): + stream.append( + ["set", self._jailName, "addjournalmatch"] + + shlex.split(match)) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 26882964..def4fcf1 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -36,6 +36,8 @@ logSys = logging.getLogger(__name__) class JailReader(ConfigReader): optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionExtractRE = re.compile( + r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) @@ -155,46 +157,13 @@ class JailReader(ConfigReader): #@staticmethod def extractOptions(option): - m = JailReader.optionCRE.match(option) - d = dict() - mgroups = m.groups() - if len(mgroups) == 2: - option_name, option_opts = mgroups - elif len(mgroups) == 1: - option_name, option_opts = mgroups[0], None - else: - raise ValueError("While reading option %s we should have got up to " - "2 groups. Got: %r" % (option, mgroups)) - if not option_opts is None: - # Huge bad hack :( This method really sucks. TODO Reimplement it. - options = "" - escapeChar = None - allowComma = False - for c in option_opts: - if c in ('"', "'") and not allowComma: - # Start - escapeChar = c - allowComma = True - elif c == escapeChar: - # End - escapeChar = None - allowComma = False - else: - if c == ',' and allowComma: - options += "" - else: - options += c - - # Split using , - optionsSplit = options.split(',') - # Replace the tag with , - optionsSplit = [n.replace("", ',') for n in optionsSplit] - - for param in optionsSplit: - p = param.split('=') - try: - d[p[0].strip()] = p[1].strip() - except IndexError: - logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) - return [option_name, d] + option_name, optstr = JailReader.optionCRE.match(option).groups() + option_opts = dict() + if optstr: + for optmatch in JailReader.optionExtractRE.finditer(optstr): + opt = optmatch.group(1) + value = [ + val for val in optmatch.group(2,3,4) if val is not None][0] + option_opts[opt.strip()] = value.strip() + return option_name, option_opts extractOptions = staticmethod(extractOptions) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 48609f0b..e0fb018d 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["set addlogpath ", "adds to the monitoring list of "], ["set dellogpath ", "removes from the monitoring list of "], ["set logencoding ", "sets the of the log files for "], +["set addjournalmatch ", "adds to the journal filter of "], +["set deljournalmatch ", "removes from the journal filter of "], ["set addfailregex ", "adds the regular expression which must match failures for "], ["set delfailregex ", "removes the regular expression at for failregex"], ["set addignoreregex ", "adds the regular expression which should match pattern to exclude for "], @@ -79,6 +81,7 @@ protocol = [ ['', "JAIL INFORMATION", ""], ["get logpath", "gets the list of the monitored files for "], ["get logencoding ", "gets the of the log files for "], +["get journalmatch", "gets the journal filter match for "], ["get ignoreip", "gets the list of ignored IP addresses for "], ["get failregex", "gets the list of regular expressions which matches the failures for "], ["get ignoreregex", "gets the list of regular expressions which matches patterns to ignore for "], diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d173f015..2e161d73 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -647,6 +647,21 @@ class FileContainer: self.__handler = None +## +# JournalFilter class. +# +# Base interface class for systemd journal filters + +class JournalFilter(Filter): # pragma: systemd no cover + + def addJournalMatch(self, match): # pragma: no cover - Base class, not used + pass + + def delJournalMatch(self, match): # pragma: no cover - Base class, not used + pass + + def getJournalMatch(self, match): # pragma: no cover - Base class, not used + return [] ## # Utils class for DNS and IP handling. diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py new file mode 100644 index 00000000..c0fe9a87 --- /dev/null +++ b/fail2ban/server/filtersystemd.py @@ -0,0 +1,263 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +__author__ = "Steven Hiscocks" +__copyright__ = "Copyright (c) 2013 Steven Hiscocks" +__license__ = "GPL" + +import logging, datetime +from distutils.version import LooseVersion + +from systemd import journal +if LooseVersion(getattr(journal, '__version__', "0")) < '204': + raise ImportError("Fail2Ban requires systemd >= 204") + +from failmanager import FailManagerEmpty +from filter import JournalFilter +from mytime import MyTime + + +# Gets the instance of the logger. +logSys = logging.getLogger("fail2ban.filter") + +## +# Journal reader class. +# +# This class reads from systemd journal and detects login failures or anything +# else that matches a given regular expression. This class is instantiated by +# a Jail object. + +class FilterSystemd(JournalFilter): # pragma: systemd no cover + ## + # Constructor. + # + # Initialize the filter object with default values. + # @param jail the jail object + + def __init__(self, jail, **kwargs): + JournalFilter.__init__(self, jail, **kwargs) + self.__modified = False + # Initialise systemd-journal connection + self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x}) + self.__matches = [] + logSys.debug("Created FilterSystemd") + + + ## + # Add a journal match filters from list structure + # + # @param matches list structure with journal matches + + def _addJournalMatches(self, matches): + if self.__matches: + self.__journal.add_disjunction() # Add OR + newMatches = [] + for match in matches: + newMatches.append([]) + for match_element in match: + self.__journal.add_match(match_element) + newMatches[-1].append(match_element) + self.__journal.add_disjunction() + self.__matches.extend(newMatches) + + ## + # Add a journal match filter + # + # @param match journalctl syntax matches in list structure + + def addJournalMatch(self, match): + newMatches = [[]] + for match_element in match: + if match_element == "+": + newMatches.append([]) + else: + newMatches[-1].append(match_element) + try: + self._addJournalMatches(newMatches) + except ValueError: + logSys.error( + "Error adding journal match for: %r", " ".join(match)) + self.resetJournalMatches() + raise + else: + logSys.info("Added journal match for: %r", " ".join(match)) + ## + # Reset a journal match filter called on removal or failure + # + # @return None + + def resetJournalMatches(self): + self.__journal.flush_matches() + logSys.debug("Flushed all journal matches") + match_copy = self.__matches[:] + self.__matches = [] + try: + self._addJournalMatches(match_copy) + except ValueError: + logSys.error("Error restoring journal matches") + raise + else: + logSys.debug("Journal matches restored") + + ## + # Delete a journal match filter + # + # @param match journalctl syntax matches + + def delJournalMatch(self, match): + if match in self.__matches: + del self.__matches[self.__matches.index(match)] + self.resetJournalMatches() + else: + raise ValueError("Match not found") + logSys.info("Removed journal match for: %r" % " ".join(match)) + + ## + # Get current journal match filter + # + # @return journalctl syntax matches + + def getJournalMatch(self): + return self.__matches + + ## + # Join group of log elements which may be a mix of bytes and strings + # + # @param elements list of strings and bytes + # @return elements joined as string + + @staticmethod + def _joinStrAndBytes(elements): + strElements = [] + for element in elements: + if isinstance(element, str): + strElements.append(element) + else: + strElements.append(str(element, errors='ignore')) + return " ".join(strElements) + + ## + # Format journal log entry into syslog style + # + # @param entry systemd journal entry dict + # @return format log line + + @staticmethod + def formatJournalEntry(logentry): + logelements = [logentry.get('_SOURCE_REALTIME_TIMESTAMP', + logentry.get('__REALTIME_TIMESTAMP')).isoformat()] + if logentry.get('_HOSTNAME'): + logelements.append(logentry['_HOSTNAME']) + if logentry.get('SYSLOG_IDENTIFIER'): + logelements.append(logentry['SYSLOG_IDENTIFIER']) + if logentry.get('_PID'): + logelements[-1] += ("[%i]" % logentry['_PID']) + logelements[-1] += ":" + elif logentry.get('_COMM'): + logelements.append(logentry['_COMM']) + if logentry.get('_PID'): + logelements[-1] += ("[%i]" % logentry['_PID']) + logelements[-1] += ":" + if logelements[-1] == "kernel:": + if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: + monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') + else: + monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] + logelements.append("[%12.6f]" % monotonic.total_seconds()) + if isinstance(logentry.get('MESSAGE',''), list): + logelements.append(" ".join(logentry['MESSAGE'])) + else: + logelements.append(logentry.get('MESSAGE', '')) + + try: + logline = u" ".join(logelements) + u"\n" + except UnicodeDecodeError: + # Python 2, so treat as string + logline = " ".join([str(logline) for logline in logelements]) + "\n" + except TypeError: + # Python 3, one or more elements bytes + logSys.warning("Error decoding log elements from journal: %s" % + repr(logelements)) + logline = self._joinStrAndBytes(logelements) + "\n" + + logSys.debug("Read systemd journal entry: %s" % repr(logline)) + return logline + + ## + # Main loop. + # + # Peridocily check for new journal entries matching the filter and + # handover to FailManager + + def run(self): + self.setActive(True) + + # Seek to now - findtime in journal + start_time = datetime.datetime.now() - \ + datetime.timedelta(seconds=int(self.getFindTime())) + self.__journal.seek_realtime(start_time) + # Move back one entry to ensure do not end up in dead space + # if start time beyond end of journal + try: + self.__journal.get_previous() + except OSError: + pass # Reading failure, so safe to ignore + + while self._isActive(): + if not self.getIdle(): + while self._isActive(): + try: + logentry = self.__journal.get_next() + except OSError: + logSys.warning( + "Error reading line from systemd journal") + continue + if logentry: + self.processLineAndAdd( + self.formatJournalEntry(logentry)) + self.__modified = True + else: + break + if self.__modified: + try: + while True: + ticket = self.failManager.toBan() + self.jail.putFailTicket(ticket) + except FailManagerEmpty: + self.failManager.cleanup(MyTime.time()) + self.dateDetector.sortTemplate() + self.__modified = False + self.__journal.wait(self.getSleepTime()) + logSys.debug((self.jail is not None and self.jail.getName() + or "jailless") +" filter terminated") + return True + + ## + # Get the status of the filter. + # + # Get some informations about the filter state such as the total + # number of failures. + # @return a list with tuple + + def status(self): + ret = JournalFilter.status(self) + ret.append(("Journal matches", + [" + ".join(" ".join(match) for match in self.__matches)])) + return ret diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 4df4d42e..d25fda7e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -35,7 +35,7 @@ class Jail: #Known backends. Each backend should have corresponding __initBackend method # yoh: stored in a list instead of a tuple since only # list had .index until 2.6 - _BACKENDS = ['pyinotify', 'gamin', 'polling'] + _BACKENDS = ['pyinotify', 'gamin', 'polling', 'systemd'] def __init__(self, name, backend = "auto"): self.setName(name) @@ -101,6 +101,13 @@ class Jail: from filterpyinotify import FilterPyinotify self.__filter = FilterPyinotify(self) + def _initSystemd(self): # pragma: systemd no cover + # Try to import systemd + import systemd + logSys.info("Jail '%s' uses systemd" % self.__name) + from filtersystemd import FilterSystemd + self.__filter = FilterSystemd(self) + def setName(self, name): if len(name) >= 20: logSys.warning("Jail name %r might be too long and some commands " diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index e2f5892e..00b838f5 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -26,6 +26,7 @@ __license__ = "GPL" from threading import Lock, RLock from jails import Jails +from filter import FileFilter, JournalFilter from transmitter import Transmitter from asyncserver import AsyncServer from asyncserver import AsyncServerException @@ -169,20 +170,51 @@ class Server: return self.__jails.getFilter(name).getIgnoreIP() def addLogPath(self, name, fileName): - self.__jails.getFilter(name).addLogPath(fileName) + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, FileFilter): + filter_.addLogPath(fileName) def delLogPath(self, name, fileName): - self.__jails.getFilter(name).delLogPath(fileName) + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, FileFilter): + filter_.delLogPath(fileName) def getLogPath(self, name): - return [m.getFileName() - for m in self.__jails.getFilter(name).getLogPath()] + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, FileFilter): + return [m.getFileName() + for m in filter_.getLogPath()] + else: # pragma: systemd no cover + logSys.info("Jail %s is not a FileFilter instance" % name) + return [] + + def addJournalMatch(self, name, match): # pragma: systemd no cover + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, JournalFilter): + filter_.addJournalMatch(match) + + def delJournalMatch(self, name, match): # pragma: systemd no cover + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, JournalFilter): + filter_.delJournalMatch(match) + + def getJournalMatch(self, name): # pragma: systemd no cover + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, JournalFilter): + return filter_.getJournalMatch() + else: + logSys.info("Jail %s is not a JournalFilter instance" % name) + return [] def setLogEncoding(self, name, encoding): - return self.__jails.getFilter(name).setLogEncoding(encoding) + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, FileFilter): + filter_.setLogEncoding(encoding) def getLogEncoding(self, name): - return self.__jails.getFilter(name).getLogEncoding() + filter_ = self.__jails.getFilter(name) + if isinstance(filter_, FileFilter): + return filter_.getLogEncoding() def setFindTime(self, name, value): self.__jails.getFilter(name).setFindTime(value) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 22681bf7..39c74452 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -144,6 +144,14 @@ class Transmitter: value = command[2] self.__server.setLogEncoding(name, value) return self.__server.getLogEncoding(name) + elif command[1] == "addjournalmatch": # pragma: systemd no cover + value = command[2:] + self.__server.addJournalMatch(name, value) + return self.__server.getJournalMatch(name) + elif command[1] == "deljournalmatch": # pragma: systemd no cover + value = command[2:] + self.__server.delJournalMatch(name, value) + return self.__server.getJournalMatch(name) elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value) @@ -250,6 +258,8 @@ class Transmitter: return self.__server.getLogPath(name) elif command[1] == "logencoding": return self.__server.getLogEncoding(name) + elif command[1] == "journalmatch": # pragma: systemd no cover + return self.__server.getJournalMatch(name) elif command[1] == "ignoreip": return self.__server.getIgnoreIP(name) elif command[1] == "failregex": diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e4a6e463..4267922c 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -145,11 +145,35 @@ class JailReaderTest(unittest.TestCase): self.assertEqual(jail.getName(), 'sshd') def testSplitOption(self): - action = "mail-whois[name=SSH]" - expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.extractOptions(action) - self.assertEquals(expected, result) + # Simple example + option = "mail-whois[name=SSH]" + expected = ('mail-whois', {'name': 'SSH'}) + result = JailReader.extractOptions(option) + self.assertEqual(expected, result) + # Empty option + option = "abc[]" + expected = ('abc', {}) + result = JailReader.extractOptions(option) + self.assertEqual(expected, result) + + # More complex examples + option = 'option[opt01=abc,opt02="123",opt03="with=okay?",opt04="andwith,okay...",opt05="how about spaces",opt06="single\'in\'double",opt07=\'double"in"single\', opt08= leave some space, opt09=one for luck, opt10=, opt11=]' + expected = ('option', { + 'opt01': "abc", + 'opt02': "123", + 'opt03': "with=okay?", + 'opt04': "andwith,okay...", + 'opt05': "how about spaces", + 'opt06': "single'in'double", + 'opt07': "double\"in\"single", + 'opt08': "leave some space", + 'opt09': "one for luck", + 'opt10': "", + 'opt11': "", + }) + result = JailReader.extractOptions(option) + self.assertEqual(expected, result) class FilterReaderTest(unittest.TestCase): @@ -173,7 +197,12 @@ class FilterReaderTest(unittest.TestCase): "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"], - ['set', 'testcase01', 'maxlines', "1"]] + ['set', 'testcase01', 'addjournalmatch', + "_COMM=sshd", "+", "_SYSTEMD_UNIT=sshd.service", "_UID=0"], + ['set', 'testcase01', 'addjournalmatch', + "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], + ['set', 'testcase01', 'maxlines', "1"], # Last for overide test + ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf index c549572d..8bc4261d 100644 --- a/fail2ban/tests/files/filter.d/testcase01.conf +++ b/fail2ban/tests/files/filter.d/testcase01.conf @@ -36,3 +36,10 @@ ignoreregex = ^.+ john from host 192.168.1.1\s*$ [Init] # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 1 + +# Option: journalmatch +# Notes.: systemd journalctl style match filter for journal based backends +# Values: TEXT +# +journalmatch = _COMM=sshd + _SYSTEMD_UNIT=sshd.service _UID=0 + "FIELD= with spaces " + AFIELD=" with + char and spaces" diff --git a/fail2ban/tests/files/testcase-journal.log b/fail2ban/tests/files/testcase-journal.log new file mode 100644 index 00000000..720a3130 --- /dev/null +++ b/fail2ban/tests/files/testcase-journal.log @@ -0,0 +1,19 @@ +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from failed.dns.ch +error: PAM: Authentication failure for kevin from failed.dns.ch +error: PAM: Authentication failure for kevin from failed.dns.ch +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 193.168.0.128 +error: PAM: Authentication failure for kevin from 87.142.124.10 +error: PAM: Authentication failure for kevin from 87.142.124.10 +error: PAM: Authentication failure for kevin from 87.142.124.10 +error: PAM: Authentication failure for kevin from 87.142.124.10 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 92beb0c5..f7c02183 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -29,6 +29,11 @@ import sys import time import tempfile +try: + from systemd import journal +except ImportError: + journal = None + from fail2ban.server.jail import Jail from fail2ban.server.filterpoll import FilterPoll from fail2ban.server.filter import FileFilter, DNSUtils @@ -160,6 +165,34 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line time.sleep(0.1) return fout +def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # pragma: systemd no cover + """Copy lines from one file to systemd journal + + Returns None + """ + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + fin = open(in_, 'r') + else: + fin = in_ + # Required for filtering + fields.update({"SYSLOG_IDENTIFIER": "fail2ban-testcases", + "PRIORITY": "7", + }) + # Skip + for i in xrange(skip): + _ = fin.readline() + # Read/Write + i = 0 + while n is None or i < n: + l = fin.readline() + if terminal_line is not None and l == terminal_line: + break + journal.send(MESSAGE=l.strip(), **fields) + i += 1 + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + # Opened earlier, therefore must close it + fin.close() + # # Actual tests # @@ -574,6 +607,129 @@ def get_monitor_failures_testcase(Filter_): % (Filter_.__name__, testclass_name) # 'tempfile') return MonitorFailures +def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover + """Generator of TestCase's for journal based filters/backends + """ + + class MonitorJournalFailures(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log") + self.jail = DummyJail() + self.filter = Filter_(self.jail) + # UUID used to ensure that only meeages generated + # as part of this test are picked up by the filter + import uuid + self.test_uuid = str(uuid.uuid4()) + self.name = "monitorjournalfailures-%s" % self.test_uuid + self.filter.addJournalMatch([ + "SYSLOG_IDENTIFIER=fail2ban-testcases", + "TEST_FIELD=1", + "TEST_UUID=%s" % self.test_uuid]) + self.filter.addJournalMatch([ + "SYSLOG_IDENTIFIER=fail2ban-testcases", + "TEST_FIELD=2", + "TEST_UUID=%s" % self.test_uuid]) + self.journal_fields = { + 'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid} + self.filter.setActive(True) + self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") + self.filter.start() + + def tearDown(self): + self.filter.stop() + self.filter.join() # wait for the thread to terminate + pass + + def __str__(self): + return "MonitorJournalFailures%s(%s)" \ + % (Filter_, hasattr(self, 'name') and self.name or 'tempfile') + + def isFilled(self, delay=2.): + """Wait up to `delay` sec to assure that it was modified or not + """ + time0 = time.time() + while time.time() < time0 + delay: + if len(self.jail): + return True + time.sleep(0.1) + return False + + def isEmpty(self, delay=0.4): + # shorter wait time for not modified status + return not self.isFilled(delay) + + def assert_correct_ban(self, test_ip, test_attempts): + self.assertTrue(self.isFilled(10)) # give Filter a chance to react + ticket = self.jail.getFailTicket() + + attempts = ticket.getAttempt() + ip = ticket.getIP() + matches = ticket.getMatches() + + self.assertEqual(ip, test_ip) + self.assertEqual(attempts, test_attempts) + + def test_grow_file(self): + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + + # Now let's feed it with entries from the file + _copy_lines_to_journal( + self.test_file, self.journal_fields, n=2) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + # and our dummy jail is empty as well + self.assertFalse(len(self.jail)) + # since it should have not been enough + + _copy_lines_to_journal( + self.test_file, self.journal_fields, skip=2, n=3) + self.assertTrue(self.isFilled(6)) + # so we sleep for up to 6 sec for it not to become empty, + # and meanwhile pass to other thread(s) and filter should + # have gathered new failures and passed them into the + # DummyJail + self.assertEqual(len(self.jail), 1) + # and there should be no "stuck" ticket in failManager + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + self.assert_correct_ban("193.168.0.128", 3) + self.assertEqual(len(self.jail), 0) + + # Lets read some more to check it bans again + _copy_lines_to_journal( + self.test_file, self.journal_fields, skip=5, n=4) + self.assert_correct_ban("193.168.0.128", 3) + + def test_delJournalMatch(self): + # Smoke test for removing of match + + # basic full test + _copy_lines_to_journal( + self.test_file, self.journal_fields, n=5) + self.assert_correct_ban("193.168.0.128", 3) + + # and now remove the JournalMatch + self.filter.delJournalMatch([ + "SYSLOG_IDENTIFIER=fail2ban-testcases", + "TEST_FIELD=1", + "TEST_UUID=%s" % self.test_uuid]) + + _copy_lines_to_journal( + self.test_file, self.journal_fields, n=5, skip=5) + # so we should get no more failures detected + self.assertTrue(self.isEmpty(2)) + + # but then if we add it back again + self.filter.addJournalMatch([ + "SYSLOG_IDENTIFIER=fail2ban-testcases", + "TEST_FIELD=1", + "TEST_UUID=%s" % self.test_uuid]) + self.assert_correct_ban("193.168.0.128", 4) + _copy_lines_to_journal( + self.test_file, self.journal_fields, n=6, skip=10) + # we should detect the failures + self.assertTrue(self.isFilled(6)) + + return MonitorJournalFailures class GetFailures(unittest.TestCase): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 113bd0b2..f32f73c6 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -29,6 +29,10 @@ import unittest, socket, time, tempfile, os, locale from fail2ban.server.server import Server from fail2ban.server.jail import Jail from fail2ban.exceptions import UnknownJailException +try: + from fail2ban.server import filtersystemd +except ImportError: + filtersystemd = None TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -499,6 +503,76 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + if filtersystemd: # pragma: systemd no cover + def testJournalMatch(self): + jailName = "TestJail2" + self.server.addJail(jailName, "systemd") + values = [ + "_SYSTEMD_UNIT=sshd.service", + "TEST_FIELD1=ABC", + "_HOSTNAME=example.com", + ] + for n, value in enumerate(values): + self.assertEqual( + self.transm.proceed( + ["set", jailName, "addjournalmatch", value]), + (0, [[val] for val in values[:n+1]])) + for n, value in enumerate(values): + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch", value]), + (0, [[val] for val in values[n+1:]])) + + # Try duplicates + value = "_COMM=sshd" + self.assertEqual( + self.transm.proceed( + ["set", jailName, "addjournalmatch", value]), + (0, [[value]])) + # Duplicates are accepted, as automatically OR'd, and journalctl + # also accepts them without issue. + self.assertEqual( + self.transm.proceed( + ["set", jailName, "addjournalmatch", value]), + (0, [[value], [value]])) + # Remove first instance + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch", value]), + (0, [[value]])) + # Remove second instance + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch", value]), + (0, [])) + + value = [ + "_COMM=sshd", "+", "_SYSTEMD_UNIT=sshd.service", "_UID=0"] + self.assertEqual( + self.transm.proceed( + ["set", jailName, "addjournalmatch"] + value), + (0, [["_COMM=sshd"], ["_SYSTEMD_UNIT=sshd.service", "_UID=0"]])) + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch"] + value[:1]), + (0, [["_SYSTEMD_UNIT=sshd.service", "_UID=0"]])) + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch"] + value[2:]), + (0, [])) + + # Invalid match + value = "This isn't valid!" + result = self.transm.proceed( + ["set", jailName, "addjournalmatch", value]) + self.assertTrue(isinstance(result[1], ValueError)) + + # Delete invalid match + value = "FIELD=NotPresent" + result = self.transm.proceed( + ["set", jailName, "deljournalmatch", value]) + self.assertTrue(isinstance(result[1], ValueError)) + class TransmitterLogging(TransmitterBase): def setUp(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8870c349..02bbeecb 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -208,6 +208,12 @@ def gatherTests(regexps=None, no_network=False): for Filter_ in filters: tests.addTest(unittest.makeSuite( filtertestcase.get_monitor_failures_testcase(Filter_))) + try: # pragma: systemd no cover + from fail2ban.server.filtersystemd import FilterSystemd + tests.addTest(unittest.makeSuite(filtertestcase.get_monitor_failures_journal_testcase(FilterSystemd))) + except Exception, e: # pragma: no cover + logSys.warning("I: Skipping systemd backend testing. Got exception '%s'" % e) + # Server test for logging elements which break logging used to support # testcases analysis diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 466172f5..3e99ba59 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -60,6 +60,26 @@ The following options are applicable to all jails. Their meaning is described in .TP \fBusedns\fR .PP +.SS Backends +\fBbackend\fR specifies the backend used to get files modification. This option can be overridden in each jail as well. +Available options are listed below. +.TP +\fIpyinotify\fR +requires pyinotify (a file alteration monitor) to be installed. If pyinotify is not installed, Fail2ban will use auto. +.TP +\fIgamin\fR +requires Gamin (a file alteration monitor) to be installed. If Gamin is not installed, Fail2ban will use auto. +.TP +\fIpolling\fR +uses a polling algorithm which does not require external libraries. +.TP +\fIsystemd\fR +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +.TP +\fIauto\fR +will try to use the following backends, in order: pyinotify, gamin, polling +.PP +.SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename. In the case where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplicatione.g.: .PP .nf @@ -154,6 +174,9 @@ Similar to actions, filters have an [Init] section which can be overridden in \f .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. +.TP +\fBjournalmatch\fR +specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP Filters can also have a section called [INCLUDES]. This is used to read other configuration files.