From 47dd8fb8972d9916e4a4d9672e26776cfa3e3c01 Mon Sep 17 00:00:00 2001 From: Lars Kneschke Date: Mon, 13 Jan 2014 06:04:59 +0100 Subject: [PATCH 01/41] ENH: filter for Tine 2.0 --- ChangeLog | 1 + config/filter.d/tine20.conf | 13 +++++++++++++ config/jail.conf | 11 +++++++++++ testcases/files/logs/tine20 | 2 ++ 4 files changed, 27 insertions(+) create mode 100644 config/filter.d/tine20.conf create mode 100644 testcases/files/logs/tine20 diff --git a/ChangeLog b/ChangeLog index 2ca227ff..de804b5f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -54,6 +54,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better ...: Auth fail". Thanks Marcel Dopita. Closes gh-289 - Added filter.d/ejabberd-auth - Improved ACL-handling for Asterisk + - Added filter.d/tine20 - New Features: diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf new file mode 100644 index 00000000..a878d890 --- /dev/null +++ b/config/filter.d/tine20.conf @@ -0,0 +1,13 @@ +# Fail2Ban filter for Tine 2.0 authentication +# +# Enable logging with: +# $config['info_log']='/var/log/tine20/tine20.log'; +# + +[Definition] + +failregex = Login with username .* from failed + +ignoreregex = + +# Author: mkl from Tine20.org forum diff --git a/config/jail.conf b/config/jail.conf index 5dcce02c..827e2287 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -442,6 +442,17 @@ logpath = /var/log/horde/horde.log action = iptables-multiport[name=horde, port="http,https"] maxretry = 5 +[tine20] + +enabled = false +filter = tine20 +logpath = /var/log/tine20/tine20.log +action = iptables-multiport[name=tine20, port="http,https"] +# Tine 2.0 logs are in UTC instead of my servers local time (= CET = UTC+1). Need to increase findtime by one hour (3600 + 600 = 4200). +# ( see: https://www.tine20.org/forum/viewtopic.php?f=12&t=976#p4746 ) +# findtime: The counter is set to zero if no match is found within "findtime" seconds. +findtime = 4200 +maxretry = 5 # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year diff --git a/testcases/files/logs/tine20 b/testcases/files/logs/tine20 new file mode 100644 index 00000000..87d5c8ae --- /dev/null +++ b/testcases/files/logs/tine20 @@ -0,0 +1,2 @@ +# failJSON: { "time": "2014-01-13T05:02:22", "match": true, "host": "127.0.0.1" } +78017 00cff -- none -- - 2014-01-13T05:02:22+00:00 WARN (4): Tinebase_Controller::login::106 Login with username sdfsadf from 127.0.0.1 failed (-1)! From 36d38043ba3cfe0e67f35c741809bc306e39cc1b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 22 Jan 2014 18:12:48 +1100 Subject: [PATCH 02/41] DOC: thanks Lars for the filter base and log samples --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index d983cf85..72bbdbda 100644 --- a/THANKS +++ b/THANKS @@ -53,6 +53,7 @@ Justin Shore Kévin Drapel kjohnsonecl kojiro +Lars Kneschke Lee Clemens Manuel Arostegui Ramirez Marcel Dopita From c8ae064b791edc0a83ddf9bd5e90f9ac174ed56e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 22 Jan 2014 22:16:03 +1100 Subject: [PATCH 03/41] ENH: tighten regex and change failJSON to support timezone. Closes gh-583 --- THANKS | 1 + config/filter.d/tine20.conf | 12 ++++++++++-- fail2ban/tests/files/logs/tine20 | 7 +++++++ testcases/files/logs/tine20 | 2 -- 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 fail2ban/tests/files/logs/tine20 delete mode 100644 testcases/files/logs/tine20 diff --git a/THANKS b/THANKS index 72bbdbda..cef5d506 100644 --- a/THANKS +++ b/THANKS @@ -67,6 +67,7 @@ mEDI Merijn Schering Michael C. Haller Michael Hanselmann +Mika (mkl) Nick Munger onorua Patrick Börjesson diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf index a878d890..0fa6eccd 100644 --- a/config/filter.d/tine20.conf +++ b/config/filter.d/tine20.conf @@ -6,8 +6,16 @@ [Definition] -failregex = Login with username .* from failed +failregex = ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0,2} - WARN \(\d+\): Tinebase_Controller::login::\d+ Login with username .*? from failed \(-[13]\)!$ ignoreregex = -# Author: mkl from Tine20.org forum +# Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766 +# Editor: Daniel Black +# Advisor: Lars Kneschke +# +# Usernames can contain spaces. +# +# Authentication: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Controller.php#l105 +# Logger: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Log/Formatter.php +# formatMicrotimeDiff: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Helper.php#l276 diff --git a/fail2ban/tests/files/logs/tine20 b/fail2ban/tests/files/logs/tine20 new file mode 100644 index 00000000..dbb9f424 --- /dev/null +++ b/fail2ban/tests/files/logs/tine20 @@ -0,0 +1,7 @@ +# Wrong username (-1) error +# failJSON: { "time": "2014-01-13T06:02:22", "match": true, "host": "127.0.0.1" } +78017 00cff -- none -- - 2014-01-13T05:02:22+00:00 WARN (4): Tinebase_Controller::login::106 Login with username sdfsadf from 127.0.0.1 failed (-1)! + +# Wrong password (-3) error +# failJSON: { "time": "2014-01-21T05:38:14", "match": true, "host": "127.0.0.1" } +8e035 ffff3 -- none -- - 2014-01-21T04:38:14+00:00 WARN (4): Tinebase_Controller::login::106 Login with username testuser from 127.0.0.1 failed (-3)! diff --git a/testcases/files/logs/tine20 b/testcases/files/logs/tine20 deleted file mode 100644 index 87d5c8ae..00000000 --- a/testcases/files/logs/tine20 +++ /dev/null @@ -1,2 +0,0 @@ -# failJSON: { "time": "2014-01-13T05:02:22", "match": true, "host": "127.0.0.1" } -78017 00cff -- none -- - 2014-01-13T05:02:22+00:00 WARN (4): Tinebase_Controller::login::106 Login with username sdfsadf from 127.0.0.1 failed (-1)! From 256c732bcdada8ee33c74bca7011589aa52c2ad2 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 25 Jan 2014 12:19:46 +1100 Subject: [PATCH 04/41] BF/ENH: filter pure-ftpd - re-add _daemon. Add translations _daemon was accidently removed in 89fd792dfbd09f331858d04bbe4f2d290d63b969 Added translations from source code --- config/filter.d/pure-ftpd.conf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/config/filter.d/pure-ftpd.conf b/config/filter.d/pure-ftpd.conf index e96009b2..b6d36603 100644 --- a/config/filter.d/pure-ftpd.conf +++ b/config/filter.d/pure-ftpd.conf @@ -12,13 +12,19 @@ before = common.conf [Definition] -# Error message specified in multiple languages -__errmsg = (?:Authentication failed for user|Erreur d'authentification pour l'utilisateur) +_daemon = pure-ftpd -failregex = ^%(__prefix_line)s\(.+?@\) \[WARNING\] %(__errmsg)s \[.+\]\s*$ +# Error message specified in multiple languages +__errmsg = (?:�ϥΪ�\[.*\]���ҥ���|ʹ����\[.*\]��֤ʧ��|\[.*\] kullan�c�s� i�in giri� hatal�|����������� �� ������� ������������ \[.*\]|Godkjennelse mislyktes for \[.*\]|Beh�righetskontroll misslyckas f�r anv�ndare \[.*\]|Autentifikacia uzivatela zlyhala \[.*\]|Autentificare esuata pentru utilizatorul \[.*\]|Autentica��o falhou para usu�rio \[.*\]|Autentyfikacja nie powiod�a si� dla u�ytkownika \[.*\]|Autorisatie faalde voor gebruiker \[.*\]|\[.*\] ��� ���� ����|Autenticazione falita per l'utente \[.*\]|Azonos�t�s sikertelen \[.*\] felhaszn�l�nak|\[.*\] c'est un batard, il connait pas son code|Erreur d'authentification pour l'utilisateur \[.*\]|Autentificaci�n fallida para el usuario \[.*\]|Authentication failed for user \[.*\]|Authentifizierung fehlgeschlagen f�r Benutzer \[.*\].|Godkendelse mislykkedes for \[.*\]|Autentifikace u�ivatele selhala \[.*\]) + +failregex = ^%(__prefix_line)s\(.+?@\) \[WARNING\] %(__errmsg)s\s*$ ignoreregex = # Author: Cyril Jaquier # Modified: Yaroslav Halchenko for pure-ftpd # Documentation thanks to Blake on http://www.fail2ban.org/wiki/index.php?title=Fail2ban:Community_Portal +# +# Only logs to syslog though facility can be changed configuration file/command line +# +# fgrep -r MSG_AUTH_FAILED_LOG pure-ftpd-1.0.36/src From 3c48e3f035ed46dd685016cbd14d5ab07b47b33e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 25 Jan 2014 12:22:27 +1100 Subject: [PATCH 05/41] DOC: changelog for pure-ftpd filter fixes --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index a03b1858..1a80427b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,10 +13,13 @@ ver. 0.8.13 (2014/XX/XXX) - maintaince-only-from-now-on - Fixes: - action firewallcmd-ipset had non-working actioncheck. Removed. redhat bug #1046816. + - filter pureftpd - added _daemon which got removed. Added - New Features: - Enhancements: + - filter pureftpd - added all translations of "Authentication failed for + user" ver. 0.8.12 (2014/01/22) - things-can-only-get-better ----------- From f2ddb3e3d099b212357f5b6cb0d2f373aae99e98 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 26 Jan 2014 22:03:55 +0000 Subject: [PATCH 06/41] RF: Refactor date detector and date template elements Changes include to use Python class properties, merge some date patterns, and change ISO8601 date template to DatePatternRegex class. --- fail2ban/server/datedetector.py | 145 ++++++---- fail2ban/server/datetemplate.py | 375 +++++++++++-------------- fail2ban/server/filter.py | 24 +- fail2ban/server/iso8601.py | 136 --------- fail2ban/server/strptime.py | 31 +- fail2ban/tests/datedetectortestcase.py | 12 +- fail2ban/tests/misctestcase.py | 50 ++-- fail2ban/tests/servertestcase.py | 2 - 8 files changed, 332 insertions(+), 443 deletions(-) delete mode 100644 fail2ban/server/iso8601.py diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index e5c57a31..c8e631ab 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -24,82 +24,94 @@ __license__ = "GPL" import sys, time, logging from threading import Lock -from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601 +from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class DateDetector: - +class DateDetector(object): + """Manages one or more date templates to find a date within a log line. + """ + def __init__(self): + """Initialise the date detector. + """ self.__lock = Lock() self.__templates = list() self.__known_names = set() def _appendTemplate(self, template): - name = template.getName() + name = template.name if name in self.__known_names: - raise ValueError("There is already a template with name %s" % name) + raise ValueError( + "There is already a template with name %s" % name) self.__known_names.add(name) self.__templates.append(template) - + def appendTemplate(self, template): + """Add a date template to manage and use in search of dates. + + Parameters + ---------- + template : DateTemplate or str + Can be either a `DateTemplate` instance, or a string which will + be used as the pattern for the `DatePatternRegex` template. The + template will then be added to the detector. + + Raises + ------ + ValueError + If a template already exists with the same name. + """ if isinstance(template, str): template = DatePatternRegex(template) - DateDetector._appendTemplate(self, template) + self._appendTemplate(template) def addDefaultTemplate(self): + """Add Fail2Ban's default set of date templates. + """ self.__lock.acquire() try: - # asctime with subsecond: Sun Jan 23 21:59:59.011 2005 - self.appendTemplate("%a %b %d %H:%M:%S\.%f %Y") - # asctime: Sun Jan 23 21:59:59 2005 - self.appendTemplate("%a %b %d %H:%M:%S %Y") - # asctime without year: Sun Jan 23 21:59:59 - self.appendTemplate("%a %b %d %H:%M:%S") - # standard: Jan 23 21:59:59 - self.appendTemplate("%b %d %H:%M:%S") - # proftpd date: 2005-01-23 21:59:59,333 - self.appendTemplate("%Y-%m-%d %H:%M:%S,%f") - # simple date: 2005-01-23 21:59:59 - self.appendTemplate("%Y-%m-%d %H:%M:%S") + # asctime with optional day, subsecond and/or year: + # Sun Jan 23 21:59:59.011 2005 + self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # simple date, optional subsecond (proftpd): + # 2005-01-23 21:59:59 + self.appendTemplate("%Y-%m-%d %H:%M:%S(?:,%f)?") # simple date: 2005/01/23 21:59:59 self.appendTemplate("%Y/%m/%d %H:%M:%S") # simple date too (from x11vnc): 23/01/2005 21:59:59 - self.appendTemplate("%d/%m/%Y %H:%M:%S") - # previous one but with year given by 2 digits: 23/01/05 21:59:59 + # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) - self.appendTemplate("%d/%m/%y %H:%M:%S") - # Apache format [31/Oct/2006:09:22:55 -0000] - self.appendTemplate("%d/%b/%Y:%H:%M:%S %z") - # [31/Oct/2006:09:22:55] - self.appendTemplate("%d/%b/%Y:%H:%M:%S") + self.appendTemplate("%d/%m/(?:%Y|%y) %H:%M:%S") + # Apache format optional time zone: + # [31/Oct/2006:09:22:55 -0000] + self.appendTemplate("%d/%b/%Y:%H:%M:%S(?: %z)?") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") # custom for syslog-ng 2006.12.21 06:43:20 self.appendTemplate("%Y\.%m\.%d %H:%M:%S") # named 26-Jul-2007 15:20:52.252 - self.appendTemplate("%d-%b-%Y %H:%M:%S\.%f") # roundcube 26-Jul-2007 15:20:52 +0200 - self.appendTemplate("%d-%b-%Y %H:%M:%S %z") + self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?") # 26-Jul-2007 15:20:52 self.appendTemplate("%d-%b-%Y %H:%M:%S") # 17-07-2008 17:23:25 self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 + # subseconds explicit to avoid possible %m<->%d confusion + # with previous self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f") # TAI64N template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" self.appendTemplate(template) # Epoch template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" self.appendTemplate(template) # ISO 8601 - template = DateISO8601() - template.setName("ISO 8601") - self.appendTemplate(template) + self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") # Only time information in the log self.appendTemplate("^%H:%M:%S") # <09/16/08@05:03:30> @@ -112,25 +124,60 @@ class DateDetector: self.appendTemplate("^%b-%d-%y %H:%M:%S") finally: self.__lock.release() - - def getTemplates(self): + + @property + def templates(self): + """List of template instances managed by the detector. + """ return self.__templates - - def matchTime(self, line, incHits=True): + + def matchTime(self, line): + """Attempts to find date on a log line using templates. + + This uses the templates' `matchDate` method in an attempt to find + a date. It also increments the match hit count for the winning + template. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + re.MatchObject + The regex match returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: match = template.matchDate(line) if not match is None: - logSys.debug("Matched time template %s" % template.getName()) - if incHits: - template.incHits() + logSys.debug("Matched time template %s" % template.name) + template.hits += 1 return match return None finally: self.__lock.release() def getTime(self, line): + """Attempts to return the date on a log line using templates. + + This uses the templates' `getDate` method in an attempt to find + a date. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + float + The Unix timestamp returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: @@ -138,7 +185,8 @@ class DateDetector: date = template.getDate(line) if date is None: continue - logSys.debug("Got time %f for \"%r\" using template %s" % (date[0], date[1].group(), template.getName())) + logSys.debug("Got time %f for \"%r\" using template %s" % + (date[0], date[1].group(), template.name)) return date except ValueError: pass @@ -146,16 +194,19 @@ class DateDetector: finally: self.__lock.release() - ## - # Sort the template lists using the hits score. This method is not called - # in this object and thus should be called from time to time. - def sortTemplate(self): + """Sort the date templates by number of hits + + Sort the template lists using the hits score. This method is not + called in this object and thus should be called from time to time. + This ensures the most commonly matched templates are checked first, + improving performance of matchTime and getTime. + """ self.__lock.acquire() try: logSys.debug("Sorting the template list") - self.__templates.sort(key=lambda x: x.getHits(), reverse=True) + self.__templates.sort(key=lambda x: x.hits, reverse=True) t = self.__templates[0] - logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits())) + logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits)) finally: self.__lock.release() diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e728ef3f..f33a4a8a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -26,211 +26,132 @@ __license__ = "GPL" import re, time, calendar import logging +from abc import abstractmethod from datetime import datetime from datetime import timedelta from .mytime import MyTime -from . import iso8601 from .strptime import reGroupDictStrptime, timeRE logSys = logging.getLogger(__name__) class DateTemplate(object): - + """A template which searches for and returns a date from a log line. + + This is an not functional abstract class which other templates should + inherit from. + """ + def __init__(self): - self.__name = "" - self.__regex = "" - self.__cRegex = None - self.__hits = 0 - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - + """Initialise the date template. + """ + self._name = "" + self._regex = "" + self._cRegex = None + self.hits = 0 + + @property + def name(self): + """Name assigned to template. + """ + return self._name + + @name.setter + def name(self, name): + self._name = name + + def getRegex(self): + return self._regex + def setRegex(self, regex, wordBegin=True): - #logSys.debug(u"setRegex for %s is %r" % (self.__name, regex)) + """Sets regex to use for searching for date in log line. + + Parameters + ---------- + regex : str + The regex the template will use for searching for a date. + wordBegin : bool + Defines whether the regex should be modified to search at + begining of a word, by adding "\\b" to start of regex. + Default True. + + Raises + ------ + re.error + If regular expression fails to compile + """ regex = regex.strip() if (wordBegin and not re.search(r'^\^', regex)): regex = r'\b' + regex - self.__regex = regex - self.__cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - - def getRegex(self): - return self.__regex - - def getHits(self): - return self.__hits + self._regex = regex + self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - def incHits(self): - self.__hits += 1 + regex = property(getRegex, setRegex, doc= + """Regex used to search for date. + """) - def resetHits(self): - self.__hits = 0 - def matchDate(self, line): - dateMatch = self.__cRegex.search(line) + """Check if regex for date matches on a log line. + """ + dateMatch = self._cRegex.search(line) return dateMatch - + + @abstractmethod def getDate(self, line): - raise Exception("matchDate() is abstract") + """Abstract method, which should return the date for a log line + + This should return the date for a log line, typically taking the + date from the part of the line which matched the templates regex. + This requires abstraction, therefore just raises exception. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Raises + ------ + NotImplementedError + Abstract method, therefore always returns this. + """ + raise NotImplementedError("getDate() is abstract") class DateEpoch(DateTemplate): - + """A date template which searches for Unix timestamps. + + This includes Unix timestamps which appear at start of a line, optionally + within square braces (nsd), or on SELinux audit log lines. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) - self.setRegex("(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))") - + self.regex = "(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))" + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch return (float(dateMatch.group()), dateMatch) return None - -## -# Use strptime() to parse a date. Our current locale is the 'C' -# one because we do not set the locale explicitly. This is POSIX -# standard. - -class DateStrptime(DateTemplate): - - TABLE = dict() - TABLE["Jan"] = ["Sty"] - TABLE["Feb"] = [u"Fév", "Lut"] - TABLE["Mar"] = [u"Mär", "Mar"] - TABLE["Apr"] = ["Avr", "Kwi"] - TABLE["May"] = ["Mai", "Maj"] - TABLE["Jun"] = ["Lip"] - TABLE["Jul"] = ["Sie"] - TABLE["Aug"] = ["Aou", "Wrz"] - TABLE["Sep"] = ["Sie"] - TABLE["Oct"] = [u"Paź"] - TABLE["Nov"] = ["Lis"] - TABLE["Dec"] = [u"Déc", "Dez", "Gru"] - - def __init__(self): - DateTemplate.__init__(self) - self._pattern = "" - self._unsupportedStrptimeBits = False - - def setPattern(self, pattern): - self._unsupported_f = not DateStrptime._f and re.search('%f', pattern) - self._unsupported_z = not DateStrptime._z and re.search('%z', pattern) - self._pattern = pattern - - def getPattern(self): - return self._pattern - - #@staticmethod - def convertLocale(date): - for t in DateStrptime.TABLE: - for m in DateStrptime.TABLE[t]: - if date.find(m) >= 0: - logSys.debug(u"Replacing %r with %r in %r" % - (m, t, date)) - return date.replace(m, t) - return date - convertLocale = staticmethod(convertLocale) - - def getDate(self, line): - dateMatch = self.matchDate(line) - - if dateMatch: - datePattern = self.getPattern() - if self._unsupported_f: - if dateMatch.group('_f'): - datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern) - logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern)) - if self._unsupported_z: - if dateMatch.group('_z'): - datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern) - logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) - try: - # Try first with 'C' locale - date = datetime.strptime(dateMatch.group(), datePattern) - except ValueError: - # Try to convert date string to 'C' locale - conv = self.convertLocale(dateMatch.group()) - try: - date = datetime.strptime(conv, self.getPattern()) - except (ValueError, re.error), e: - # Try to add the current year to the pattern. Should fix - # the "Feb 29" issue. - opattern = self.getPattern() - # makes sense only if %Y is not in already: - if not '%Y' in opattern: - pattern = "%s %%Y" % opattern - conv += " %s" % MyTime.gmtime()[0] - date = datetime.strptime(conv, pattern) - else: - # we are helpless here - raise ValueError( - "Given pattern %r does not match. Original " - "exception was %r and Feb 29 workaround could not " - "be tested due to already present year mark in the " - "pattern" % (opattern, e)) - - if self._unsupported_z: - z = dateMatch.group('_z') - if z: - delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:])) - direction = z[0] - logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta)) - # here we reverse the effect of the timezone and force it to UTC - if direction == '+': - date -= delta - else: - date += delta - date = date.replace(tzinfo=iso8601.Utc()) - else: - logSys.warning("No _z group captured and %%z is not supported on current platform" - " - timezone ignored and assumed to be localtime. date: %s on line: %s" - % (date, line)) - - if date.year < 2000: - # There is probably no year field in the logs - # NOTE: Possibly makes week/year day incorrect - date = date.replace(year=MyTime.gmtime()[0]) - # Bug fix for #1241756 - # If the date is greater than the current time, we suppose - # that the log is not from this year but from the year before - if date > MyTime.now(): - logSys.debug( - u"Correcting deduced year by one since %s > now (%s)" % - (date, MyTime.time())) - date = date.replace(year=date.year-1) - elif date.month == 1 and date.day == 1: - # If it is Jan 1st, it is either really Jan 1st or there - # is neither month nor day in the log. - # NOTE: Possibly makes week/year day incorrect - date = date.replace( - month=MyTime.gmtime()[1], day=MyTime.gmtime()[2]) - - if date.tzinfo: - return ( calendar.timegm(date.utctimetuple()), dateMatch ) - else: - return ( time.mktime(date.utctimetuple()), dateMatch ) - - return None - -try: - time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") - DateStrptime._f = True -except (ValueError, KeyError): - DateTemplate._f = False - -try: - time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z") - DateStrptime._z = True -except ValueError: - DateStrptime._z = False - -class DatePatternRegex(DateStrptime): +class DatePatternRegex(DateTemplate): _patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys()) _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", @@ -241,39 +162,97 @@ class DatePatternRegex(DateStrptime): for key in set(timeRE) - set(_patternName): # may not have them all... _patternName[key] = "%%%s" % key - def __init__(self, pattern=None, **kwargs): - super(DatePatternRegex, self).__init__() - if pattern: - self.setPattern(pattern, **kwargs) + def __init__(self, pattern=None): + """Initialise date template, with optional regex/pattern - def setPattern(self, pattern): - super(DatePatternRegex, self).setPattern(pattern) - super(DatePatternRegex, self).setName( - re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName) + Parameters + ---------- + pattern : str + Sets the date templates pattern. + """ + super(DatePatternRegex, self).__init__() + self._pattern = None + if pattern is not None: + self.pattern = pattern + + @property + def pattern(self): + """The pattern used for regex with strptime "%" time fields. + + This should be a valid regular expression, of which matching string + will be extracted from the log line. strptime style "%" fields will + be replaced by appropriate regular expressions, or custom regex + groups with names as per the strptime fields can also be used + instead. + """ + return self._pattern + + @pattern.setter + def pattern(self, pattern): + self._pattern = pattern + self._name = re.sub( + self._patternRE, r'%(\1)s', pattern) % self._patternName super(DatePatternRegex, self).setRegex( re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE) - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - return reGroupDictStrptime(dateMatch.groupdict()), dateMatch - - def setRegex(self, line): + def setRegex(self, value): raise NotImplementedError("Regex derived from pattern") - def setName(self, line): + @DateTemplate.name.setter + def name(self, value): raise NotImplementedError("Name derived from pattern") + def getDate(self, line): + """Method to return the date for a log line. + + This uses a custom version of strptime, using the named groups + from the instances `pattern` property. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ + dateMatch = self.matchDate(line) + if dateMatch: + groupdict = dict( + (key, value) + for key, value in dateMatch.groupdict().iteritems() + if value is not None) + return reGroupDictStrptime(groupdict), dateMatch class DateTai64n(DateTemplate): - + """A date template which matches TAI64N formate timestamps. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) # We already know the format for TAI64N # yoh: we should not add an additional front anchor self.setRegex("@[0-9a-f]{24}", wordBegin=False) - + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch @@ -282,19 +261,3 @@ class DateTai64n(DateTemplate): # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) return None - - -class DateISO8601(DateTemplate): - - def __init__(self): - DateTemplate.__init__(self) - self.setRegex(iso8601.ISO8601_REGEX_RAW) - - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - # Parses the date. - value = dateMatch.group() - return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch) - return None - diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9d801afe..eae79bac 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -27,7 +27,7 @@ from .failmanager import FailManagerEmpty, FailManager from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DatePatternRegex, DateISO8601, DateEpoch, DateTai64n +from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -202,24 +202,20 @@ class Filter(JailThread): if pattern is None: self.dateDetector = None return - elif pattern.upper() == "ISO8601": - template = DateISO8601() - template.setName("ISO8601") elif pattern.upper() == "EPOCH": template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" elif pattern.upper() == "TAI64N": template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" else: - template = DatePatternRegex() - template.setPattern(pattern) + template = DatePatternRegex(pattern) self.dateDetector = DateDetector() self.dateDetector.appendTemplate(template) logSys.info("Date pattern set to `%r`: `%s`" % - (pattern, template.getName())) + (pattern, template.name)) logSys.debug("Date pattern regex for %r: %s" % - (pattern, template.getRegex())) + (pattern, template.regex)) ## # Get the date detector pattern, or Default Detectors if not changed @@ -228,15 +224,15 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: - templates = self.dateDetector.getTemplates() + templates = self.dateDetector.templates if len(templates) > 1: return None, "Default Detectors" elif len(templates) == 1: - if hasattr(templates[0], "getPattern"): - pattern = templates[0].getPattern() + if hasattr(templates[0], "pattern"): + pattern = templates[0].pattern else: pattern = None - return pattern, templates[0].getName() + return pattern, templates[0].name ## # Set the maximum retry value. diff --git a/fail2ban/server/iso8601.py b/fail2ban/server/iso8601.py deleted file mode 100644 index a28ce13b..00000000 --- a/fail2ban/server/iso8601.py +++ /dev/null @@ -1,136 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: - -# Copyright (c) 2007 Michael Twomey -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""ISO 8601 date time string parsing - -Basic usage: ->>> import iso8601 ->>> iso8601.parse_date("2007-01-25T12:00:00Z") -datetime.datetime(2007, 1, 25, 12, 0, tzinfo=) ->>> - -""" - -from datetime import datetime, timedelta, tzinfo -import time -import re - -__all__ = ["parse_date", "ParseError"] - -# Adapted from http://delete.me.uk/2005/03/iso8601.html -ISO8601_REGEX_RAW = "(?P[0-9]{4})-(?P[0-9]{1,2})-(?P[0-9]{1,2})" \ - "T(?P[0-9]{2}):(?P[0-9]{2})(:(?P[0-9]{2})(\.(?P[0-9]+))?)?" \ - "(?PZ|[-+][0-9]{2}(:?[0-9]{2})?)?" -ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW) -TIMEZONE_REGEX = re.compile("(?P[+-])(?P[0-9]{2}):?(?P[0-9]{2})?") - -class ParseError(Exception): - """Raised when there is a problem parsing a date string""" - -# Yoinked from python docs -ZERO = timedelta(0) -class Utc(tzinfo): - """UTC - - """ - def utcoffset(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return ZERO -UTC = Utc() - -class FixedOffset(tzinfo): - """Fixed offset in hours and minutes from UTC - - """ - def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0): - self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds) - self.__name = name - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return self.__name - - def dst(self, dt): - return ZERO - - def __repr__(self): - return "" % self.__name - -def parse_timezone(tzstring): - """Parses ISO 8601 time zone specs into tzinfo offsets - - """ - if tzstring == "Z": - return UTC - - if tzstring is None: - zone_sec = -time.timezone - return FixedOffset(name=time.tzname[0],offset_hours=(zone_sec / 3600), offset_minutes=(zone_sec % 3600)/60, offset_seconds=zone_sec % 60) - - m = TIMEZONE_REGEX.match(tzstring) - prefix, hours, minutes = m.groups() - if minutes is None: - minutes = 0 - else: - minutes = int(minutes) - hours = int(hours) - if prefix == "-": - hours = -hours - minutes = -minutes - return FixedOffset(tzstring, hours, minutes) - -def parse_date(datestring): - """Parses ISO 8601 dates into datetime objects - - The timezone is parsed from the date string. However it is quite common to - have dates without a timezone (not strictly correct). In this case the - default timezone specified in default_timezone is used. This is UTC by - default. - """ - if not isinstance(datestring, basestring): - raise ValueError("Expecting a string %r" % datestring) - m = ISO8601_REGEX.match(datestring) - if not m: - raise ParseError("Unable to parse date string %r" % datestring) - groups = m.groupdict() - tz = parse_timezone(groups["timezone"]) - if groups["fraction"] is None: - groups["fraction"] = 0 - else: - groups["fraction"] = int(float("0.%s" % groups["fraction"]) * 1e6) - - try: - return datetime(int(groups["year"]), int(groups["month"]), int(groups["day"]), - int(groups["hour"]), int(groups["minute"]), int(groups["second"]), - int(groups["fraction"]), tz) - except Exception, e: - raise ParseError("Failed to create a valid datetime record due to: %s" - % e) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index da04495f..b23a424e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -26,11 +26,24 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() -if 'z' not in timeRE: # python2.6 not present - timeRE['z'] = r"(?P[+-]\d{2}[0-5]\d)" +timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" def reGroupDictStrptime(found_dict): - """This is tweaked from python built-in _strptime""" + """Return time from dictionary of strptime fields + + This is tweaked from python built-in _strptime. + + Parameters + ---------- + found_dict : dict + Dictionary where keys represent the strptime fields, and values the + respective value. + + Returns + ------- + float + Unix time stamp. + """ now = MyTime.now() year = month = day = hour = minute = None @@ -119,9 +132,14 @@ def reGroupDictStrptime(found_dict): week_of_year_start = 0 elif group_key == 'z': z = found_dict['z'] - tzoffset = int(z[1:3]) * 60 + int(z[3:5]) - if z.startswith("-"): - tzoffset = -tzoffset + if z == "Z": + tzoffset = 0 + else: + tzoffset = int(z[1:3]) * 60 # Hours... + if len(z)>3: + tzoffset += int(z[-2:]) # ...and minutes + if z.startswith("-"): + tzoffset = -tzoffset elif group_key == 'Z': # Since -1 is default value only need to worry about setting tz if # it can be something other than -1. @@ -158,7 +176,6 @@ def reGroupDictStrptime(found_dict): month = datetime_result.month day = datetime_result.day # Add timezone info - tzname = found_dict.get("Z") if tzoffset is not None: gmtoff = tzoffset * 60 else: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 79723e31..45e1bbf5 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -27,7 +27,6 @@ __license__ = "GPL" import unittest, calendar, time, datetime, re, pprint from ..server.datedetector import DateDetector from ..server.datetemplate import DateTemplate -from ..server.iso8601 import Utc from .utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): @@ -88,8 +87,9 @@ class DateDetectorTest(unittest.TestCase): (False, "23-01-2005 21:59:59"), (False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break (False, "@4000000041f4104f00000000"), # TAI64N - (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 + (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC) (False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ + (False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local (True, "<01/23/05@21:59:59>"), (True, "050123 21:59:59"), # MySQL (True, "Jan-23-05 21:59:59"), # ASSP like @@ -116,15 +116,15 @@ class DateDetectorTest(unittest.TestCase): self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) def testStableSortTemplate(self): - old_names = [x.getName() for x in self.__datedetector.getTemplates()] + old_names = [x.name for x in self.__datedetector.templates] self.__datedetector.sortTemplate() # If there were no hits -- sorting should not change the order - for old_name, n in zip(old_names, self.__datedetector.getTemplates()): - self.assertEqual(old_name, n.getName()) # "Sort must be stable" + for old_name, n in zip(old_names, self.__datedetector.templates): + self.assertEqual(old_name, n.name) # "Sort must be stable" def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.__datedetector.appendTemplate, - self.__datedetector.getTemplates()[0]) + self.__datedetector.templates[0]) def testFullYearMatch_gh130(self): # see https://github.com/fail2ban/fail2ban/pull/130 diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 64b4355c..e9209b42 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -170,46 +170,46 @@ class TestsUtilsTest(unittest.TestCase): self.assertTrue(pindex > 10) # we should have some traceback self.assertEqual(s[:pindex], s[pindex+1:pindex*2 + 1]) -from ..server import iso8601 import datetime import time +from ..server.datetemplate import DatePatternRegex + +iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") + class CustomDateFormatsTest(unittest.TestCase): def testIso8601(self): - date = iso8601.parse_date("2007-01-25T12:00:00Z") + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00Z")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 12, 0, tzinfo=iso8601.Utc())) - self.assertRaises(ValueError, iso8601.parse_date, None) - self.assertRaises(ValueError, iso8601.parse_date, date) + datetime.datetime(2007, 1, 25, 12, 0)) + self.assertRaises(TypeError, iso8601.getDate, None) + self.assertRaises(TypeError, iso8601.getDate, date) - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "") - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "Z") + self.assertEqual(iso8601.getDate(""), None) + self.assertEqual(iso8601.getDate("Z"), None) - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-01-01T120:00:00Z") - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-13-01T12:00:00Z") - date = iso8601.parse_date("2007-01-25T12:00:00+0400") + self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) + self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00+04:00") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-0400") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-04") + datetime.datetime(2007, 1, 25, 16, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-04")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - - def testTimeZone(self): - # Just verify consistent operation and improve coverage ;) - self.assertEqual((iso8601.parse_timezone(None).tzname(False), iso8601.parse_timezone(None).tzname(True)), time.tzname) - self.assertEqual(iso8601.parse_timezone('Z').tzname(True), "UTC") - self.assertEqual(iso8601.parse_timezone('Z').dst(True), datetime.timedelta(0)) + datetime.datetime(2007, 1, 25, 16, 0)) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 648a6bf9..0d1c52d1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -278,8 +278,6 @@ class Transmitter(TransmitterBase): "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) self.setGetTest( "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) - self.setGetTest( - "datepattern", "ISO8601", (None, "ISO8601"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self): From 8b51d0c39474d9cf618d0b4be5394e3c335d89cb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 27 Jan 2014 10:10:06 +1100 Subject: [PATCH 07/41] ENH: compress DateDetector templates more --- fail2ban/server/datedetector.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index c8e631ab..35fb4d86 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -77,9 +77,9 @@ class DateDetector(object): self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") # simple date, optional subsecond (proftpd): # 2005-01-23 21:59:59 - self.appendTemplate("%Y-%m-%d %H:%M:%S(?:,%f)?") # simple date: 2005/01/23 21:59:59 - self.appendTemplate("%Y/%m/%d %H:%M:%S") + # custom for syslog-ng 2006.12.21 06:43:20 + self.appendTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?") # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) @@ -89,13 +89,10 @@ class DateDetector(object): self.appendTemplate("%d/%b/%Y:%H:%M:%S(?: %z)?") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") - # custom for syslog-ng 2006.12.21 06:43:20 - self.appendTemplate("%Y\.%m\.%d %H:%M:%S") # named 26-Jul-2007 15:20:52.252 # roundcube 26-Jul-2007 15:20:52 +0200 - self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?") # 26-Jul-2007 15:20:52 - self.appendTemplate("%d-%b-%Y %H:%M:%S") + self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?") # 17-07-2008 17:23:25 self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 From e7d4cf6296dda64627c366f4452518d426490440 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 26 Jan 2014 23:37:57 +0000 Subject: [PATCH 08/41] TST: Fix dates in ISO8601 being converted back to local time. --- fail2ban/tests/misctestcase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index e9209b42..0e1b7126 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -180,7 +180,7 @@ iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): def testIso8601(self): - date = datetime.datetime.fromtimestamp( + date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00Z")[0]) self.assertEqual( date, @@ -193,22 +193,22 @@ class CustomDateFormatsTest(unittest.TestCase): self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) - date = datetime.datetime.fromtimestamp( + date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00+0400")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 8, 0)) - date = datetime.datetime.fromtimestamp( + date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 8, 0)) - date = datetime.datetime.fromtimestamp( + date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00-0400")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) - date = datetime.datetime.fromtimestamp( + date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00-04")[0]) self.assertEqual( date, From cc1a9cc45d1b1136d214ded4cc5384e863570af4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Jan 2014 06:59:01 +1100 Subject: [PATCH 09/41] BF: match up fail2ban-regex for datedetector/datetemplate changes --- bin/fail2ban-regex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index cfaa4a89..ccdb7eac 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -438,10 +438,10 @@ class Fail2banRegex(object): if self._filter.dateDetector is not None: print "\nDate template hits:" out = [] - for template in self._filter.dateDetector.getTemplates(): - if self._verbose or template.getHits(): + for template in self._filter.dateDetector.templates: + if self._verbose or template.hits: out.append("[%d] %s" % ( - template.getHits(), template.getName())) + template.hits, template.name)) pprint_list(out, "[# of hits] date format") print "\nLines: %s" % self._line_stats From a7456377b5790bc76c7fc4715f519fcde1c953e8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Jan 2014 08:15:48 +1100 Subject: [PATCH 10/41] ENH: more datetemplate compression --- fail2ban/server/datedetector.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 35fb4d86..50f34eb2 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -83,18 +83,16 @@ class DateDetector(object): # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) - self.appendTemplate("%d/%m/(?:%Y|%y) %H:%M:%S") + # 17-07-2008 17:23:25 + self.appendTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] - self.appendTemplate("%d/%b/%Y:%H:%M:%S(?: %z)?") + # 26-Jul-2007 15:20:52 + self.appendTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") # named 26-Jul-2007 15:20:52.252 # roundcube 26-Jul-2007 15:20:52 +0200 - # 26-Jul-2007 15:20:52 - self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?") - # 17-07-2008 17:23:25 - self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion # with previous From 08171ba52f24b538045082a7d1bce90c4df12336 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 28 Jan 2014 12:44:46 +0100 Subject: [PATCH 11/41] Removed the -no auth attempts- from the triggers because of lots of FP --- config/filter.d/dovecot.conf | 4 +++- testcases/files/logs/dovecot | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index c4ce7d7c..5b6a4d8f 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ignoreregex = @@ -18,6 +18,8 @@ ignoreregex = # DEV Notes: # * the first regex is essentially a copy of pam-generic.conf # * Probably doesn't do dovecot sql/ldap backends properly +# * Removed the 'no auth attempts' log lines from the matches because produces +# lots of false positives on misconfigured MTAs making regexp unuseable # # Author: Martin Waschbuesch # Daniel Black (rewrote with begin and end anchors) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 5fe89c56..4735e4f8 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -19,19 +19,11 @@ Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disallo Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS # failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } -Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } -Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS -# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } -Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected # failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 # failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" } Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= -# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" } -Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= # failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "200.76.17.206" } Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSERVERNAME.com,200.76.17.206): pam_authenticate() failed: User not known to the underlying authentication module: 2 Time(s) From 84617fa6dab684983a71587287867a20c16defcd Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 28 Jan 2014 16:19:35 +0100 Subject: [PATCH 12/41] Fixed a failing case --- config/filter.d/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 5b6a4d8f..a444bde8 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ignoreregex = From aaa86cd10fcd43aee5fc93a4c528e7b0821663ec Mon Sep 17 00:00:00 2001 From: Joan Date: Wed, 29 Jan 2014 08:31:29 +0100 Subject: [PATCH 13/41] As suggested by @grooverdan, grouping the tests and making them false to avoid accidentally reenabling them in the future --- testcases/files/logs/dovecot | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 4735e4f8..5fe89c56 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -19,11 +19,19 @@ Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disallo Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS # failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } +Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } +Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } +Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected # failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 # failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" } Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= +# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" } +Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= # failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "200.76.17.206" } Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSERVERNAME.com,200.76.17.206): pam_authenticate() failed: User not known to the underlying authentication module: 2 Time(s) From 9c6aab37d607eea8a0a4a2e31d352df64ab25514 Mon Sep 17 00:00:00 2001 From: Joan Date: Wed, 29 Jan 2014 08:32:14 +0100 Subject: [PATCH 14/41] As suggested by @grooverdan, grouping the tests and making them false to avoid accidentally reenabling them in the future --- testcases/files/logs/dovecot | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 5fe89c56..c72a60ac 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -19,19 +19,11 @@ Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disallo Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS # failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } -Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } -Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS -# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } -Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected # failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 # failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" } Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= -# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" } -Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= # failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "200.76.17.206" } Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSERVERNAME.com,200.76.17.206): pam_authenticate() failed: User not known to the underlying authentication module: 2 Time(s) @@ -48,3 +40,11 @@ Jan 13 20:51:05 valhalla dovecot: pop3-login: Disconnected: Inactivity (auth fai # failJSON: { "time": "2005-01-14T15:54:30", "match": true , "host": "1.2.3.4" } Jan 14 15:54:30 valhalla dovecot: pop3-login: Disconnected (auth failed, 1 attempts in 2 secs): user=, method=PLAIN, rip=1.2.3.4, lip=1.1.2.2, TLS: Disconnected, session= +# failJSON: { "time": "2005-06-13T20:48:11", "match": false , "host": "121.44.24.254" } +Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T21:48:06", "match": false , "host": "180.200.180.81" } +Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-13T20:20:21", "match": false , "host": "180.189.168.166" } +Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected +# failJSON: { "time": "2005-07-02T13:49:32", "match": false , "host": "192.51.100.13" } +Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= From 9b614ce4862bc1bb4790c49bb04fe37e05f5266c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 29 Jan 2014 20:27:45 +1100 Subject: [PATCH 15/41] ENH: dovecot filter enhancements --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/dovecot.conf | 2 +- testcases/files/logs/dovecot | 13 +++++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 1a80427b..ba31b47a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ ver. 0.8.13 (2014/XX/XXX) - maintaince-only-from-now-on - Enhancements: - filter pureftpd - added all translations of "Authentication failed for user" + - filter dovecot - lip= was optional and extended TLS errors can occur. + Thanks Noel Butler. ver. 0.8.12 (2014/01/22) - things-can-only-get-better ----------- diff --git a/THANKS b/THANKS index a1e2b92d..b92d09a7 100644 --- a/THANKS +++ b/THANKS @@ -67,6 +67,7 @@ Merijn Schering Michael C. Haller Michael Hanselmann Nick Munger +Noel Butler Patrick Börjesson Raphaël Marichez RealRancor diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index c4ce7d7c..053caa45 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=(, lip=(\d{1,3}\.){3}\d{1,3})?(, TLS( handshaking(: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ignoreregex = diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 5fe89c56..add41946 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -48,3 +48,16 @@ Jan 13 20:51:05 valhalla dovecot: pop3-login: Disconnected: Inactivity (auth fai # failJSON: { "time": "2005-01-14T15:54:30", "match": true , "host": "1.2.3.4" } Jan 14 15:54:30 valhalla dovecot: pop3-login: Disconnected (auth failed, 1 attempts in 2 secs): user=, method=PLAIN, rip=1.2.3.4, lip=1.1.2.2, TLS: Disconnected, session= + +# failJSON: { "time": "2005-01-29T09:33:58", "match": true , "host": "212.9.180.3" } +Jan 29 09:33:58 pop3-login: Info: Aborted login (auth failed, 1 attempts in 2 secs): user=, method=PLAIN, rip=212.9.180.3 + +# failJSON: { "time": "2005-01-29T09:34:17", "match": true , "host": "1.2.3.4" } +Jan 29 09:34:17 pop3-login: Info: Aborted login (auth failed, 1 attempts in 62 secs): user=, method=PLAIN, rip=1.2.3.4, TLS + +# failJSON: { "time": "2005-01-29T09:38:03", "match": true , "host": "117.218.51.80" } +Jan 29 09:38:03 pop3-login: Info: Disconnected: Inactivity (auth failed, 1 attempts in 178 secs): user=, method=PLAIN, rip=117.218.51.80 + +# failJSON: { "time": "2005-01-29T09:38:46", "match": true , "host": "176.61.137.100" } +Jan 29 09:38:46 pop3-login: Info: Disconnected (no auth attempts in 10 secs): user=<>, rip=176.61.137.100, TLS handshaking: SSL_accept() failed: error:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol + From ef82eac790ec93803569f81d5e6d20c3e2de3b00 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 2 Feb 2014 15:16:40 +1100 Subject: [PATCH 16/41] DOC: openssh real protection is pubkey --- config/filter.d/sshd.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 8d39f412..92b6ed19 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -1,5 +1,8 @@ # Fail2Ban filter for openssh # +# If you want to protect OpenSSH from being bruteforced by password +# authentication then get public key authentication working before disabling +# PasswordAuthentication in sshd.conf. [INCLUDES] From 7b56daee8f609b277f42bdf4cbdf38bb9af88c4f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 2 Feb 2014 15:17:10 +1100 Subject: [PATCH 17/41] DOC: set realistic expectations of what fail2ban protects against --- README.md | 5 +++++ man/fail2ban.1 | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 2482856f..20b0b077 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,11 @@ password failures. It updates firewall rules to reject the IP address. These rules can be defined by the user. Fail2Ban can read multiple log files such as sshd or Apache web server ones. +Fail2Ban is able to reduce the rate of incorrect authentications attempts +however it cannot eliminate the risk that weak authentication presents. +Configure services to use only two factor or public/private authentication +mechanisms if you really want to protect services. + This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs are available in fail2ban(1) manpage and on the website http://www.fail2ban.org diff --git a/man/fail2ban.1 b/man/fail2ban.1 index 8f93dbc7..660168f1 100644 --- a/man/fail2ban.1 +++ b/man/fail2ban.1 @@ -25,6 +25,17 @@ For testing regular expressions specified in a filter using the fail2ban-regex program may be of use and its manual page is fail2ban-regex(1). +.SH LIMITATION + +Fail2Ban is able to reduce the rate of incorrect authentications attempts +however it cannot eliminate the risk that weak authentication presents. +Configure services to use only two factor or public/private authentication +mechanisms if you really want to protect services. + +A local user is able to inject messages into syslog and using a Fail2Ban +jail that reads from syslog, they can effectively trigger a DoS attack against +any IP. Know this risk and configure Fail2Ban/grant shell access acordingly. + .SH FILES \fI/etc/fail2ban/*\fR .SH AUTHOR From 110b8e690517001a9393cf81ee6ea2339b00e213 Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 21:39:52 +0100 Subject: [PATCH 18/41] ENH: Nagios filter Sample log entry from /var/log/messages for a denied access to the nrpe2 (Nagios Remote Plugin Executor) daemon --- testcases/files/logs/nagios | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 testcases/files/logs/nagios diff --git a/testcases/files/logs/nagios b/testcases/files/logs/nagios new file mode 100644 index 00000000..9577b327 --- /dev/null +++ b/testcases/files/logs/nagios @@ -0,0 +1,4 @@ +# Access of unauthorized host in /etc/messages +# failJSON: { "time": "2005-02-39T11:22:44", "match": true , "host": "50.97.225.132" } +Feb 3 11:22:44 valhalla nrpe[63284]: Host 50.97.225.132 is not allowed to talk to us! + From c91fda8619d8729920d695ab70e26464a0a4ae4d Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 21:46:07 +0100 Subject: [PATCH 19/41] ENH: Nagios filter Sample log for the first failregex is available in the testcases. No example available for the IPv6 denial yet. --- config/filter.d/nagios.conf | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 config/filter.d/nagios.conf diff --git a/config/filter.d/nagios.conf b/config/filter.d/nagios.conf new file mode 100644 index 00000000..d01769cc --- /dev/null +++ b/config/filter.d/nagios.conf @@ -0,0 +1,21 @@ +# Fail2Ban filter for Nagios Remote Plugin Executor (nrpe2) +# Detecting unauthorized access to the nrpe2 daemon +# typically logged in /var/log/messages syslog +# + + +[INCLUDES] +# Read syslog common prefixes +before = common.conf + + +[Definition] +_daemon = nrpe +failregex = ^%(__prefix_line)sHost is not allowed to talk to us!\s*$ + = ^%(__prefix_line)sConnection from closed. We don't support AF_INET6 addreess family in ACL\s*$ +ignoreregex =. + + +# DEV Notes: +# +# Author: Ivo Truxa - 2014/02/03 From dac4dd465e7da1f5b357613ef5695f2e91998129 Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 21:51:49 +0100 Subject: [PATCH 20/41] ENH: Nagios filter added typical configuration settings for the nagios filter --- config/jail.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index 5dcce02c..74d1e1d2 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -758,3 +758,15 @@ action = iptables[name=SSH, port=ssh, protocol=tcp] blocklist_de[email="fail2ban@example.com", apikey="xxxxxx", service=%(filter)s] logpath = /var/log/sshd.log maxretry = 20 + + +# consider low maxretry and a long bantime +# nobody except your own Nagios server should ever probe nrpe +[nagios] +enabled = false +filter = nagios +action = iptables[name=Nagios, port=5666, protocol=tcp] + sendmail-whois[name=Nagios, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] +logpath = /var/log/messages ; nrpe.cfg may define a different log_facility +ignoreip = 123.12.123.12 ; your Nagios server +maxretry = 1 From a8a43e8f3804d1f398b7232a5269869114c17ec8 Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 22:01:22 +0100 Subject: [PATCH 21/41] ENH: Nagios filter new filter Nagios added --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ba31b47a..a9126208 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,7 @@ ver. 0.8.13 (2014/XX/XXX) - maintaince-only-from-now-on - filter pureftpd - added _daemon which got removed. Added - New Features: + - filter nagios - detects unauthorized access to the nrpe daemon (Ivo Truxa) - Enhancements: - filter pureftpd - added all translations of "Authentication failed for From a71bb89ccd908bf16678e9cc436c0d317eb8f5a3 Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 23:12:56 +0100 Subject: [PATCH 22/41] removing a dot (typo) The dot at the ignoregex did not belong there. Somehow it was added during the copying and pasting. Thanks for reporting it, I did not see it. Otherwise, empty ignoregexes are in all filters, and if they are missing, fail2ban client shows warnings when starting the filter, which I prefer avoiding. --- config/filter.d/nagios.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nagios.conf b/config/filter.d/nagios.conf index d01769cc..283aa987 100644 --- a/config/filter.d/nagios.conf +++ b/config/filter.d/nagios.conf @@ -13,7 +13,7 @@ before = common.conf _daemon = nrpe failregex = ^%(__prefix_line)sHost is not allowed to talk to us!\s*$ = ^%(__prefix_line)sConnection from closed. We don't support AF_INET6 addreess family in ACL\s*$ -ignoreregex =. +ignoreregex = # DEV Notes: From f6ccd8878d80753e6b1f2f61b79ed702896e418f Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Mon, 3 Feb 2014 23:27:19 +0100 Subject: [PATCH 23/41] date fix sorry, need to get some glasses --- testcases/files/logs/nagios | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testcases/files/logs/nagios b/testcases/files/logs/nagios index 9577b327..cbeb0a87 100644 --- a/testcases/files/logs/nagios +++ b/testcases/files/logs/nagios @@ -1,4 +1,4 @@ -# Access of unauthorized host in /etc/messages -# failJSON: { "time": "2005-02-39T11:22:44", "match": true , "host": "50.97.225.132" } +# Access of unauthorized host in /var/log/messages +# failJSON: { "time": "2005-02-03T11:22:44", "match": true , "host": "50.97.225.132" } Feb 3 11:22:44 valhalla nrpe[63284]: Host 50.97.225.132 is not allowed to talk to us! From 20886288e585c07f21e013315da57899e6fca7f2 Mon Sep 17 00:00:00 2001 From: Chris Markle Date: Wed, 5 Feb 2014 10:44:46 -0800 Subject: [PATCH 24/41] Correct spelling error in changelog I know it's a nit but still... ;) --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index ba31b47a..299329b7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,7 +7,7 @@ Fail2Ban (version 0.8.12.dev) 2014/01/22 ================================================================================ -ver. 0.8.13 (2014/XX/XXX) - maintaince-only-from-now-on +ver. 0.8.13 (2014/XX/XXX) - maintenance-only-from-now-on ----------- - Fixes: From f5f434f846550e14abbb1244a71bd729e8f033b4 Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Thu, 6 Feb 2014 00:22:05 +0100 Subject: [PATCH 25/41] removing the second failregex The second failregex was supposed to catch an error concerning an ACL denial over IPv6, but this message is no more generated by the nrpe version (v2.15) that introduced the IPv6 support, so the first failregex seems to be sufficient. --- config/filter.d/nagios.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/filter.d/nagios.conf b/config/filter.d/nagios.conf index 283aa987..0429d3ff 100644 --- a/config/filter.d/nagios.conf +++ b/config/filter.d/nagios.conf @@ -3,19 +3,15 @@ # typically logged in /var/log/messages syslog # - [INCLUDES] # Read syslog common prefixes before = common.conf - [Definition] _daemon = nrpe failregex = ^%(__prefix_line)sHost is not allowed to talk to us!\s*$ - = ^%(__prefix_line)sConnection from closed. We don't support AF_INET6 addreess family in ACL\s*$ ignoreregex = - # DEV Notes: # # Author: Ivo Truxa - 2014/02/03 From c207ad6058905992aab0b9584b7335ec2bd94a4b Mon Sep 17 00:00:00 2001 From: Ivo Truxa Date: Thu, 6 Feb 2014 00:27:38 +0100 Subject: [PATCH 26/41] removing ignoreip at [nagios] I removed the ignoreip setting from the nagios section. As pointed out, it is redundant here. Nagios server, under normal circumstances should not trigger any access errors, and would be included in the global ignoreips anyway. --- config/jail.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 74d1e1d2..ffbdf33e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -768,5 +768,4 @@ filter = nagios action = iptables[name=Nagios, port=5666, protocol=tcp] sendmail-whois[name=Nagios, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"] logpath = /var/log/messages ; nrpe.cfg may define a different log_facility -ignoreip = 123.12.123.12 ; your Nagios server maxretry = 1 From c424e4032d7d3fbfc2423930273bb95bdb365b5d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 7 Feb 2014 00:41:22 -0500 Subject: [PATCH 27/41] DOC: minor - replace tabs with spaces for consistent formatting --- ChangeLog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0f1287dc..948cfcd6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,7 +31,7 @@ ver. 0.8.12 (2014/01/22) - things-can-only-get-better - Rename firewall-cmd-direct-new to firewallcmd-new to fit within jail name name length. As per gh-395 - mysqld-syslog-iptables jailname was too long. Renamed to mysqld-syslog. - Part of gh-447. + Part of gh-447. - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log @@ -48,7 +48,7 @@ ver. 0.8.12 (2014/01/22) - things-can-only-get-better - Fix apache-common for apache-2.4 log file format. Thanks Mark White. Closes gh-516 - Asynchat changed to use push method which verifys whether all data was - send. This ensures that all data is sent before closing the connection. + send. This ensures that all data is sent before closing the connection. - Removed unnecessary reference to as yet undeclared $jail_name when checking a specific jail. - Filter dovecot reordered session and TLS items in regex with wider scope @@ -890,7 +890,7 @@ ver. 0.5.4 (2005/09/13) - beta * Fixed errata in config/gentoo-confd * Introduced findtime configuration variable to control the lifetime of caught "failed" log entries - + ver. 0.5.3 (2005/09/08) - beta ---------- - Fixed a bug when overriding "maxfailures" or "bantime". Thanks to Yaroslav From 530cd53addcbff64c3d17ba86d0888b5e1d5dc5d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 8 Feb 2014 20:47:12 +0000 Subject: [PATCH 28/41] BF: Due to python3 bug, importlib.machinery may need explicit import --- fail2ban/server/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 1390cf8b..52624f29 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -28,7 +28,7 @@ import time, logging import os import sys if sys.version_info >= (3, 3): - import importlib + import importlib.machinery else: import imp from collections import Mapping From 3cbfe9b057e493cfaedee26342923889bbf789ba Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 8 Feb 2014 22:15:19 +0000 Subject: [PATCH 29/41] BF: `ret` now changed after beautifier called --- fail2ban-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-client b/fail2ban-client index 0f863ab0..af1ac856 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -156,10 +156,10 @@ class Fail2banClient: if showRet: print beautifier.beautify(ret[1]) else: - ret = False logSys.error("NOK: " + `ret[1].args`) if showRet: print beautifier.beautifyError(ret[1]) + ret = False except socket.error: if showRet: logSys.error("Unable to contact server. Is it running?") From a9f0545d8fbb88ed5be5518dca1b706009b1e794 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 9 Feb 2014 23:16:36 +0000 Subject: [PATCH 30/41] BF: Add threading lock to database --- fail2ban/server/database.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 32e4f173..82bc86ba 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -28,6 +28,7 @@ import sqlite3 import json import locale from functools import wraps +from threading import Lock from .mytime import MyTime from .ticket import FailTicket @@ -51,8 +52,9 @@ else: def commitandrollback(f): @wraps(f) def wrapper(self, *args, **kwargs): - with self._db: # Auto commit and rollback on exception - return f(self, self._db.cursor(), *args, **kwargs) + with self._lock: # Threading lock + with self._db: # Auto commit and rollback on exception + return f(self, self._db.cursor(), *args, **kwargs) return wrapper class Fail2BanDb(object): @@ -92,6 +94,7 @@ class Fail2BanDb(object): def __init__(self, filename, purgeAge=24*60*60): try: + self._lock = Lock() self._db = sqlite3.connect( filename, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES) From 993b7d3dfb644b54fd336f5f61c9a1a044ffdb45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aar=C3=B3n=20Nieves=20Fern=C3=A1ndez?= Date: Mon, 10 Feb 2014 21:41:50 +0100 Subject: [PATCH 31/41] Duplicate jail "php-url-fopen" --- config/jail.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index ffbdf33e..3d74a2f5 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -448,16 +448,6 @@ maxretry = 5 # of usage in production environments. [php-url-fopen] -enabled = false -action = iptables-multiport[name=php-url-open, port="http,https"] -filter = php-url-fopen -logpath = /var/www/*/logs/access_log -maxretry = 1 -# Ban attackers that try to use PHP's URL-fopen() functionality -# through GET/POST variables. - Experimental, with more than a year -# of usage in production environments. -[php-url-fopen] - enabled = false action = iptables-multiport[name=php-url-open, port="http,https"] filter = php-url-fopen From bda9b7d7252f3e968f741251640eaacb33d64a3c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 12 Feb 2014 18:07:31 +0000 Subject: [PATCH 32/41] BF: Add handling of exception in pyinotify callback If error isn't handled, no error messages are printed and the jail ceases to function. --- server/filterpyinotify.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/filterpyinotify.py b/server/filterpyinotify.py index 9ecb999d..530b712d 100644 --- a/server/filterpyinotify.py +++ b/server/filterpyinotify.py @@ -209,4 +209,7 @@ class ProcessPyinotify(pyinotify.ProcessEvent): # just need default, since using mask on watch to limit events def process_default(self, event): - self.__FileFilter.callback(event, origin='Default ') + try: + self.__FileFilter.callback(event, origin='Default ') + except Exception as e: + logSys.error("Error in FilterPyinotify callback: %s", e) From 5f4d0ed576be7d52be6af6ec735c6bd5686730a6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Feb 2014 09:13:46 +1100 Subject: [PATCH 33/41] ENH: ssh filter - "Disconnecting: Too many authentication failures.." matching Connection log message --- ChangeLog | 3 +++ THANKS | 1 + config/filter.d/sshd.conf | 1 + fail2ban/tests/files/logs/sshd | 4 ++++ 4 files changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index ac775a44..ed2add0f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -69,6 +69,9 @@ configuration before relying on it. * Multiline filter for sendmail-spam. Close gh-418 * Multiline regex for Disconnecting: Too many authentication failures for root [preauth]\nConnection closed by 6X.XXX.XXX.XXX [preauth] + * Multiline regex for Disconnecting: Connection from 61.XX.XX.XX port + 51353\nToo many authentication failures for root [preauth]. Thanks + Helmut Grohne. Close gh-457 * Replacing use of deprecated API (.warning, .assertEqual, etc) * [..a648cc2] Filters can have options now too which are substituted into failregex / ignoreregex diff --git a/THANKS b/THANKS index a0132f55..f252edbf 100644 --- a/THANKS +++ b/THANKS @@ -40,6 +40,7 @@ Georgiy Mernov Guilhem Lettron Guillaume Delvit Hanno 'Rince' Wagner +Helmut Grohne Iain Lea Ivo Truxa John Thoe diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 92b6ed19..93a43ea2 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -28,6 +28,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: Bye Bye \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ + ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ ignoreregex = diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 53f0cc67..b1559d47 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -132,3 +132,7 @@ Nov 23 21:50:37 sshd[7148]: Connection closed by 61.0.0.1 [preauth] # failJSON: { "time": "2005-07-13T18:44:28", "match": true , "host": "89.24.13.192", "desc": "from gh-289" } Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail +# failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } +Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 +# failJSON: { "match": false } +Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] From 1e9910fcb0e2eae1004b149d10fdc5876c25c48f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 12 Feb 2014 22:18:09 +0000 Subject: [PATCH 34/41] ENH: Added traceback for error log on pyinotify callback when in debug Thanks to Helmut Grohne for idea on #fail2ban IRC --- server/filterpyinotify.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/filterpyinotify.py b/server/filterpyinotify.py index 530b712d..42243552 100644 --- a/server/filterpyinotify.py +++ b/server/filterpyinotify.py @@ -212,4 +212,5 @@ class ProcessPyinotify(pyinotify.ProcessEvent): try: self.__FileFilter.callback(event, origin='Default ') except Exception as e: - logSys.error("Error in FilterPyinotify callback: %s", e) + logSys.error("Error in FilterPyinotify callback: %s", + e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) From 45157ddc86db00122355d3b4aefce59410ae50a0 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Feb 2014 09:26:59 +1100 Subject: [PATCH 35/41] TST: fix failJSON for ssh filter change --- fail2ban/tests/files/logs/sshd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index b1559d47..e2246cf8 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -132,7 +132,7 @@ Nov 23 21:50:37 sshd[7148]: Connection closed by 61.0.0.1 [preauth] # failJSON: { "time": "2005-07-13T18:44:28", "match": true , "host": "89.24.13.192", "desc": "from gh-289" } Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail -# failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } -Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "match": false } +Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 +# failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] From c701ac9276a5e2719103b119e66ae41477d16854 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 13 Feb 2014 16:20:36 +1100 Subject: [PATCH 36/41] DOC: document LogLevel requirement for "Connection from" regex" --- config/filter.d/sshd.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 93a43ea2..059052fc 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -2,7 +2,11 @@ # # If you want to protect OpenSSH from being bruteforced by password # authentication then get public key authentication working before disabling -# PasswordAuthentication in sshd.conf. +# PasswordAuthentication in sshd_config. +# +# +# "Connection from port \d+" requires LogLevel VERBOSE in sshd_config +# [INCLUDES] From 9bfc77c32003f0156b64a3ab04766b07d55a8e79 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 13 Feb 2014 20:03:45 +0000 Subject: [PATCH 37/41] BF: fail2ban-client processCmd ret was being overwritten each loop --- fail2ban-client | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban-client b/fail2ban-client index af1ac856..b33a27ae 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -145,7 +145,7 @@ class Fail2banClient: def __processCmd(self, cmd, showRet = True): beautifier = Beautifier() - ret = True + streamRet = True for c in cmd: beautifier.setInputCmd(c) try: @@ -159,7 +159,7 @@ class Fail2banClient: logSys.error("NOK: " + `ret[1].args`) if showRet: print beautifier.beautifyError(ret[1]) - ret = False + streamRet = False except socket.error: if showRet: logSys.error("Unable to contact server. Is it running?") @@ -168,7 +168,7 @@ class Fail2banClient: if showRet: logSys.error(e) return False - return ret + return streamRet ## # Process a command line. From 9bbf4ea258908d87de898262b1c900e097b6efbe Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 13 Feb 2014 20:07:12 +0000 Subject: [PATCH 38/41] BF: Keep sure database errors are captured during Fail2Ban startup --- fail2ban/client/fail2banreader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 5abdc141..f432a47f 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -62,5 +62,6 @@ class Fail2banReader(ConfigReader): stream.append(["set", "dbfile", self.__opts[opt]]) elif opt == "dbpurgeage": stream.append(["set", "dbpurgeage", self.__opts[opt]]) - return stream + # Ensure logtarget/level set first so any db errors are captured + return sorted(stream, reverse=True) From fceac53776027ee4d731e5357729b60f6fad0bc8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 13 Feb 2014 21:02:26 +0000 Subject: [PATCH 39/41] TST: Move nagios log sample to correct folder --- {testcases => fail2ban/tests}/files/logs/nagios | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {testcases => fail2ban/tests}/files/logs/nagios (100%) diff --git a/testcases/files/logs/nagios b/fail2ban/tests/files/logs/nagios similarity index 100% rename from testcases/files/logs/nagios rename to fail2ban/tests/files/logs/nagios From df3e4a2742221bbd0fb594aa1a3e28f623aeb40c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 15 Feb 2014 14:42:44 +0000 Subject: [PATCH 40/41] ENH: Warn when multiline regex used when maxlines not greater than 1 --- fail2ban/client/filterreader.py | 4 +++- fail2ban/server/filter.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 6728a546..612fd350 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -61,7 +61,9 @@ class FilterReader(DefinitionInitConfigReader): stream.append(["set", self._jailName, "addignoreregex", regex]) if self._initOpts: if 'maxlines' in self._initOpts: - stream.append(["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) + # We warn when multiline regex is used without maxlines > 1 + # therefore keep sure we set this option first. + stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) if 'datepattern' in self._initOpts: stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]]) # Do not send a command if the match is empty. diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index eae79bac..e777d973 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -95,6 +95,10 @@ class Filter(JailThread): try: regex = FailRegex(value) self.__failRegex.append(regex) + if "\n" in regex.getRegex() and not self.getMaxLines() > 1: + logSys.warning( + "Mutliline regex set for jail '%s' " + "but maxlines not greater than 1") except RegexException, e: logSys.error(e) raise e From c6fc57d59474a0a7a92c714e0423f5e93c2d39af Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 15 Feb 2014 15:20:59 +0000 Subject: [PATCH 41/41] TST: Fix TravisCI build for python2.7 --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea84432e..4dafda11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,12 @@ python: - "3.3" - "pypy" before_install: - - sudo apt-get update -qq + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: - pip install pyinotify - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi script: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: # Coverage config file must be .coveragerc for coveralls