diff --git a/.travis.yml b/.travis.yml index ea84432e..4dafda11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,12 @@ python: - "3.3" - "pypy" before_install: - - sudo apt-get update -qq + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: - pip install pyinotify - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi script: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: # Coverage config file must be .coveragerc for coveralls diff --git a/ChangeLog b/ChangeLog index cceb88f6..5e625c1c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -61,6 +61,7 @@ configuration before relying on it. * Filter for Counter Strike 1.6. Thanks to onorua for logs. Close gh-347 * Filter for squirrelmail. Close gh-261 + * Filter for tine20. Close gh-583 - Enhancements * Jail names increased to 26 characters and iptables prefix reduced @@ -68,6 +69,9 @@ configuration before relying on it. * Multiline filter for sendmail-spam. Close gh-418 * Multiline regex for Disconnecting: Too many authentication failures for root [preauth]\nConnection closed by 6X.XXX.XXX.XXX [preauth] + * Multiline regex for Disconnecting: Connection from 61.XX.XX.XX port + 51353\nToo many authentication failures for root [preauth]. Thanks + Helmut Grohne. Close gh-457 * Replacing use of deprecated API (.warning, .assertEqual, etc) * [..a648cc2] Filters can have options now too which are substituted into failregex / ignoreregex @@ -76,16 +80,22 @@ configuration before relying on it. * Add honeypot email address to exim-spam filter as argument -ver. 0.8.13 (2014/XX/XXX) - maintaince-only-from-now-on ----------- +ver. 0.8.13 (2014/XX/XXX) - maintenance-only-from-now-on +----------- - Fixes: - action firewallcmd-ipset had non-working actioncheck. Removed. redhat bug #1046816. + - filter pureftpd - added _daemon which got removed. Added - New Features: + - filter nagios - detects unauthorized access to the nrpe daemon (Ivo Truxa) - Enhancements: + - filter pureftpd - added all translations of "Authentication failed for + user" + - filter dovecot - lip= was optional and extended TLS errors can occur. + Thanks Noel Butler. ver. 0.8.12 (2014/01/22) - things-can-only-get-better ---------- @@ -94,7 +104,7 @@ ver. 0.8.12 (2014/01/22) - things-can-only-get-better - Rename firewall-cmd-direct-new to firewallcmd-new to fit within jail name name length. As per gh-395 - mysqld-syslog-iptables jailname was too long. Renamed to mysqld-syslog. - Part of gh-447. + Part of gh-447. - Fixes: - allow for ",milliseconds" in the custom date format of proftpd.log @@ -111,7 +121,7 @@ ver. 0.8.12 (2014/01/22) - things-can-only-get-better - Fix apache-common for apache-2.4 log file format. Thanks Mark White. Closes gh-516 - Asynchat changed to use push method which verifys whether all data was - send. This ensures that all data is sent before closing the connection. + send. This ensures that all data is sent before closing the connection. - Removed unnecessary reference to as yet undeclared $jail_name when checking a specific jail in nagios script. - Filter dovecot reordered session and TLS items in regex with wider scope @@ -958,7 +968,7 @@ ver. 0.5.4 (2005/09/13) - beta * Fixed errata in config/gentoo-confd * Introduced findtime configuration variable to control the lifetime of caught "failed" log entries - + ver. 0.5.3 (2005/09/08) - beta ---------- - Fixed a bug when overriding "maxfailures" or "bantime". Thanks to Yaroslav diff --git a/README.md b/README.md index 2482856f..20b0b077 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,11 @@ password failures. It updates firewall rules to reject the IP address. These rules can be defined by the user. Fail2Ban can read multiple log files such as sshd or Apache web server ones. +Fail2Ban is able to reduce the rate of incorrect authentications attempts +however it cannot eliminate the risk that weak authentication presents. +Configure services to use only two factor or public/private authentication +mechanisms if you really want to protect services. + This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs are available in fail2ban(1) manpage and on the website http://www.fail2ban.org diff --git a/THANKS b/THANKS index 32b1958f..f252edbf 100644 --- a/THANKS +++ b/THANKS @@ -40,6 +40,7 @@ Georgiy Mernov Guilhem Lettron Guillaume Delvit Hanno 'Rince' Wagner +Helmut Grohne Iain Lea Ivo Truxa John Thoe @@ -54,6 +55,7 @@ Justin Shore Kévin Drapel kjohnsonecl kojiro +Lars Kneschke Lee Clemens Manuel Arostegui Ramirez Marcel Dopita @@ -67,8 +69,10 @@ mEDI Merijn Schering Michael C. Haller Michael Hanselmann +Mika (mkl) Nick Munger onorua +Noel Butler Patrick Börjesson Raphaël Marichez RealRancor diff --git a/bin/fail2ban-client b/bin/fail2ban-client index c8778849..15fad1b0 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -137,7 +137,7 @@ class Fail2banClient: def __processCmd(self, cmd, showRet = True): beautifier = Beautifier() - ret = True + streamRet = True for c in cmd: beautifier.setInputCmd(c) try: @@ -148,10 +148,10 @@ class Fail2banClient: if showRet: print beautifier.beautify(ret[1]) else: - ret = False logSys.error("NOK: " + `ret[1].args`) if showRet: print beautifier.beautifyError(ret[1]) + streamRet = False except socket.error: if showRet: logSys.error("Unable to contact server. Is it running?") @@ -160,7 +160,7 @@ class Fail2banClient: if showRet: logSys.error(e) return False - return ret + return streamRet ## # Process a command line. diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index cfaa4a89..ccdb7eac 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -438,10 +438,10 @@ class Fail2banRegex(object): if self._filter.dateDetector is not None: print "\nDate template hits:" out = [] - for template in self._filter.dateDetector.getTemplates(): - if self._verbose or template.getHits(): + for template in self._filter.dateDetector.templates: + if self._verbose or template.hits: out.append("[%d] %s" % ( - template.getHits(), template.getName())) + template.hits, template.name)) pprint_list(out, "[# of hits] date format") print "\nLines: %s" % self._line_stats diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index a0c93834..732eb3a3 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, TLS( handshaking)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=(, lip=(\d{1,3}\.){3}\d{1,3})?(, TLS( handshaking(: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ignoreregex = @@ -22,6 +22,8 @@ journalmatch = _SYSTEMD_UNIT=dovecot.service # DEV Notes: # * the first regex is essentially a copy of pam-generic.conf # * Probably doesn't do dovecot sql/ldap backends properly +# * Removed the 'no auth attempts' log lines from the matches because produces +# lots of false positives on misconfigured MTAs making regexp unuseable # # Author: Martin Waschbuesch # Daniel Black (rewrote with begin and end anchors) diff --git a/config/filter.d/nagios.conf b/config/filter.d/nagios.conf new file mode 100644 index 00000000..0429d3ff --- /dev/null +++ b/config/filter.d/nagios.conf @@ -0,0 +1,17 @@ +# Fail2Ban filter for Nagios Remote Plugin Executor (nrpe2) +# Detecting unauthorized access to the nrpe2 daemon +# typically logged in /var/log/messages syslog +# + +[INCLUDES] +# Read syslog common prefixes +before = common.conf + +[Definition] +_daemon = nrpe +failregex = ^%(__prefix_line)sHost is not allowed to talk to us!\s*$ +ignoreregex = + +# DEV Notes: +# +# Author: Ivo Truxa - 2014/02/03 diff --git a/config/filter.d/pure-ftpd.conf b/config/filter.d/pure-ftpd.conf index e96009b2..b6d36603 100644 --- a/config/filter.d/pure-ftpd.conf +++ b/config/filter.d/pure-ftpd.conf @@ -12,13 +12,19 @@ before = common.conf [Definition] -# Error message specified in multiple languages -__errmsg = (?:Authentication failed for user|Erreur d'authentification pour l'utilisateur) +_daemon = pure-ftpd -failregex = ^%(__prefix_line)s\(.+?@\) \[WARNING\] %(__errmsg)s \[.+\]\s*$ +# Error message specified in multiple languages +__errmsg = (?:�ϥΪ�\[.*\]���ҥ���|ʹ����\[.*\]��֤ʧ��|\[.*\] kullan�c�s� i�in giri� hatal�|����������� �� ������� ������������ \[.*\]|Godkjennelse mislyktes for \[.*\]|Beh�righetskontroll misslyckas f�r anv�ndare \[.*\]|Autentifikacia uzivatela zlyhala \[.*\]|Autentificare esuata pentru utilizatorul \[.*\]|Autentica��o falhou para usu�rio \[.*\]|Autentyfikacja nie powiod�a si� dla u�ytkownika \[.*\]|Autorisatie faalde voor gebruiker \[.*\]|\[.*\] ��� ���� ����|Autenticazione falita per l'utente \[.*\]|Azonos�t�s sikertelen \[.*\] felhaszn�l�nak|\[.*\] c'est un batard, il connait pas son code|Erreur d'authentification pour l'utilisateur \[.*\]|Autentificaci�n fallida para el usuario \[.*\]|Authentication failed for user \[.*\]|Authentifizierung fehlgeschlagen f�r Benutzer \[.*\].|Godkendelse mislykkedes for \[.*\]|Autentifikace u�ivatele selhala \[.*\]) + +failregex = ^%(__prefix_line)s\(.+?@\) \[WARNING\] %(__errmsg)s\s*$ ignoreregex = # Author: Cyril Jaquier # Modified: Yaroslav Halchenko for pure-ftpd # Documentation thanks to Blake on http://www.fail2ban.org/wiki/index.php?title=Fail2ban:Community_Portal +# +# Only logs to syslog though facility can be changed configuration file/command line +# +# fgrep -r MSG_AUTH_FAILED_LOG pure-ftpd-1.0.36/src diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 8d39f412..059052fc 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -1,5 +1,12 @@ # Fail2Ban filter for openssh # +# If you want to protect OpenSSH from being bruteforced by password +# authentication then get public key authentication working before disabling +# PasswordAuthentication in sshd_config. +# +# +# "Connection from port \d+" requires LogLevel VERBOSE in sshd_config +# [INCLUDES] @@ -25,6 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: Bye Bye \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ + ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ ignoreregex = diff --git a/config/filter.d/tine20.conf b/config/filter.d/tine20.conf new file mode 100644 index 00000000..0fa6eccd --- /dev/null +++ b/config/filter.d/tine20.conf @@ -0,0 +1,21 @@ +# Fail2Ban filter for Tine 2.0 authentication +# +# Enable logging with: +# $config['info_log']='/var/log/tine20/tine20.log'; +# + +[Definition] + +failregex = ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0,2} - WARN \(\d+\): Tinebase_Controller::login::\d+ Login with username .*? from failed \(-[13]\)!$ + +ignoreregex = + +# Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766 +# Editor: Daniel Black +# Advisor: Lars Kneschke +# +# Usernames can contain spaces. +# +# Authentication: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Controller.php#l105 +# Logger: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Log/Formatter.php +# formatMicrotimeDiff: http://git.tine20.org/git?p=tine20;a=blob;f=tine20/Tinebase/Helper.php#l276 diff --git a/config/jail.conf b/config/jail.conf index 6a1e1c80..b1bcdbef 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -437,6 +437,12 @@ port = http,https logpath = /var/log/sogo/sogo.log +[tine20] + +logpath = /var/log/tine20/tine20.log +port = http,https +maxretry = 5 + # # Web Applications @@ -610,7 +616,6 @@ logpath = %(solidpop3d_log)s port = smtp,465,submission logpath = /var/log/exim/mainlog - [exim-spam] port = smtp,465,submission logpath = /var/log/exim/mainlog @@ -823,3 +828,11 @@ tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] + +# consider low maxretry and a long bantime +# nobody except your own Nagios server should ever probe nrpe +[nagios] + +enabled = false +logpath = /var/log/messages ; nrpe.cfg may define a different log_facility +maxretry = 1 diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 5abdc141..f432a47f 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -62,5 +62,6 @@ class Fail2banReader(ConfigReader): stream.append(["set", "dbfile", self.__opts[opt]]) elif opt == "dbpurgeage": stream.append(["set", "dbpurgeage", self.__opts[opt]]) - return stream + # Ensure logtarget/level set first so any db errors are captured + return sorted(stream, reverse=True) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 6728a546..612fd350 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -61,7 +61,9 @@ class FilterReader(DefinitionInitConfigReader): stream.append(["set", self._jailName, "addignoreregex", regex]) if self._initOpts: if 'maxlines' in self._initOpts: - stream.append(["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) + # We warn when multiline regex is used without maxlines > 1 + # therefore keep sure we set this option first. + stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) if 'datepattern' in self._initOpts: stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]]) # Do not send a command if the match is empty. diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 1390cf8b..52624f29 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -28,7 +28,7 @@ import time, logging import os import sys if sys.version_info >= (3, 3): - import importlib + import importlib.machinery else: import imp from collections import Mapping diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 32e4f173..82bc86ba 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -28,6 +28,7 @@ import sqlite3 import json import locale from functools import wraps +from threading import Lock from .mytime import MyTime from .ticket import FailTicket @@ -51,8 +52,9 @@ else: def commitandrollback(f): @wraps(f) def wrapper(self, *args, **kwargs): - with self._db: # Auto commit and rollback on exception - return f(self, self._db.cursor(), *args, **kwargs) + with self._lock: # Threading lock + with self._db: # Auto commit and rollback on exception + return f(self, self._db.cursor(), *args, **kwargs) return wrapper class Fail2BanDb(object): @@ -92,6 +94,7 @@ class Fail2BanDb(object): def __init__(self, filename, purgeAge=24*60*60): try: + self._lock = Lock() self._db = sqlite3.connect( filename, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index e5c57a31..50f34eb2 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -24,82 +24,89 @@ __license__ = "GPL" import sys, time, logging from threading import Lock -from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601 +from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class DateDetector: - +class DateDetector(object): + """Manages one or more date templates to find a date within a log line. + """ + def __init__(self): + """Initialise the date detector. + """ self.__lock = Lock() self.__templates = list() self.__known_names = set() def _appendTemplate(self, template): - name = template.getName() + name = template.name if name in self.__known_names: - raise ValueError("There is already a template with name %s" % name) + raise ValueError( + "There is already a template with name %s" % name) self.__known_names.add(name) self.__templates.append(template) - + def appendTemplate(self, template): + """Add a date template to manage and use in search of dates. + + Parameters + ---------- + template : DateTemplate or str + Can be either a `DateTemplate` instance, or a string which will + be used as the pattern for the `DatePatternRegex` template. The + template will then be added to the detector. + + Raises + ------ + ValueError + If a template already exists with the same name. + """ if isinstance(template, str): template = DatePatternRegex(template) - DateDetector._appendTemplate(self, template) + self._appendTemplate(template) def addDefaultTemplate(self): + """Add Fail2Ban's default set of date templates. + """ self.__lock.acquire() try: - # asctime with subsecond: Sun Jan 23 21:59:59.011 2005 - self.appendTemplate("%a %b %d %H:%M:%S\.%f %Y") - # asctime: Sun Jan 23 21:59:59 2005 - self.appendTemplate("%a %b %d %H:%M:%S %Y") - # asctime without year: Sun Jan 23 21:59:59 - self.appendTemplate("%a %b %d %H:%M:%S") - # standard: Jan 23 21:59:59 - self.appendTemplate("%b %d %H:%M:%S") - # proftpd date: 2005-01-23 21:59:59,333 - self.appendTemplate("%Y-%m-%d %H:%M:%S,%f") - # simple date: 2005-01-23 21:59:59 - self.appendTemplate("%Y-%m-%d %H:%M:%S") + # asctime with optional day, subsecond and/or year: + # Sun Jan 23 21:59:59.011 2005 + self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # simple date, optional subsecond (proftpd): + # 2005-01-23 21:59:59 # simple date: 2005/01/23 21:59:59 - self.appendTemplate("%Y/%m/%d %H:%M:%S") + # custom for syslog-ng 2006.12.21 06:43:20 + self.appendTemplate("%Y(?P<_sep>[-/.])%m(?P=_sep)%d %H:%M:%S(?:,%f)?") # simple date too (from x11vnc): 23/01/2005 21:59:59 - self.appendTemplate("%d/%m/%Y %H:%M:%S") - # previous one but with year given by 2 digits: 23/01/05 21:59:59 + # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) - self.appendTemplate("%d/%m/%y %H:%M:%S") - # Apache format [31/Oct/2006:09:22:55 -0000] - self.appendTemplate("%d/%b/%Y:%H:%M:%S %z") - # [31/Oct/2006:09:22:55] - self.appendTemplate("%d/%b/%Y:%H:%M:%S") + # 17-07-2008 17:23:25 + self.appendTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%Y|%y) %H:%M:%S") + # Apache format optional time zone: + # [31/Oct/2006:09:22:55 -0000] + # 26-Jul-2007 15:20:52 + self.appendTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%Y[ :]?%H:%M:%S(?:\.%f)?(?: %z)?") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") - # custom for syslog-ng 2006.12.21 06:43:20 - self.appendTemplate("%Y\.%m\.%d %H:%M:%S") # named 26-Jul-2007 15:20:52.252 - self.appendTemplate("%d-%b-%Y %H:%M:%S\.%f") # roundcube 26-Jul-2007 15:20:52 +0200 - self.appendTemplate("%d-%b-%Y %H:%M:%S %z") - # 26-Jul-2007 15:20:52 - self.appendTemplate("%d-%b-%Y %H:%M:%S") - # 17-07-2008 17:23:25 - self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 + # subseconds explicit to avoid possible %m<->%d confusion + # with previous self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f") # TAI64N template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" self.appendTemplate(template) # Epoch template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" self.appendTemplate(template) # ISO 8601 - template = DateISO8601() - template.setName("ISO 8601") - self.appendTemplate(template) + self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") # Only time information in the log self.appendTemplate("^%H:%M:%S") # <09/16/08@05:03:30> @@ -112,25 +119,60 @@ class DateDetector: self.appendTemplate("^%b-%d-%y %H:%M:%S") finally: self.__lock.release() - - def getTemplates(self): + + @property + def templates(self): + """List of template instances managed by the detector. + """ return self.__templates - - def matchTime(self, line, incHits=True): + + def matchTime(self, line): + """Attempts to find date on a log line using templates. + + This uses the templates' `matchDate` method in an attempt to find + a date. It also increments the match hit count for the winning + template. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + re.MatchObject + The regex match returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: match = template.matchDate(line) if not match is None: - logSys.debug("Matched time template %s" % template.getName()) - if incHits: - template.incHits() + logSys.debug("Matched time template %s" % template.name) + template.hits += 1 return match return None finally: self.__lock.release() def getTime(self, line): + """Attempts to return the date on a log line using templates. + + This uses the templates' `getDate` method in an attempt to find + a date. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + float + The Unix timestamp returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: @@ -138,7 +180,8 @@ class DateDetector: date = template.getDate(line) if date is None: continue - logSys.debug("Got time %f for \"%r\" using template %s" % (date[0], date[1].group(), template.getName())) + logSys.debug("Got time %f for \"%r\" using template %s" % + (date[0], date[1].group(), template.name)) return date except ValueError: pass @@ -146,16 +189,19 @@ class DateDetector: finally: self.__lock.release() - ## - # Sort the template lists using the hits score. This method is not called - # in this object and thus should be called from time to time. - def sortTemplate(self): + """Sort the date templates by number of hits + + Sort the template lists using the hits score. This method is not + called in this object and thus should be called from time to time. + This ensures the most commonly matched templates are checked first, + improving performance of matchTime and getTime. + """ self.__lock.acquire() try: logSys.debug("Sorting the template list") - self.__templates.sort(key=lambda x: x.getHits(), reverse=True) + self.__templates.sort(key=lambda x: x.hits, reverse=True) t = self.__templates[0] - logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits())) + logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits)) finally: self.__lock.release() diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e728ef3f..f33a4a8a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -26,211 +26,132 @@ __license__ = "GPL" import re, time, calendar import logging +from abc import abstractmethod from datetime import datetime from datetime import timedelta from .mytime import MyTime -from . import iso8601 from .strptime import reGroupDictStrptime, timeRE logSys = logging.getLogger(__name__) class DateTemplate(object): - + """A template which searches for and returns a date from a log line. + + This is an not functional abstract class which other templates should + inherit from. + """ + def __init__(self): - self.__name = "" - self.__regex = "" - self.__cRegex = None - self.__hits = 0 - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - + """Initialise the date template. + """ + self._name = "" + self._regex = "" + self._cRegex = None + self.hits = 0 + + @property + def name(self): + """Name assigned to template. + """ + return self._name + + @name.setter + def name(self, name): + self._name = name + + def getRegex(self): + return self._regex + def setRegex(self, regex, wordBegin=True): - #logSys.debug(u"setRegex for %s is %r" % (self.__name, regex)) + """Sets regex to use for searching for date in log line. + + Parameters + ---------- + regex : str + The regex the template will use for searching for a date. + wordBegin : bool + Defines whether the regex should be modified to search at + begining of a word, by adding "\\b" to start of regex. + Default True. + + Raises + ------ + re.error + If regular expression fails to compile + """ regex = regex.strip() if (wordBegin and not re.search(r'^\^', regex)): regex = r'\b' + regex - self.__regex = regex - self.__cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - - def getRegex(self): - return self.__regex - - def getHits(self): - return self.__hits + self._regex = regex + self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - def incHits(self): - self.__hits += 1 + regex = property(getRegex, setRegex, doc= + """Regex used to search for date. + """) - def resetHits(self): - self.__hits = 0 - def matchDate(self, line): - dateMatch = self.__cRegex.search(line) + """Check if regex for date matches on a log line. + """ + dateMatch = self._cRegex.search(line) return dateMatch - + + @abstractmethod def getDate(self, line): - raise Exception("matchDate() is abstract") + """Abstract method, which should return the date for a log line + + This should return the date for a log line, typically taking the + date from the part of the line which matched the templates regex. + This requires abstraction, therefore just raises exception. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Raises + ------ + NotImplementedError + Abstract method, therefore always returns this. + """ + raise NotImplementedError("getDate() is abstract") class DateEpoch(DateTemplate): - + """A date template which searches for Unix timestamps. + + This includes Unix timestamps which appear at start of a line, optionally + within square braces (nsd), or on SELinux audit log lines. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) - self.setRegex("(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))") - + self.regex = "(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))" + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch return (float(dateMatch.group()), dateMatch) return None - -## -# Use strptime() to parse a date. Our current locale is the 'C' -# one because we do not set the locale explicitly. This is POSIX -# standard. - -class DateStrptime(DateTemplate): - - TABLE = dict() - TABLE["Jan"] = ["Sty"] - TABLE["Feb"] = [u"Fév", "Lut"] - TABLE["Mar"] = [u"Mär", "Mar"] - TABLE["Apr"] = ["Avr", "Kwi"] - TABLE["May"] = ["Mai", "Maj"] - TABLE["Jun"] = ["Lip"] - TABLE["Jul"] = ["Sie"] - TABLE["Aug"] = ["Aou", "Wrz"] - TABLE["Sep"] = ["Sie"] - TABLE["Oct"] = [u"Paź"] - TABLE["Nov"] = ["Lis"] - TABLE["Dec"] = [u"Déc", "Dez", "Gru"] - - def __init__(self): - DateTemplate.__init__(self) - self._pattern = "" - self._unsupportedStrptimeBits = False - - def setPattern(self, pattern): - self._unsupported_f = not DateStrptime._f and re.search('%f', pattern) - self._unsupported_z = not DateStrptime._z and re.search('%z', pattern) - self._pattern = pattern - - def getPattern(self): - return self._pattern - - #@staticmethod - def convertLocale(date): - for t in DateStrptime.TABLE: - for m in DateStrptime.TABLE[t]: - if date.find(m) >= 0: - logSys.debug(u"Replacing %r with %r in %r" % - (m, t, date)) - return date.replace(m, t) - return date - convertLocale = staticmethod(convertLocale) - - def getDate(self, line): - dateMatch = self.matchDate(line) - - if dateMatch: - datePattern = self.getPattern() - if self._unsupported_f: - if dateMatch.group('_f'): - datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern) - logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern)) - if self._unsupported_z: - if dateMatch.group('_z'): - datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern) - logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) - try: - # Try first with 'C' locale - date = datetime.strptime(dateMatch.group(), datePattern) - except ValueError: - # Try to convert date string to 'C' locale - conv = self.convertLocale(dateMatch.group()) - try: - date = datetime.strptime(conv, self.getPattern()) - except (ValueError, re.error), e: - # Try to add the current year to the pattern. Should fix - # the "Feb 29" issue. - opattern = self.getPattern() - # makes sense only if %Y is not in already: - if not '%Y' in opattern: - pattern = "%s %%Y" % opattern - conv += " %s" % MyTime.gmtime()[0] - date = datetime.strptime(conv, pattern) - else: - # we are helpless here - raise ValueError( - "Given pattern %r does not match. Original " - "exception was %r and Feb 29 workaround could not " - "be tested due to already present year mark in the " - "pattern" % (opattern, e)) - - if self._unsupported_z: - z = dateMatch.group('_z') - if z: - delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:])) - direction = z[0] - logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta)) - # here we reverse the effect of the timezone and force it to UTC - if direction == '+': - date -= delta - else: - date += delta - date = date.replace(tzinfo=iso8601.Utc()) - else: - logSys.warning("No _z group captured and %%z is not supported on current platform" - " - timezone ignored and assumed to be localtime. date: %s on line: %s" - % (date, line)) - - if date.year < 2000: - # There is probably no year field in the logs - # NOTE: Possibly makes week/year day incorrect - date = date.replace(year=MyTime.gmtime()[0]) - # Bug fix for #1241756 - # If the date is greater than the current time, we suppose - # that the log is not from this year but from the year before - if date > MyTime.now(): - logSys.debug( - u"Correcting deduced year by one since %s > now (%s)" % - (date, MyTime.time())) - date = date.replace(year=date.year-1) - elif date.month == 1 and date.day == 1: - # If it is Jan 1st, it is either really Jan 1st or there - # is neither month nor day in the log. - # NOTE: Possibly makes week/year day incorrect - date = date.replace( - month=MyTime.gmtime()[1], day=MyTime.gmtime()[2]) - - if date.tzinfo: - return ( calendar.timegm(date.utctimetuple()), dateMatch ) - else: - return ( time.mktime(date.utctimetuple()), dateMatch ) - - return None - -try: - time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") - DateStrptime._f = True -except (ValueError, KeyError): - DateTemplate._f = False - -try: - time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z") - DateStrptime._z = True -except ValueError: - DateStrptime._z = False - -class DatePatternRegex(DateStrptime): +class DatePatternRegex(DateTemplate): _patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys()) _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", @@ -241,39 +162,97 @@ class DatePatternRegex(DateStrptime): for key in set(timeRE) - set(_patternName): # may not have them all... _patternName[key] = "%%%s" % key - def __init__(self, pattern=None, **kwargs): - super(DatePatternRegex, self).__init__() - if pattern: - self.setPattern(pattern, **kwargs) + def __init__(self, pattern=None): + """Initialise date template, with optional regex/pattern - def setPattern(self, pattern): - super(DatePatternRegex, self).setPattern(pattern) - super(DatePatternRegex, self).setName( - re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName) + Parameters + ---------- + pattern : str + Sets the date templates pattern. + """ + super(DatePatternRegex, self).__init__() + self._pattern = None + if pattern is not None: + self.pattern = pattern + + @property + def pattern(self): + """The pattern used for regex with strptime "%" time fields. + + This should be a valid regular expression, of which matching string + will be extracted from the log line. strptime style "%" fields will + be replaced by appropriate regular expressions, or custom regex + groups with names as per the strptime fields can also be used + instead. + """ + return self._pattern + + @pattern.setter + def pattern(self, pattern): + self._pattern = pattern + self._name = re.sub( + self._patternRE, r'%(\1)s', pattern) % self._patternName super(DatePatternRegex, self).setRegex( re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE) - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - return reGroupDictStrptime(dateMatch.groupdict()), dateMatch - - def setRegex(self, line): + def setRegex(self, value): raise NotImplementedError("Regex derived from pattern") - def setName(self, line): + @DateTemplate.name.setter + def name(self, value): raise NotImplementedError("Name derived from pattern") + def getDate(self, line): + """Method to return the date for a log line. + + This uses a custom version of strptime, using the named groups + from the instances `pattern` property. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ + dateMatch = self.matchDate(line) + if dateMatch: + groupdict = dict( + (key, value) + for key, value in dateMatch.groupdict().iteritems() + if value is not None) + return reGroupDictStrptime(groupdict), dateMatch class DateTai64n(DateTemplate): - + """A date template which matches TAI64N formate timestamps. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) # We already know the format for TAI64N # yoh: we should not add an additional front anchor self.setRegex("@[0-9a-f]{24}", wordBegin=False) - + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch @@ -282,19 +261,3 @@ class DateTai64n(DateTemplate): # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) return None - - -class DateISO8601(DateTemplate): - - def __init__(self): - DateTemplate.__init__(self) - self.setRegex(iso8601.ISO8601_REGEX_RAW) - - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - # Parses the date. - value = dateMatch.group() - return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch) - return None - diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9d801afe..e777d973 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -27,7 +27,7 @@ from .failmanager import FailManagerEmpty, FailManager from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DatePatternRegex, DateISO8601, DateEpoch, DateTai64n +from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -95,6 +95,10 @@ class Filter(JailThread): try: regex = FailRegex(value) self.__failRegex.append(regex) + if "\n" in regex.getRegex() and not self.getMaxLines() > 1: + logSys.warning( + "Mutliline regex set for jail '%s' " + "but maxlines not greater than 1") except RegexException, e: logSys.error(e) raise e @@ -202,24 +206,20 @@ class Filter(JailThread): if pattern is None: self.dateDetector = None return - elif pattern.upper() == "ISO8601": - template = DateISO8601() - template.setName("ISO8601") elif pattern.upper() == "EPOCH": template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" elif pattern.upper() == "TAI64N": template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" else: - template = DatePatternRegex() - template.setPattern(pattern) + template = DatePatternRegex(pattern) self.dateDetector = DateDetector() self.dateDetector.appendTemplate(template) logSys.info("Date pattern set to `%r`: `%s`" % - (pattern, template.getName())) + (pattern, template.name)) logSys.debug("Date pattern regex for %r: %s" % - (pattern, template.getRegex())) + (pattern, template.regex)) ## # Get the date detector pattern, or Default Detectors if not changed @@ -228,15 +228,15 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: - templates = self.dateDetector.getTemplates() + templates = self.dateDetector.templates if len(templates) > 1: return None, "Default Detectors" elif len(templates) == 1: - if hasattr(templates[0], "getPattern"): - pattern = templates[0].getPattern() + if hasattr(templates[0], "pattern"): + pattern = templates[0].pattern else: pattern = None - return pattern, templates[0].getName() + return pattern, templates[0].name ## # Set the maximum retry value. diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 9ab8a83a..ab57290f 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -208,4 +208,8 @@ class ProcessPyinotify(pyinotify.ProcessEvent): # just need default, since using mask on watch to limit events def process_default(self, event): - self.__FileFilter.callback(event, origin='Default ') + try: + self.__FileFilter.callback(event, origin='Default ') + except Exception as e: + logSys.error("Error in FilterPyinotify callback: %s", + e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) diff --git a/fail2ban/server/iso8601.py b/fail2ban/server/iso8601.py deleted file mode 100644 index a28ce13b..00000000 --- a/fail2ban/server/iso8601.py +++ /dev/null @@ -1,136 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: - -# Copyright (c) 2007 Michael Twomey -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""ISO 8601 date time string parsing - -Basic usage: ->>> import iso8601 ->>> iso8601.parse_date("2007-01-25T12:00:00Z") -datetime.datetime(2007, 1, 25, 12, 0, tzinfo=) ->>> - -""" - -from datetime import datetime, timedelta, tzinfo -import time -import re - -__all__ = ["parse_date", "ParseError"] - -# Adapted from http://delete.me.uk/2005/03/iso8601.html -ISO8601_REGEX_RAW = "(?P[0-9]{4})-(?P[0-9]{1,2})-(?P[0-9]{1,2})" \ - "T(?P[0-9]{2}):(?P[0-9]{2})(:(?P[0-9]{2})(\.(?P[0-9]+))?)?" \ - "(?PZ|[-+][0-9]{2}(:?[0-9]{2})?)?" -ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW) -TIMEZONE_REGEX = re.compile("(?P[+-])(?P[0-9]{2}):?(?P[0-9]{2})?") - -class ParseError(Exception): - """Raised when there is a problem parsing a date string""" - -# Yoinked from python docs -ZERO = timedelta(0) -class Utc(tzinfo): - """UTC - - """ - def utcoffset(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return ZERO -UTC = Utc() - -class FixedOffset(tzinfo): - """Fixed offset in hours and minutes from UTC - - """ - def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0): - self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds) - self.__name = name - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return self.__name - - def dst(self, dt): - return ZERO - - def __repr__(self): - return "" % self.__name - -def parse_timezone(tzstring): - """Parses ISO 8601 time zone specs into tzinfo offsets - - """ - if tzstring == "Z": - return UTC - - if tzstring is None: - zone_sec = -time.timezone - return FixedOffset(name=time.tzname[0],offset_hours=(zone_sec / 3600), offset_minutes=(zone_sec % 3600)/60, offset_seconds=zone_sec % 60) - - m = TIMEZONE_REGEX.match(tzstring) - prefix, hours, minutes = m.groups() - if minutes is None: - minutes = 0 - else: - minutes = int(minutes) - hours = int(hours) - if prefix == "-": - hours = -hours - minutes = -minutes - return FixedOffset(tzstring, hours, minutes) - -def parse_date(datestring): - """Parses ISO 8601 dates into datetime objects - - The timezone is parsed from the date string. However it is quite common to - have dates without a timezone (not strictly correct). In this case the - default timezone specified in default_timezone is used. This is UTC by - default. - """ - if not isinstance(datestring, basestring): - raise ValueError("Expecting a string %r" % datestring) - m = ISO8601_REGEX.match(datestring) - if not m: - raise ParseError("Unable to parse date string %r" % datestring) - groups = m.groupdict() - tz = parse_timezone(groups["timezone"]) - if groups["fraction"] is None: - groups["fraction"] = 0 - else: - groups["fraction"] = int(float("0.%s" % groups["fraction"]) * 1e6) - - try: - return datetime(int(groups["year"]), int(groups["month"]), int(groups["day"]), - int(groups["hour"]), int(groups["minute"]), int(groups["second"]), - int(groups["fraction"]), tz) - except Exception, e: - raise ParseError("Failed to create a valid datetime record due to: %s" - % e) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index da04495f..b23a424e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -26,11 +26,24 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() -if 'z' not in timeRE: # python2.6 not present - timeRE['z'] = r"(?P[+-]\d{2}[0-5]\d)" +timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" def reGroupDictStrptime(found_dict): - """This is tweaked from python built-in _strptime""" + """Return time from dictionary of strptime fields + + This is tweaked from python built-in _strptime. + + Parameters + ---------- + found_dict : dict + Dictionary where keys represent the strptime fields, and values the + respective value. + + Returns + ------- + float + Unix time stamp. + """ now = MyTime.now() year = month = day = hour = minute = None @@ -119,9 +132,14 @@ def reGroupDictStrptime(found_dict): week_of_year_start = 0 elif group_key == 'z': z = found_dict['z'] - tzoffset = int(z[1:3]) * 60 + int(z[3:5]) - if z.startswith("-"): - tzoffset = -tzoffset + if z == "Z": + tzoffset = 0 + else: + tzoffset = int(z[1:3]) * 60 # Hours... + if len(z)>3: + tzoffset += int(z[-2:]) # ...and minutes + if z.startswith("-"): + tzoffset = -tzoffset elif group_key == 'Z': # Since -1 is default value only need to worry about setting tz if # it can be something other than -1. @@ -158,7 +176,6 @@ def reGroupDictStrptime(found_dict): month = datetime_result.month day = datetime_result.day # Add timezone info - tzname = found_dict.get("Z") if tzoffset is not None: gmtoff = tzoffset * 60 else: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 79723e31..45e1bbf5 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -27,7 +27,6 @@ __license__ = "GPL" import unittest, calendar, time, datetime, re, pprint from ..server.datedetector import DateDetector from ..server.datetemplate import DateTemplate -from ..server.iso8601 import Utc from .utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): @@ -88,8 +87,9 @@ class DateDetectorTest(unittest.TestCase): (False, "23-01-2005 21:59:59"), (False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break (False, "@4000000041f4104f00000000"), # TAI64N - (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 + (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC) (False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ + (False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local (True, "<01/23/05@21:59:59>"), (True, "050123 21:59:59"), # MySQL (True, "Jan-23-05 21:59:59"), # ASSP like @@ -116,15 +116,15 @@ class DateDetectorTest(unittest.TestCase): self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) def testStableSortTemplate(self): - old_names = [x.getName() for x in self.__datedetector.getTemplates()] + old_names = [x.name for x in self.__datedetector.templates] self.__datedetector.sortTemplate() # If there were no hits -- sorting should not change the order - for old_name, n in zip(old_names, self.__datedetector.getTemplates()): - self.assertEqual(old_name, n.getName()) # "Sort must be stable" + for old_name, n in zip(old_names, self.__datedetector.templates): + self.assertEqual(old_name, n.name) # "Sort must be stable" def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.__datedetector.appendTemplate, - self.__datedetector.getTemplates()[0]) + self.__datedetector.templates[0]) def testFullYearMatch_gh130(self): # see https://github.com/fail2ban/fail2ban/pull/130 diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 164a24cc..5c3acb93 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -19,19 +19,11 @@ Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disallo Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS # failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" } Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" } -Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected -# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" } -Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS -# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" } -Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected # failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" } Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210 # failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" } Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session= -# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" } -Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= # failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "200.76.17.206" } Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSERVERNAME.com,200.76.17.206): pam_authenticate() failed: User not known to the underlying authentication module: 2 Time(s) @@ -48,3 +40,24 @@ Jan 13 20:51:05 valhalla dovecot: pop3-login: Disconnected: Inactivity (auth fai # failJSON: { "time": "2005-01-14T15:54:30", "match": true , "host": "1.2.3.4" } Jan 14 15:54:30 valhalla dovecot: pop3-login: Disconnected (auth failed, 1 attempts in 2 secs): user=, method=PLAIN, rip=1.2.3.4, lip=1.1.2.2, TLS: Disconnected, session= + +# failJSON: { "time": "2005-01-29T09:33:58", "match": true , "host": "212.9.180.3" } +Jan 29 09:33:58 pop3-login: Info: Aborted login (auth failed, 1 attempts in 2 secs): user=, method=PLAIN, rip=212.9.180.3 + +# failJSON: { "time": "2005-01-29T09:34:17", "match": true , "host": "1.2.3.4" } +Jan 29 09:34:17 pop3-login: Info: Aborted login (auth failed, 1 attempts in 62 secs): user=, method=PLAIN, rip=1.2.3.4, TLS + +# failJSON: { "time": "2005-01-29T09:38:03", "match": true , "host": "117.218.51.80" } +Jan 29 09:38:03 pop3-login: Info: Disconnected: Inactivity (auth failed, 1 attempts in 178 secs): user=, method=PLAIN, rip=117.218.51.80 + +# failJSON: { "time": "2005-01-29T09:38:46", "match": false , "host": "176.61.137.100" } +Jan 29 09:38:46 pop3-login: Info: Disconnected (no auth attempts in 10 secs): user=<>, rip=176.61.137.100, TLS handshaking: SSL_accept() failed: error:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol + +# failJSON: { "time": "2005-06-13T20:48:11", "match": false , "host": "121.44.24.254" } +Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +# failJSON: { "time": "2005-06-13T21:48:06", "match": false , "host": "180.200.180.81" } +Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +# failJSON: { "time": "2005-06-13T20:20:21", "match": false , "host": "180.189.168.166" } +Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected +# failJSON: { "time": "2005-07-02T13:49:32", "match": false , "host": "192.51.100.13" } +Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session= diff --git a/fail2ban/tests/files/logs/nagios b/fail2ban/tests/files/logs/nagios new file mode 100644 index 00000000..cbeb0a87 --- /dev/null +++ b/fail2ban/tests/files/logs/nagios @@ -0,0 +1,4 @@ +# Access of unauthorized host in /var/log/messages +# failJSON: { "time": "2005-02-03T11:22:44", "match": true , "host": "50.97.225.132" } +Feb 3 11:22:44 valhalla nrpe[63284]: Host 50.97.225.132 is not allowed to talk to us! + diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 53f0cc67..e2246cf8 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -132,3 +132,7 @@ Nov 23 21:50:37 sshd[7148]: Connection closed by 61.0.0.1 [preauth] # failJSON: { "time": "2005-07-13T18:44:28", "match": true , "host": "89.24.13.192", "desc": "from gh-289" } Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail +# failJSON: { "match": false } +Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 +# failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } +Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] diff --git a/fail2ban/tests/files/logs/tine20 b/fail2ban/tests/files/logs/tine20 new file mode 100644 index 00000000..dbb9f424 --- /dev/null +++ b/fail2ban/tests/files/logs/tine20 @@ -0,0 +1,7 @@ +# Wrong username (-1) error +# failJSON: { "time": "2014-01-13T06:02:22", "match": true, "host": "127.0.0.1" } +78017 00cff -- none -- - 2014-01-13T05:02:22+00:00 WARN (4): Tinebase_Controller::login::106 Login with username sdfsadf from 127.0.0.1 failed (-1)! + +# Wrong password (-3) error +# failJSON: { "time": "2014-01-21T05:38:14", "match": true, "host": "127.0.0.1" } +8e035 ffff3 -- none -- - 2014-01-21T04:38:14+00:00 WARN (4): Tinebase_Controller::login::106 Login with username testuser from 127.0.0.1 failed (-3)! diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 64b4355c..0e1b7126 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -170,46 +170,46 @@ class TestsUtilsTest(unittest.TestCase): self.assertTrue(pindex > 10) # we should have some traceback self.assertEqual(s[:pindex], s[pindex+1:pindex*2 + 1]) -from ..server import iso8601 import datetime import time +from ..server.datetemplate import DatePatternRegex + +iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") + class CustomDateFormatsTest(unittest.TestCase): def testIso8601(self): - date = iso8601.parse_date("2007-01-25T12:00:00Z") + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00Z")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 12, 0, tzinfo=iso8601.Utc())) - self.assertRaises(ValueError, iso8601.parse_date, None) - self.assertRaises(ValueError, iso8601.parse_date, date) + datetime.datetime(2007, 1, 25, 12, 0)) + self.assertRaises(TypeError, iso8601.getDate, None) + self.assertRaises(TypeError, iso8601.getDate, date) - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "") - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "Z") + self.assertEqual(iso8601.getDate(""), None) + self.assertEqual(iso8601.getDate("Z"), None) - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-01-01T120:00:00Z") - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-13-01T12:00:00Z") - date = iso8601.parse_date("2007-01-25T12:00:00+0400") + self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) + self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00+04:00") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-0400") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-04") + datetime.datetime(2007, 1, 25, 16, 0)) + date = datetime.datetime.utcfromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-04")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - - def testTimeZone(self): - # Just verify consistent operation and improve coverage ;) - self.assertEqual((iso8601.parse_timezone(None).tzname(False), iso8601.parse_timezone(None).tzname(True)), time.tzname) - self.assertEqual(iso8601.parse_timezone('Z').tzname(True), "UTC") - self.assertEqual(iso8601.parse_timezone('Z').dst(True), datetime.timedelta(0)) + datetime.datetime(2007, 1, 25, 16, 0)) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 648a6bf9..0d1c52d1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -278,8 +278,6 @@ class Transmitter(TransmitterBase): "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) self.setGetTest( "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) - self.setGetTest( - "datepattern", "ISO8601", (None, "ISO8601"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self): diff --git a/man/fail2ban.1 b/man/fail2ban.1 index 8f93dbc7..660168f1 100644 --- a/man/fail2ban.1 +++ b/man/fail2ban.1 @@ -25,6 +25,17 @@ For testing regular expressions specified in a filter using the fail2ban-regex program may be of use and its manual page is fail2ban-regex(1). +.SH LIMITATION + +Fail2Ban is able to reduce the rate of incorrect authentications attempts +however it cannot eliminate the risk that weak authentication presents. +Configure services to use only two factor or public/private authentication +mechanisms if you really want to protect services. + +A local user is able to inject messages into syslog and using a Fail2Ban +jail that reads from syslog, they can effectively trigger a DoS attack against +any IP. Know this risk and configure Fail2Ban/grant shell access acordingly. + .SH FILES \fI/etc/fail2ban/*\fR .SH AUTHOR