From ab10664b57630fcf012ee5cf8cc1f00c2eadf7b4 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 14 Jul 2013 16:20:21 +1000 Subject: [PATCH 01/18] ENH: action.d/hostsdeny to take daemon_list arguement as suggested in README.Solaris --- ChangeLog | 2 +- README.Solaris | 6 ++---- config/action.d/hostsdeny.conf | 11 +++++++++-- config/jail.conf | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5b9ec79e..2fba9401 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,7 +29,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Daniel Black * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across all platforms to ensure permissions are the same before and after a ban - - closes gh-266 + closes gh-266. hostsdeny supports daemon_list now too. - New Features: Daniel Black & ykimon * filter.d/3proxy.conf -- filter added diff --git a/README.Solaris b/README.Solaris index 10a5f88c..6a3335d7 100644 --- a/README.Solaris +++ b/README.Solaris @@ -99,7 +99,7 @@ REQ: Create /etc/fail2ban/jail.local containing: enabled = true filter = sshd -action = hostsdeny +action = hostsdeny[sshd] sendmail-whois[name=SSH, dest=you@example.com] ignoreregex = for myuser from logpath = /var/adm/auth.log @@ -136,6 +136,4 @@ GOTCHAS AND FIXMES * Fail2ban adds lines like these to /etc/hosts.deny: - ALL: 1.2.3.4 - - wouldn't it be better to just block sshd? + sshd: 1.2.3.4 diff --git a/config/action.d/hostsdeny.conf b/config/action.d/hostsdeny.conf index 36e34948..d74f498d 100644 --- a/config/action.d/hostsdeny.conf +++ b/config/action.d/hostsdeny.conf @@ -1,6 +1,7 @@ # Fail2Ban configuration file # # Author: Cyril Jaquier +# Edited for cross platform by: James Stout, Yaroslav Halchenko and Daniel Black # # @@ -31,7 +32,7 @@ actioncheck = # Values: CMD # actionban = IP= && - printf %%b "ALL: $IP\n" >> + printf %%b ": $IP\n" >> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -39,7 +40,7 @@ actionban = IP= && # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = echo "/ALL: $/
d
w
q" | ed +actionunban = echo "/^: $/
d
w
q" | ed [Init] @@ -48,3 +49,9 @@ actionunban = echo "/ALL: $/
d
w
q" | ed # Values: STR Default: /etc/hosts.deny # file = /etc/hosts.deny + +# Option: daemon_list +# Notes: The list of services that this action will deny. See the man page +# for hosts.deny/hosts_access. Default is all services. +# Values: STR Default: ALL +daemon_list = ALL diff --git a/config/jail.conf b/config/jail.conf index e3b92038..ccc1ae9a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -103,7 +103,7 @@ logpath = /root/path/to/assp/logs/maillog.txt enabled = false filter = sshd -action = hostsdeny +action = hostsdeny[daemon_list=sshd] sendmail-whois[name=SSH, dest=you@example.com] ignoreregex = for myuser from logpath = /var/log/sshd.log From 6e4b1441644c2399a30c6cc311a52d79a5f22d83 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 14 Jul 2013 21:43:14 +1000 Subject: [PATCH 02/18] DOC: fix for README.Solaris. Thanks Steven. --- README.Solaris | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.Solaris b/README.Solaris index 6a3335d7..5157c099 100644 --- a/README.Solaris +++ b/README.Solaris @@ -99,7 +99,7 @@ REQ: Create /etc/fail2ban/jail.local containing: enabled = true filter = sshd -action = hostsdeny[sshd] +action = hostsdeny[daemon_list=sshd] sendmail-whois[name=SSH, dest=you@example.com] ignoreregex = for myuser from logpath = /var/adm/auth.log From 60aa44f42d34c050a20228462bf6bf380752b057 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 18 Jul 2013 14:43:41 -0400 Subject: [PATCH 03/18] DOC: added a note on commit comments for merges with Conflicts --- DEVELOP | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DEVELOP b/DEVELOP index b0f7b47e..61248393 100644 --- a/DEVELOP +++ b/DEVELOP @@ -253,6 +253,10 @@ Use the text "closes #333"/"resolves #333 "/"fixes #333" where 333 represents an issue that is closed. Other text and details in link below. See: https://help.github.com/articles/closing-issues-via-commit-messages +If merge resulted in conflicts, clarify what changes were done to +corresponding files in the 'Conflicts:' section of the merge commit +message. See e.g. https://github.com/fail2ban/fail2ban/commit/f5a8a8ac + Adding Actions -------------- From 2a3a6273223aa98c12d2071adb385ff073252da9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Jul 2013 22:17:14 +0100 Subject: [PATCH 04/18] TST: Add sample for sieve regex --- testcases/files/logs/sieve | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 testcases/files/logs/sieve diff --git a/testcases/files/logs/sieve b/testcases/files/logs/sieve new file mode 100644 index 00000000..5cc19673 --- /dev/null +++ b/testcases/files/logs/sieve @@ -0,0 +1,2 @@ +# failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" } +Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure From a012b5411715f3a478cd7b19acd70e652044d625 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Jul 2013 22:17:31 +0100 Subject: [PATCH 05/18] TST: Add additional postfix filter sample --- testcases/files/logs/postfix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testcases/files/logs/postfix b/testcases/files/logs/postfix index 50e3a360..e4d07b58 100644 --- a/testcases/files/logs/postfix +++ b/testcases/files/logs/postfix @@ -2,3 +2,5 @@ # and https://github.com/fail2ban/fail2ban/issues/126 # failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" } Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= +# failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" } +Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 : Relay access denied; from= to= proto=SMTP helo=<198.51.100.17> From fcf79b475f26884a584617f3fd5ff66afa6df371 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 19 Jul 2013 20:14:53 +1000 Subject: [PATCH 06/18] ENH: new filter perdition.conf --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/perdition.conf | 16 ++++++++++++++++ testcases/files/logs/perdition | 4 ++++ 4 files changed, 23 insertions(+) create mode 100644 config/filter.d/perdition.conf create mode 100644 testcases/files/logs/perdition diff --git a/ChangeLog b/ChangeLog index a6a6ab23..4139dfe1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,8 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests Daniel Black * filter.d/exim-spam.conf -- a splitout of exim's spam regexes with additions for greater control over filtering spam. + Christophe Carles & Daniel Black + * filter.d/perdition.conf -- filter added - Enhancements: Daniel Black * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening diff --git a/THANKS b/THANKS index af790f67..b853c0dc 100644 --- a/THANKS +++ b/THANKS @@ -11,6 +11,7 @@ Axel Thimm Bill Heaton Carlos Alberto Lopez Perez Christian Rauch +Christophe Carles Christoph Haas Christos Psonis Daniel B. Cid diff --git a/config/filter.d/perdition.conf b/config/filter.d/perdition.conf new file mode 100644 index 00000000..e7cb0f7d --- /dev/null +++ b/config/filter.d/perdition.conf @@ -0,0 +1,16 @@ +# Fail2Ban configuration file +# +# Author: Christophe Carles and Daniel Black +# +# + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon=perdition.imaps + +failregex = ^%(__prefix_line)sAuth: :\d+->(\d{1,3}\.){3}\d{1,3}:\d+ client-secure=\S+ authorisation_id=NONE authentication_id="\S+" server="\S+" protocol=\S+ server-secure=\S+ status="failed: Re-Authentication Failure"$ + ^%(__prefix_line)sFatal Error reading authentication information from client :\d+->(\d{1,3}\.){3}\d{1,3}:\d+: Exiting child$ diff --git a/testcases/files/logs/perdition b/testcases/files/logs/perdition new file mode 100644 index 00000000..2304e372 --- /dev/null +++ b/testcases/files/logs/perdition @@ -0,0 +1,4 @@ +# failJSON: { "time": "2013-07-18T16:07:18", "match": true , "host": "192.168.8.100" } +Jul 18 16:07:18 ares perdition.imaps[3194]: Auth: 192.168.8.100:2274->193.48.191.9:993 client-secure=ssl authorisation_id=NONE authentication_id="carles" server="imap.biotoul.fr:993" protocol=IMAP4S server-secure=ssl status="failed: Re-Authentication Failure" +# failJSON: { "time": "2013-07-18T16:08:58", "match": true , "host": "192.168.8.100" } +Jul 18 16:08:58 ares perdition.imaps[3194]: Fatal Error reading authentication information from client 192.168.8.100:2274->193.48.191.9:993: Exiting child From eea5b071e61ba5cd5f4943100dc4bc0a278cf358 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 19 Jul 2013 20:27:15 +1000 Subject: [PATCH 07/18] ENH: jail for perdition --- config/jail.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index e3b92038..735f028c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -419,3 +419,10 @@ enabled = false filter = exim-spam action = iptables-multiport[name=exim-spam,port="25,465,587"] logpath = /var/log/exim/mainlog + +[perdition] +enabled = false +filter = perdition +action = iptables-multiport[name=perdition="110,143,993,995"] +logpath = /var/log/maillog + From 6fdfd8d356838512bf8582b8bc2cff0cfb5fade9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 20 Jul 2013 15:09:25 +1000 Subject: [PATCH 08/18] BF: fix port --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 735f028c..07f9cacc 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -423,6 +423,6 @@ logpath = /var/log/exim/mainlog [perdition] enabled = false filter = perdition -action = iptables-multiport[name=perdition="110,143,993,995"] +action = iptables-multiport[name=perdition,port="110,143,993,995"] logpath = /var/log/maillog From bdcde678d199a360f39e03fa27719fc0fa68fbc6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 20 Jul 2013 15:15:02 +1000 Subject: [PATCH 09/18] TST: fix year --- testcases/files/logs/perdition | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testcases/files/logs/perdition b/testcases/files/logs/perdition index 2304e372..24848e6f 100644 --- a/testcases/files/logs/perdition +++ b/testcases/files/logs/perdition @@ -1,4 +1,4 @@ -# failJSON: { "time": "2013-07-18T16:07:18", "match": true , "host": "192.168.8.100" } +# failJSON: { "time": "2005-07-18T16:07:18", "match": true , "host": "192.168.8.100" } Jul 18 16:07:18 ares perdition.imaps[3194]: Auth: 192.168.8.100:2274->193.48.191.9:993 client-secure=ssl authorisation_id=NONE authentication_id="carles" server="imap.biotoul.fr:993" protocol=IMAP4S server-secure=ssl status="failed: Re-Authentication Failure" -# failJSON: { "time": "2013-07-18T16:08:58", "match": true , "host": "192.168.8.100" } +# failJSON: { "time": "2005-07-18T16:08:58", "match": true , "host": "192.168.8.100" } Jul 18 16:08:58 ares perdition.imaps[3194]: Fatal Error reading authentication information from client 192.168.8.100:2274->193.48.191.9:993: Exiting child From b92316ff17024ef1c0f0fa48298b5b51f60026b6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Jul 2013 11:47:36 -0400 Subject: [PATCH 10/18] RF(ENH): JailsReader.getOptions -- avoid code duplication when asking for 1 jail or all --- client/jailsreader.py | 45 +++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/client/jailsreader.py b/client/jailsreader.py index 408a40bf..00c63e3c 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -18,7 +18,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Author: Cyril Jaquier -# +# __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" @@ -32,7 +32,7 @@ from jailreader import JailReader logSys = logging.getLogger("fail2ban.client.config") class JailsReader(ConfigReader): - + def __init__(self, force_enable=False, **kwargs): """ Parameters @@ -44,17 +44,25 @@ class JailsReader(ConfigReader): ConfigReader.__init__(self, **kwargs) self.__jails = list() self.__force_enable = force_enable - + def read(self): return ConfigReader.read(self, "jail") - - def getOptions(self, section = None): + + def getOptions(self, section=None): + """Reads configuration for jail(s) and adds enabled jails to __jails + """ opts = [] self.__opts = ConfigReader.getOptions(self, "Definition", opts) - if section: - # Get the options of a specific jail. - jail = JailReader(section, basedir=self.getBaseDir(), force_enable=self.__force_enable) + if section is None: + sections = self.sections() + else: + sections = [ section ] + + # Get the options of all jails. + for sec in sections: + jail = JailReader(sec, basedir=self.getBaseDir(), + force_enable=self.__force_enable) jail.read() ret = jail.getOptions() if ret: @@ -62,23 +70,10 @@ class JailsReader(ConfigReader): # We only add enabled jails self.__jails.append(jail) else: - logSys.error("Errors in jail '%s'. Skipping..." % section) + logSys.error("Errors in jail %r. Skipping..." % sec) return False - else: - # Get the options of all jails. - for sec in self.sections(): - jail = JailReader(sec, basedir=self.getBaseDir(), force_enable=self.__force_enable) - jail.read() - ret = jail.getOptions() - if ret: - if jail.isEnabled(): - # We only add enabled jails - self.__jails.append(jail) - else: - logSys.error("Errors in jail '" + sec + "'. Skipping...") - return False return True - + def convert(self, allow_no_files=False): """Convert read before __opts and jails to the commands stream @@ -99,6 +94,6 @@ class JailsReader(ConfigReader): # Start jails for jail in self.__jails: stream.append(["start", jail.getName()]) - + return stream - + From 149a83545fe45d1b66d0ee525ee29ccce5c2288d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Jul 2013 11:52:51 -0400 Subject: [PATCH 11/18] TST: basic test for reading of a bogus jail --- testcases/clientreadertestcase.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 695b3877..4e8cdd9d 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -126,6 +126,13 @@ class JailsReaderTest(unittest.TestCase): # commands to communicate to the server self.assertEqual(comm_commands, []) + # We should not "read" some bogus jail + old_comm_commands = comm_commands[:] # make a copy + self.assertFalse(jails.getOptions("BOGUS")) + # and there should be no side-effects + self.assertEqual(jails.convert(), old_comm_commands) + + def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises # on users' systems when enabling shipped jails From 3b52eca608d858ce42a6b6109ad5d1cc7a01d3b9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Jul 2013 12:09:33 -0400 Subject: [PATCH 12/18] ENH+TST: Ticket -- drop unused/bogus get|setFile + enh __str__ + basic testing --- server/ticket.py | 8 +------- testcases/failmanagertestcase.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/server/ticket.py b/server/ticket.py index 8826f26b..e0b2cb2e 100644 --- a/server/ticket.py +++ b/server/ticket.py @@ -47,7 +47,7 @@ class Ticket: def __str__(self): return "%s: ip=%s time=%s #attempts=%d" % \ - (self.__class__, self.__ip, self.__time, self.__attempt) + (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt) def setIP(self, value): @@ -59,12 +59,6 @@ class Ticket: def getIP(self): return self.__ip - def setFile(self, value): - self.__file = value - - def getFile(self): - return self.__file - def setTime(self, value): self.__time = value diff --git a/testcases/failmanagertestcase.py b/testcases/failmanagertestcase.py index 1bfab339..bd34e25f 100644 --- a/testcases/failmanagertestcase.py +++ b/testcases/failmanagertestcase.py @@ -78,6 +78,20 @@ class AddFailure(unittest.TestCase): ticket = self.__failManager.toBan() self.assertEqual(ticket.getIP(), "193.168.0.128") self.assertTrue(isinstance(ticket.getIP(), str)) + + # finish with rudimentary tests of the ticket + # verify consistent str + ticket_str = str(ticket) + self.assertEqual( + ticket_str, + 'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5') + # and some get/set-ers otherwise not tested + ticket.setTime(1000002000.0) + self.assertEqual(ticket.getTime(), 1000002000.0) + # and str() adjusted correspondingly + self.assertEqual( + str(ticket), + 'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5') def testbanNOK(self): self.__failManager.setMaxRetry(10) From abc41460799fb10b710b2684c0408240ef90d1ec Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 24 Jul 2013 10:27:12 +1000 Subject: [PATCH 13/18] ENH: perdition proxies other types hence daemon can include (perdidtion.(imap|pop)s?|managesieve). Also support local authentication resulting in the log message: local authentication failure --- config/filter.d/perdition.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/perdition.conf b/config/filter.d/perdition.conf index e7cb0f7d..41980cb1 100644 --- a/config/filter.d/perdition.conf +++ b/config/filter.d/perdition.conf @@ -10,7 +10,7 @@ before = common.conf [Definition] -_daemon=perdition.imaps +_daemon=perdition.\S+ -failregex = ^%(__prefix_line)sAuth: :\d+->(\d{1,3}\.){3}\d{1,3}:\d+ client-secure=\S+ authorisation_id=NONE authentication_id="\S+" server="\S+" protocol=\S+ server-secure=\S+ status="failed: Re-Authentication Failure"$ +failregex = ^%(__prefix_line)sAuth: :\d+->(\d{1,3}\.){3}\d{1,3}:\d+ client-secure=\S+ authorisation_id=NONE authentication_id="\S+" server="\S+" protocol=\S+ server-secure=\S+ status="failed: (local authentication failure|Re-Authentication Failure)"$ ^%(__prefix_line)sFatal Error reading authentication information from client :\d+->(\d{1,3}\.){3}\d{1,3}:\d+: Exiting child$ From 7d7ef081457f7c55137e18f3235b3c36c356ee3b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 24 Jul 2013 10:44:52 +1000 Subject: [PATCH 14/18] ENH: authentication_id can be an imap4 quoted string, whatever that is, so using .+ as its id --- config/filter.d/perdition.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/perdition.conf b/config/filter.d/perdition.conf index 41980cb1..7fdca14b 100644 --- a/config/filter.d/perdition.conf +++ b/config/filter.d/perdition.conf @@ -12,5 +12,5 @@ before = common.conf _daemon=perdition.\S+ -failregex = ^%(__prefix_line)sAuth: :\d+->(\d{1,3}\.){3}\d{1,3}:\d+ client-secure=\S+ authorisation_id=NONE authentication_id="\S+" server="\S+" protocol=\S+ server-secure=\S+ status="failed: (local authentication failure|Re-Authentication Failure)"$ +failregex = ^%(__prefix_line)sAuth: :\d+->(\d{1,3}\.){3}\d{1,3}:\d+ client-secure=\S+ authorisation_id=NONE authentication_id=".+" server="\S+" protocol=\S+ server-secure=\S+ status="failed: (local authentication failure|Re-Authentication Failure)"$ ^%(__prefix_line)sFatal Error reading authentication information from client :\d+->(\d{1,3}\.){3}\d{1,3}:\d+: Exiting child$ From 8f532f9148a79a4caea77275660aa7fe18e3d14e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 24 Jul 2013 11:29:58 +1000 Subject: [PATCH 15/18] NIT: space remove --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 07f9cacc..8d086da4 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -424,5 +424,5 @@ logpath = /var/log/exim/mainlog enabled = false filter = perdition action = iptables-multiport[name=perdition,port="110,143,993,995"] -logpath = /var/log/maillog +logpath = /var/log/maillog From af46cd0e4f6ed194f2661b70ea83e5c7ab5c6e77 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Jul 2013 11:34:10 +0100 Subject: [PATCH 16/18] ENH: fail2ban-regex now sorts date templates, similar to actual filters Sort occurs every 10 lines, as to not overly effect performance --- fail2ban-regex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index e19b1bc9..ff4bbd10 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -240,7 +240,7 @@ class Fail2banRegex(object): def process(self, test_lines): - for line in test_lines: + for line_no, line in enumerate(test_lines): if line.startswith('#') or not line.strip(): # skip comment and empty lines continue @@ -256,6 +256,9 @@ class Fail2banRegex(object): self._line_stats.missed_lines.append(line) self._line_stats.tested += 1 + if line_no % 10 == 0: + self._filter.dateDetector.sortTemplate() + def printLines(self, ltype): lstats = self._line_stats assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) From 879c1a32f7e1d5ea72c72c950d97c72dc6c464ef Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Jul 2013 11:36:01 +0100 Subject: [PATCH 17/18] ENH: fail2ban-regex now uses iteraable for log file Previosuly, the whole file was read in first --- fail2ban-regex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index ff4bbd10..4c5a0a11 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -377,7 +377,7 @@ if __name__ == "__main__": try: hdlr = open(cmd_log) print "Use log file : %s" % cmd_log - test_lines = hdlr.readlines() + test_lines = hdlr # Iterable except IOError, e: print e sys.exit(-1) From 1e270078b47ed8051f12b99f33857af2508f1899 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Jul 2013 20:21:05 +0100 Subject: [PATCH 18/18] TST: Warn if date templates overlap in default detectors --- server/datetemplate.py | 3 +++ testcases/datedetectortestcase.py | 41 ++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/server/datetemplate.py b/server/datetemplate.py index 86eeee8e..0754391b 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -62,6 +62,9 @@ class DateTemplate: def incHits(self): self.__hits += 1 + + def resetHits(self): + self.__hits = 0 def matchDate(self, line): dateMatch = self.__cRegex.search(line) diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index 8a838700..218abba9 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest +import unittest, calendar, datetime, re, pprint from server.datedetector import DateDetector from server.datetemplate import DateTemplate @@ -122,6 +122,45 @@ class DateDetectorTest(unittest.TestCase): self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6], m1) + def testDateDetectorTemplateOverlap(self): + patterns = [template.getPattern() + for template in self.__datedetector.getTemplates() + if hasattr(template, "getPattern")] + + year = 2008 # Leap year, 08 for %y can be confused with both %d and %m + def iterDates(year): + for month in xrange(1, 13): + for day in xrange(2, calendar.monthrange(year, month)[1]+1, 9): + for hour in xrange(0, 24, 6): + for minute in xrange(0, 60, 15): + for second in xrange(0, 60, 15): # Far enough? + yield datetime.datetime( + year, month, day, hour, minute, second) + + overlapedTemplates = set() + for date in iterDates(year): + for pattern in patterns: + datestr = date.strftime(pattern) + datestrs = set([ + datestr, + re.sub(r"(\s)0", r"\1 ", datestr), + re.sub(r"(\s)0", r"\1", datestr)]) + for template in self.__datedetector.getTemplates(): + template.resetHits() + for datestr in datestrs: + if template.matchDate(datestr): # or getDate? + template.incHits() + + matchedTemplates = [template + for template in self.__datedetector.getTemplates() + if template.getHits() > 0] + assert matchedTemplates != [] # Should match at least one + if len(matchedTemplates) > 1: + overlapedTemplates.add((pattern, tuple(sorted(template.getName() + for template in matchedTemplates)))) + if overlapedTemplates: + print "WARNING: The following date templates overlap:" + pprint.pprint(overlapedTemplates) # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")