From fc27e00290af3b455118054fec13a1bdf154ede2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 7 Dec 2012 15:17:08 -0500 Subject: [PATCH 001/114] ENH: tune up sshd-ddos to use common.conf and allow training spaces --- config/filter.d/sshd-ddos.conf | 12 +++++++++--- testcases/files/logs/sshd-ddos | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 testcases/files/logs/sshd-ddos diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf index 4f4b6fa2..266594ba 100644 --- a/config/filter.d/sshd-ddos.conf +++ b/config/filter.d/sshd-ddos.conf @@ -2,11 +2,17 @@ # # Author: Yaroslav Halchenko # -# $Revision$ -# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf [Definition] +_daemon = sshd + # Option: failregex # Notes.: regex to match the password failures messages in the logfile. The # host must be matched by a group named "host". The tag "" can @@ -14,7 +20,7 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = sshd(?:\[\d+\])?: Did not receive identification string from $ +failregex = ^%(__prefix_line)sDid not receive identification string from \s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/sshd-ddos b/testcases/files/logs/sshd-ddos new file mode 100644 index 00000000..d71c6bb2 --- /dev/null +++ b/testcases/files/logs/sshd-ddos @@ -0,0 +1,2 @@ +# http://forums.powervps.com/showthread.php?t=1667 +Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 From 7ede1e85187764caec77512146f69c76f0621d28 Mon Sep 17 00:00:00 2001 From: hamilton5 Date: Mon, 10 Dec 2012 19:17:04 -0500 Subject: [PATCH 002/114] Update config/filter.d/dovecot.conf added failregex line for debian and centos per http://www.fail2ban.org/wiki/index.php/Talk:Dovecot --- config/filter.d/dovecot.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 153b9bb0..f37dc892 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -15,6 +15,7 @@ # Values: TEXT # failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(?P\S*),.* + pam.*dovecot.*(?:authentication failure).*rhost=(?:::f{4,6}:)?(?P\S*) # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From e040c6d8a3bfa62d358083300119c259cd44dcd0 Mon Sep 17 00:00:00 2001 From: hamilton5 Date: Tue, 11 Dec 2012 03:26:14 -0500 Subject: [PATCH 003/114] Update config/filter.d/dovecot.conf site actually needs updated because of alias per Notes above. --- config/filter.d/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index f37dc892..acbae3a8 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -15,7 +15,7 @@ # Values: TEXT # failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(?P\S*),.* - pam.*dovecot.*(?:authentication failure).*rhost=(?:::f{4,6}:)?(?P\S*) + pam.*dovecot.*(?:authentication failure).*rhost= # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From c534c1d03d7d4cf59941fd6b7cc7b1dee62d5cd1 Mon Sep 17 00:00:00 2001 From: hamilton5 Date: Tue, 11 Dec 2012 11:05:22 -0500 Subject: [PATCH 004/114] Update testcases/files/logs/dovecot --- testcases/files/logs/dovecot | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index b975e808..a469d892 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1 +1,6 @@ -@400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS +@400000004c91b044077a9e94 + imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS + +@e040c6d8a3bfa62d358083300119c259cd44dcd0 + dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 + dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root From 7bd977e2df2877614b9eb0547d58b5fb537b1074 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 11 Dec 2012 10:59:01 -0500 Subject: [PATCH 005/114] ENH: fail2ban-testscases -- allow to specify regexps for tests to be ran Eventually we will switch to use nose or py.test -- for now this homebrew solution could be used to run selected suites only --- fail2ban-testcases | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index aaf78525..0ee2c53c 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -42,7 +42,7 @@ from optparse import OptionParser, Option def get_opt_parser(): # use module docstring for help output p = OptionParser( - usage="%s [OPTIONS]\n" % sys.argv[0] + __doc__, + usage="%s [OPTIONS] [regexps]\n" % sys.argv[0] + __doc__, version="%prog " + version) p.add_options([ @@ -56,8 +56,7 @@ def get_opt_parser(): return p parser = get_opt_parser() -(opts, files) = parser.parse_args() -assert(not len(files)) +(opts, regexps) = parser.parse_args() # # Logging @@ -103,7 +102,20 @@ if not opts.log_level or opts.log_level != 'fatal': # # Gather the tests # -tests = unittest.TestSuite() +if not len(regexps): + tests = unittest.TestSuite() +else: + import re + class FilteredTestSuite(unittest.TestSuite): + _regexps = [re.compile(r) for r in regexps] + def addTest(self, suite): + suite_str = str(suite) + for r in self._regexps: + if r.search(suite_str): + super(FilteredTestSuite, self).addTest(suite) + return + + tests = FilteredTestSuite() # Server #tests.addTest(unittest.makeSuite(servertestcase.StartStop)) From 2b6366656f15d63985a2a5b7d5a73c2364de1132 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 11 Dec 2012 11:04:40 -0500 Subject: [PATCH 006/114] BF: make sorting of date templates stable Before, it would first do stable sort followed with explicit reverse. Now reverse is given as an argument to sort, and it results in actually preserving the order in case of e.g. no sorting needed --- server/datedetector.py | 5 +++-- testcases/datedetectortestcase.py | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/datedetector.py b/server/datedetector.py index 7301ec91..5b1f3c6c 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -194,7 +194,8 @@ class DateDetector: self.__lock.acquire() try: logSys.debug("Sorting the template list") - self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits())) - self.__templates.reverse() + self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits()), reverse=True) + t = self.__templates[0] + logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits())) finally: self.__lock.release() diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index 1f73cd1e..d6946144 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -78,6 +78,14 @@ class DateDetectorTest(unittest.TestCase): self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6]) self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix) + def testStableSortTemplate(self): + old_names = [x.getName() for x in self.__datedetector.getTemplates()] + self.__datedetector.sortTemplate() + # If there were no hits -- sorting should not change the order + for old_name, n in zip(old_names, self.__datedetector.getTemplates()): + self.assertEqual(old_name, n.getName()) # "Sort must be stable" + + # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") # self.__datedetector.setDefaultPattern("%b %d %H:%M:%S") From d1625253eb2d8d8c3ce2df984b95c9be3ad71dcc Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 11 Dec 2012 11:06:03 -0500 Subject: [PATCH 007/114] ENH: debug msgs on which template was taken (+ use "is" for None comparisons) --- server/datedetector.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/datedetector.py b/server/datedetector.py index 5b1f3c6c..f0ba2635 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -77,7 +77,7 @@ class DateDetector: # previous one but with year given by 2 digits # (See http://bugs.debian.org/537610) template = DateStrptime() - template.setName("Day/Month/Year Hour:Minute:Second") + template.setName("Day/Month/Year2 Hour:Minute:Second") template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%d/%m/%y %H:%M:%S") self.__templates.append(template) @@ -158,7 +158,8 @@ class DateDetector: try: for template in self.__templates: match = template.matchDate(line) - if not match == None: + if not match is None: + logSys.debug("Matched time template %s" % template.getName()) return match return None finally: @@ -170,8 +171,9 @@ class DateDetector: for template in self.__templates: try: date = template.getDate(line) - if date == None: + if date is None: continue + logSys.debug("Got time using template %s" % template.getName()) return date except ValueError: pass From 67145d8b78a557931e8d8cb1d5b61a92748ac10b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 11 Dec 2012 11:13:36 -0500 Subject: [PATCH 008/114] ENH: assure that all date templates have unique names --- server/datedetector.py | 44 ++++++++++++++++++------------- testcases/datedetectortestcase.py | 5 +++- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/server/datedetector.py b/server/datedetector.py index f0ba2635..c013d551 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -40,6 +40,14 @@ class DateDetector: def __init__(self): self.__lock = Lock() self.__templates = list() + self.__known_names = set() + + def _appendTemplate(self, template): + name = template.getName() + if name in self.__known_names: + raise ValueError("There is already a template with name %s" % name) + self.__known_names.add(name) + self.__templates.append(template) def addDefaultTemplate(self): self.__lock.acquire() @@ -49,104 +57,104 @@ class DateDetector: template.setName("MONTH Day Hour:Minute:Second") template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") template.setPattern("%b %d %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # asctime template = DateStrptime() template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year") template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}") template.setPattern("%a %b %d %H:%M:%S %Y") - self.__templates.append(template) + self._appendTemplate(template) # asctime without year template = DateStrptime() template.setName("WEEKDAY MONTH Day Hour:Minute:Second") template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") template.setPattern("%a %b %d %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # simple date template = DateStrptime() template.setName("Year/Month/Day Hour:Minute:Second") template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%Y/%m/%d %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # simple date too (from x11vnc) template = DateStrptime() template.setName("Day/Month/Year Hour:Minute:Second") template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}") template.setPattern("%d/%m/%Y %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # previous one but with year given by 2 digits # (See http://bugs.debian.org/537610) template = DateStrptime() template.setName("Day/Month/Year2 Hour:Minute:Second") template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%d/%m/%y %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # Apache format [31/Oct/2006:09:22:55 -0000] template = DateStrptime() template.setName("Day/MONTH/Year:Hour:Minute:Second") template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}") template.setPattern("%d/%b/%Y:%H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # CPanel 05/20/2008:01:57:39 template = DateStrptime() template.setName("Month/Day/Year:Hour:Minute:Second") template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}") template.setPattern("%m/%d/%Y:%H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # Exim 2006-12-21 06:43:20 template = DateStrptime() template.setName("Year-Month-Day Hour:Minute:Second") template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%Y-%m-%d %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # custom for syslog-ng 2006.12.21 06:43:20 template = DateStrptime() template.setName("Year.Month.Day Hour:Minute:Second") template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%Y.%m.%d %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # named 26-Jul-2007 15:20:52.252 template = DateStrptime() template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]") template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}") template.setPattern("%d-%b-%Y %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # 17-07-2008 17:23:25 template = DateStrptime() template.setName("Day-Month-Year Hour:Minute:Second") template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}") template.setPattern("%d-%m-%Y %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # 01-27-2012 16:22:44.252 template = DateStrptime() template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]") template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}") template.setPattern("%m-%d-%Y %H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # TAI64N template = DateTai64n() template.setName("TAI64N") - self.__templates.append(template) + self._appendTemplate(template) # Epoch template = DateEpoch() template.setName("Epoch") - self.__templates.append(template) + self._appendTemplate(template) # ISO 8601 template = DateISO8601() template.setName("ISO 8601") - self.__templates.append(template) + self._appendTemplate(template) # Only time information in the log template = DateStrptime() template.setName("Hour:Minute:Second") template.setRegex("^\d{2}:\d{2}:\d{2}") template.setPattern("%H:%M:%S") - self.__templates.append(template) + self._appendTemplate(template) # <09/16/08@05:03:30> template = DateStrptime() template.setName("") template.setRegex("^<\d{2}/\d{2}/\d{2}@\d{2}:\d{2}:\d{2}>") template.setPattern("<%m/%d/%y@%H:%M:%S>") - self.__templates.append(template) + self._appendTemplate(template) finally: self.__lock.release() diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index d6946144..34ce22ce 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -85,7 +85,10 @@ class DateDetectorTest(unittest.TestCase): for old_name, n in zip(old_names, self.__datedetector.getTemplates()): self.assertEqual(old_name, n.getName()) # "Sort must be stable" - + def testAllUniqueTemplateNames(self): + self.assertRaises(ValueError, self.__datedetector._appendTemplate, + self.__datedetector.getTemplates()[0]) + # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") # self.__datedetector.setDefaultPattern("%b %d %H:%M:%S") From ccc62ddbf353dedcc90f18469f6e7ebe9db4e565 Mon Sep 17 00:00:00 2001 From: hamilton5 Date: Tue, 11 Dec 2012 12:05:01 -0500 Subject: [PATCH 009/114] Update testcases/files/logs/dovecot --- testcases/files/logs/dovecot | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index a469d892..f76c1bce 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1,6 +1,4 @@ -@400000004c91b044077a9e94 - imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS +@400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS -@e040c6d8a3bfa62d358083300119c259cd44dcd0 - dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 - dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root +@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 +@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root From 266cdc29a6bf5aafc7d35f000a5254f416231c73 Mon Sep 17 00:00:00 2001 From: hamilton5 Date: Tue, 11 Dec 2012 12:09:28 -0500 Subject: [PATCH 010/114] Update config/filter.d/dovecot.conf even tho not on the fail2ban site.. suggested to not be greedy by yarikoptic --- config/filter.d/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index acbae3a8..42c5ef33 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -15,7 +15,7 @@ # Values: TEXT # failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(?P\S*),.* - pam.*dovecot.*(?:authentication failure).*rhost= + pam.*dovecot.*(?:authentication failure).*rhost=(?:\s+user=.*)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From b95dc592b92ea136da314b2849c756cb4b2b7f3c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 12 Dec 2012 21:43:27 +1100 Subject: [PATCH 011/114] ignore build directory and compiled python bits --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b25c15b8..6a0d5e64 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *~ +build +*.pyc From fd7929863b1c33e8e212e4b85175c968378b8325 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 12 Dec 2012 21:59:01 +1100 Subject: [PATCH 012/114] name -> IP is a normal DNS lookup not a reverse --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index b37e37e6..fb79fcbe 100644 --- a/server/filter.py +++ b/server/filter.py @@ -639,7 +639,7 @@ class DNSUtils: ip = DNSUtils.dnsToIp(text) ipList.extend(ip) if ip and useDns == "warn": - logSys.warning("Determined IP using DNS Reverse Lookup: %s = %s", + logSys.warning("Determined IP using DNS Lookup: %s = %s", text, ipList) return ipList From 68c1defb76a50bdc0ad3600519df3f446eaab72b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 12 Dec 2012 09:16:27 -0500 Subject: [PATCH 013/114] ENH: added dovecot example from Daniel Black + example with DoS attempt via injected rhost --- testcases/files/logs/dovecot | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index f76c1bce..434acade 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1,4 +1,8 @@ @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 +# Above example with injected rhost into ruser -- should not match for 1.2.3.4 +@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root + +Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 From 3969e3f77b6fccf6d308c42c9182f9d58f2ee406 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 12 Dec 2012 09:16:52 -0500 Subject: [PATCH 014/114] ENH: dovecot.conf - require space(s) before rip/rhost log entry --- config/filter.d/dovecot.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 42c5ef33..18451e42 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -14,8 +14,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(?P\S*),.* - pam.*dovecot.*(?:authentication failure).*rhost=(?:\s+user=.*)?\s*$ +failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*\s+rip=(?P\S*),.* + pam.*dovecot.*(?:authentication failure).*\s+rhost=(?:\s+user=.*)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From dc67b24270a972b7097fc039692f557330d859a6 Mon Sep 17 00:00:00 2001 From: pigsyn Date: Wed, 12 Dec 2012 23:07:39 +0100 Subject: [PATCH 015/114] Update config/filter.d/webmin-auth.conf Added a trailing '.*$' to each regex so they can find expressions in targeted log files. --- config/filter.d/webmin-auth.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/webmin-auth.conf b/config/filter.d/webmin-auth.conf index 70997e01..fc0b7760 100644 --- a/config/filter.d/webmin-auth.conf +++ b/config/filter.d/webmin-auth.conf @@ -18,8 +18,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = webmin.* Non-existent login as .+ from $ - webmin.* Invalid login as .+ from $ +failregex = webmin.* Non-existent login as .+ from .*$ + webmin.* Invalid login as .+ from .*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From f336d9f8765e8e6eafb9c683caeffaa5a4c5c34d Mon Sep 17 00:00:00 2001 From: pigsyn Date: Thu, 13 Dec 2012 08:14:49 +0100 Subject: [PATCH 016/114] Update config/filter.d/webmin-auth.conf Added '\s*$' to the regular expression to match the space written by webmin logs at line-endings --- config/filter.d/webmin-auth.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/webmin-auth.conf b/config/filter.d/webmin-auth.conf index fc0b7760..b1df45dc 100644 --- a/config/filter.d/webmin-auth.conf +++ b/config/filter.d/webmin-auth.conf @@ -18,8 +18,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = webmin.* Non-existent login as .+ from .*$ - webmin.* Invalid login as .+ from .*$ +failregex = webmin.* Non-existent login as .+ from \s*$ + webmin.* Invalid login as .+ from \s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 38dd1506cc631f0ee5900a4e01b1c615d4e2716e Mon Sep 17 00:00:00 2001 From: pigsyn Date: Wed, 12 Dec 2012 23:25:31 -0800 Subject: [PATCH 017/114] Sample Webmin logs --- testcases/files/logs/Webmin | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 testcases/files/logs/Webmin diff --git a/testcases/files/logs/Webmin b/testcases/files/logs/Webmin new file mode 100644 index 00000000..4be0f1c4 --- /dev/null +++ b/testcases/files/logs/Webmin @@ -0,0 +1,7 @@ +# Webmin authentication failures from /var/log/auth.log + +#1 User exists, bad password +Dec 13 08:15:18 sb1 webmin[25875]: Invalid login as root from 89.2.49.230 + +#2 User does not exists +Dec 12 23:14:19 sb1 webmin[22134]: Non-existent login as robert from 188.40.105.142 From 123d457924f814d17c29398bad909002283a6805 Mon Sep 17 00:00:00 2001 From: pigsyn Date: Thu, 13 Dec 2012 08:33:07 +0100 Subject: [PATCH 018/114] Update testcases/files/logs/Webmin --- testcases/files/logs/Webmin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/Webmin b/testcases/files/logs/Webmin index 4be0f1c4..86d3f321 100644 --- a/testcases/files/logs/Webmin +++ b/testcases/files/logs/Webmin @@ -1,4 +1,4 @@ -# Webmin authentication failures from /var/log/auth.log +#Webmin authentication failures from /var/log/auth.log #1 User exists, bad password Dec 13 08:15:18 sb1 webmin[25875]: Invalid login as root from 89.2.49.230 From 21e966e4bb291142bc83494fc479e8895f0ff474 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 13 Dec 2012 08:24:02 -0500 Subject: [PATCH 019/114] example logs should carry the same name as the filter they are devised for --- testcases/files/logs/{Webmin => webmin-auth} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename testcases/files/logs/{Webmin => webmin-auth} (100%) diff --git a/testcases/files/logs/Webmin b/testcases/files/logs/webmin-auth similarity index 100% rename from testcases/files/logs/Webmin rename to testcases/files/logs/webmin-auth From 05af52e833bca96d3d67cb30378e51856ad3458e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 24 Dec 2012 11:05:44 -0500 Subject: [PATCH 020/114] ENH: fail2ban-regex -- __str__ for RegexStat + modeline --- fail2ban-regex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fail2ban-regex b/fail2ban-regex index 3900c909..a42ed96d 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -1,4 +1,7 @@ #!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify @@ -42,6 +45,10 @@ class RegexStat: self.__stats = 0 self.__failregex = failregex self.__ipList = list() + + def __str__(self): + return "%s(%r) %d failed: %s" \ + % (self.__class__, self.__failregex, self.__stats, self.__ipList) def inc(self): self.__stats += 1 From abd5984234dec44efe1f98c05d124501a60bb146 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 11 Dec 2012 22:22:51 +1100 Subject: [PATCH 021/114] base ipset support --- config/action.d/iptables-ipset-proto4.conf | 58 +++++++++++++++++++ config/action.d/iptables-ipset-proto6.conf | 65 ++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 config/action.d/iptables-ipset-proto4.conf create mode 100644 config/action.d/iptables-ipset-proto6.conf diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf new file mode 100644 index 00000000..21401ad7 --- /dev/null +++ b/config/action.d/iptables-ipset-proto4.conf @@ -0,0 +1,58 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# +# Tested against protocol 4 (ipset v4.2) +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = ipset --create fail2ban- iphash + iptables -I INPUT -p -m multiport --dports -m set --match-set fail2ban- src -j DROP + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = iptables -D INPUT -p -m multiport --dports -m set --match-set fail2ban- src -j DROP + ipset --flush fail2ban- + ipset --destroy fail2ban- + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionban = ipset --test fail2ban- || ipset --add fail2ban- -exist + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionunban = ipset --test fail2ban- && ipset --del fail2ban- + +[Init] + +# Defaut name of the chain +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: ssh +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp | icmp | all ] Default: tcp +# +protocol = tcp + diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf new file mode 100644 index 00000000..084f8738 --- /dev/null +++ b/config/action.d/iptables-ipset-proto6.conf @@ -0,0 +1,65 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# +# Tested against protocol 6 (ipset v6.14) +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = ipset create fail2ban- hash:ip timeout + iptables -I INPUT -p -m multiport --dports -m set --match-set fail2ban- src -j DROP + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = iptables -D INPUT -p -m multiport --dports -m set --match-set fail2ban- src -j DROP + ipset flush fail2ban- + ipset destroy fail2ban- + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionban = ipset add fail2ban- -exist + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionunban = ipset del fail2ban- -exist + +[Init] + +# Defaut name of the chain +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: ssh +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp | icmp | all ] Default: tcp +# +protocol = tcp + +# Option: bantime +# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) +# Values: [ NUM ] Default: 600 + +bantime = 600 + + From 9221886df69091f07a0feaff6e617749b804b452 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 11 Dec 2012 23:58:48 +1100 Subject: [PATCH 022/114] more documentation and optimisations/fixes based on testing --- config/action.d/iptables-ipset-proto4.conf | 19 ++++++++++++++++--- config/action.d/iptables-ipset-proto6.conf | 19 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index 21401ad7..f7d03f67 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -2,8 +2,21 @@ # # Author: Daniel Black # -# Tested against protocol 4 (ipset v4.2) +# This is for ipset protocol 4 (ipset v4.2). If you have a later version +# of ipset try to use the iptables-ipset-proto6.conf as it does some things +# nicer. +# +# This requires the program ipset which is normally in package called ipset. # +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# +# If you are running on an older kernel you make need to patch in external +# modules. +# +# On Debian machines this can be done with: +# +# apt-get install ipset xtables-addons-source +# module-assistant auto-install xtables-addons [Definition] @@ -28,7 +41,7 @@ actionstop = iptables -D INPUT -p -m multiport --dports -m set # Tags: IP address # Values: CMD # -actionban = ipset --test fail2ban- || ipset --add fail2ban- -exist +actionban = ipset --test fail2ban- || ipset --add fail2ban- # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -40,7 +53,7 @@ actionunban = ipset --test fail2ban- && ipset --del fail2ban- < [Init] -# Defaut name of the chain +# Defaut name of the ipset # name = default diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 084f8738..3352d63d 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -2,8 +2,21 @@ # # Author: Daniel Black # -# Tested against protocol 6 (ipset v6.14) +# This is for ipset protocol 6 (and hopefully later) (ipset v6.14). +# Use ipset -V to see the protocol and version. Version 4 should use +# iptables-ipset-proto4.conf. # +# This requires the program ipset which is normally in package called ipset. +# +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# +# If you are running on an older kernel you make need to patch in external +# modules. +# +# On Debian machines this can be done with: +# +# apt-get install ipset xtables-addons-source +# module-assistant auto-install xtables-addons [Definition] @@ -28,7 +41,7 @@ actionstop = iptables -D INPUT -p -m multiport --dports -m set # Tags: IP address # Values: CMD # -actionban = ipset add fail2ban- -exist +actionban = ipset add fail2ban- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -40,7 +53,7 @@ actionunban = ipset del fail2ban- -exist [Init] -# Defaut name of the chain +# Defaut name of the ipset # name = default From da0ba8ab4cbba68cdf570419d5c14704485e4a96 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 31 Dec 2012 14:38:51 +1100 Subject: [PATCH 023/114] ENH: add example jail for ipset --- config/jail.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index ff0287a2..3f2425b4 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -101,6 +101,26 @@ action = hostsdeny ignoreregex = for myuser from logpath = /var/log/sshd.log +# Here we use a combination of Netfilter/Iptables and IPsets +# for storing large volumes of banned IPs +# +# IPset comes in two versions. See ipset -V for which one to use +# requires the ipset package and kernel support. +[ssh-iptables-ipset4] + +enabled = false +filter = sshd +action = iptables-ipset-proto4[name=SSH, port=ssh, protocol=tcp] +logpath = /var/log/sshd.log +maxretry = 5 + +[ssh-iptables-ipset6] +enabled = false +filter = sshd +action = iptables-ipset-proto6[name=SSH, port=ssh, protocol=tcp, bantime=600] +logpath = /var/log/sshd.log +maxretry = 5 + # This jail demonstrates the use of wildcards in "logpath". # Moreover, it is possible to give other files on a new line. From f9b78ba92798da40509c5743c1d15e13d6a7f3ec Mon Sep 17 00:00:00 2001 From: Michael Gebetsroither Date: Thu, 3 Jan 2013 18:46:31 +0100 Subject: [PATCH 024/114] add support for blocking through blackhole routes --- config/action.d/route.conf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 config/action.d/route.conf diff --git a/config/action.d/route.conf b/config/action.d/route.conf new file mode 100644 index 00000000..2d11c700 --- /dev/null +++ b/config/action.d/route.conf @@ -0,0 +1,19 @@ +# Fail2Ban configuration file +# +# Author: Michael Gebetsroither +# +# This is for blocking whole hosts through blackhole routes. +# +# PRO: +# - Works on all kernel versions and as no compatibility problems (back to debian lenny and WAY further). +# - It's FAST for very large numbers of blocked ips. +# - It's FAST because it Blocks traffic before it enters common iptables chains used for filtering. +# - It's per host, ideal as action against ssh password bruteforcing to block further attack attempts. +# - No additional software required beside iproute/iproute2 +# +# CON: +# - Blocking is per IP and NOT per service, but ideal as action against ssh password bruteforcing hosts + +[Definition] +actionban = ip route add blackhole +actionunban = ip route del blackhole From 03433f79cd52a87c3fd7309e7fe9967076bbb866 Mon Sep 17 00:00:00 2001 From: Michael Gebetsroither Date: Fri, 4 Jan 2013 16:09:04 +0100 Subject: [PATCH 025/114] add example jail.conf for blocking through blackhole routes for ssh --- config/jail.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index 3f2425b4..fb9f9ca0 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -101,6 +101,17 @@ action = hostsdeny ignoreregex = for myuser from logpath = /var/log/sshd.log +# Here we use blackhole routes for not requiring any additional kernel support +# to store large volumes of banned IPs + +[ssh-route] + +enabled = false +filter = sshd +action = route +logpath = /var/log/sshd.log +maxretry = 5 + # Here we use a combination of Netfilter/Iptables and IPsets # for storing large volumes of banned IPs # From 8f0c533d644bfb2316661d2b204658950d0ab37c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 4 Jan 2013 10:55:14 -0500 Subject: [PATCH 026/114] DOC: Mention that logrotate configuration needs to be adjusted if logtarget is changed (Closes: #697333) --- config/fail2ban.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index a8e2eb9a..f2f1b215 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -24,6 +24,10 @@ loglevel = 3 # Option: logtarget # Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. # Only one log target can be specified. +# If you change logtarget from the default value and you are +# using logrotate -- also adjust or disable rotation in the +# corresponding configuration file +# (e.g. /etc/logrotate.d/fail2ban on Debian systems) # Values: STDOUT STDERR SYSLOG file Default: /var/log/fail2ban.log # logtarget = /var/log/fail2ban.log From 3ce53e87984d7abc84ea4d34ded985e31f29d830 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 4 Jan 2013 15:22:18 -0500 Subject: [PATCH 027/114] ENH: Added login authenticator failed regexp for exim filter --- config/filter.d/exim.conf | 1 + testcases/files/logs/exim | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 testcases/files/logs/exim diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index a25ef3db..8bf4fc5f 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -15,6 +15,7 @@ # Values: TEXT # failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) + login authenticator failed for .* \[\]: 535 Incorrect authentication data \(set_id=.*\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim new file mode 100644 index 00000000..dd507379 --- /dev/null +++ b/testcases/files/logs/exim @@ -0,0 +1,2 @@ +# From IRC 2013-01-04 +2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) From b3d8ba146b52478262d0637b8bab06cb6cb6aa20 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 4 Jan 2013 10:55:14 -0500 Subject: [PATCH 028/114] DOC: Mention that logrotate configuration needs to be adjusted if logtarget is changed (Closes: #697333) --- config/fail2ban.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index a8e2eb9a..f2f1b215 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -24,6 +24,10 @@ loglevel = 3 # Option: logtarget # Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. # Only one log target can be specified. +# If you change logtarget from the default value and you are +# using logrotate -- also adjust or disable rotation in the +# corresponding configuration file +# (e.g. /etc/logrotate.d/fail2ban on Debian systems) # Values: STDOUT STDERR SYSLOG file Default: /var/log/fail2ban.log # logtarget = /var/log/fail2ban.log From 9a392928132efccc5f93ba0e2e4717e781d826c3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 4 Jan 2013 15:22:18 -0500 Subject: [PATCH 029/114] ENH: Added login authenticator failed regexp for exim filter --- config/filter.d/exim.conf | 1 + testcases/files/logs/exim | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 testcases/files/logs/exim diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index a25ef3db..8bf4fc5f 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -15,6 +15,7 @@ # Values: TEXT # failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) + login authenticator failed for .* \[\]: 535 Incorrect authentication data \(set_id=.*\)\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/testcases/files/logs/exim b/testcases/files/logs/exim new file mode 100644 index 00000000..dd507379 --- /dev/null +++ b/testcases/files/logs/exim @@ -0,0 +1,2 @@ +# From IRC 2013-01-04 +2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) From bb7628591c124309774d5f4787d2a4f07cf6b906 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 18 Jan 2013 14:44:49 -0700 Subject: [PATCH 030/114] Update config/filter.d/sshd.conf Do not trigger sshd bans on pam_unix authentication failures, this will trigger on successful logins on systems that use non-pam_unix authentication (sssd, ldap, etc.). --- config/filter.d/sshd.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 88615957..e4339c78 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -30,7 +30,6 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ - ^%(__prefix_line)s(?:pam_unix\(sshd:auth\):\s)?authentication failure; logname=\S* uid=\S* euid=\S* tty=\S* ruser=\S* rhost=(?:\s+user=.*)?\s*$ ^%(__prefix_line)srefused connect from \S+ \(\)\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ From 96eb8986cc732c8a021e6a9f31d8505b976782b9 Mon Sep 17 00:00:00 2001 From: blotus Date: Fri, 25 Jan 2013 13:37:22 +0100 Subject: [PATCH 031/114] Escape ' and " in matches tag --- server/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/action.py b/server/action.py index f2614b33..35974c70 100644 --- a/server/action.py +++ b/server/action.py @@ -243,7 +243,7 @@ class Action: return Action.executeCmd(stopCmd) def escapeTag(tag): - for c in '\\#&;`|*?~<>^()[]{}$\n': + for c in '\\#&;`|*?~<>^()[]{}$\n\'"': if c in tag: tag = tag.replace(c, '\\' + c) return tag From fdd9dfb4b5c543ba31f1c21b67f26ce1d9decfe9 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 25 Jan 2013 12:56:00 -0700 Subject: [PATCH 032/114] Initial support for --no-network option for fail2ban-testcases --- fail2ban-testcases | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 0ee2c53c..20d3b226 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -53,6 +53,12 @@ def get_opt_parser(): help="Log level for the logger to use during running tests"), ]) + p.add_options([ + Option('-n', "--no-network", action="store_true", + dest="no_network", + help="Do not run tests that require the network"), + ]) + return p parser = get_opt_parser() @@ -90,6 +96,8 @@ else: stdout.setFormatter(logging.Formatter(' %(message)s')) logSys.addHandler(stdout) +if opts.no_network is None: + opts.no_network = False # # Let know the version @@ -129,11 +137,13 @@ tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) # Filter -tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) +if not opts.no_network: + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) -tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) -tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) +if not opts.no_network: + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) # DateDetector From 6b2e76ba7fce1841411183a72a2c75b6826c4b4c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 25 Jan 2013 16:01:35 -0500 Subject: [PATCH 033/114] BF: pyinotify - use bitwise op on masks and do not try tracking newly created directories --- server/filterpyinotify.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/filterpyinotify.py b/server/filterpyinotify.py index fdc7256e..a2eea9d7 100644 --- a/server/filterpyinotify.py +++ b/server/filterpyinotify.py @@ -65,7 +65,11 @@ class FilterPyinotify(FileFilter): def callback(self, event): path = event.pathname - if event.mask == pyinotify.IN_CREATE: + if event.mask & pyinotify.IN_CREATE: + # skip directories altogether + if event.mask & pyinotify.IN_ISDIR: + logSys.debug("Ignoring creation of directory %s" % path) + return # check if that is a file we care about if not path in self.__watches: logSys.debug("Ignoring creation of %s we do not monitor" % path) From 431489c9b94c001200ae8ec15935a3d75d972940 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Fri, 25 Jan 2013 14:19:10 -0700 Subject: [PATCH 034/114] Remove unneeded setting of opts.no_network --- fail2ban-testcases | 3 --- 1 file changed, 3 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 20d3b226..99fefd57 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -96,9 +96,6 @@ else: stdout.setFormatter(logging.Formatter(' %(message)s')) logSys.addHandler(stdout) -if opts.no_network is None: - opts.no_network = False - # # Let know the version # From acab23bdfe12a2fa76d1d30e55a1daeb4ffa909f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 28 Jan 2013 09:46:50 -0500 Subject: [PATCH 035/114] RF: move exceptions used by both client and server into common/exceptions.py this prevents importing of server while operating with client only --- client/beautifier.py | 14 ++++---------- common/exceptions.py | 36 ++++++++++++++++++++++++++++++++++++ server/jails.py | 17 +++-------------- 3 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 common/exceptions.py diff --git a/client/beautifier.py b/client/beautifier.py index a75655e7..7e48016c 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -17,20 +17,14 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# Author: Cyril Jaquier -# -# $Revision$ - -__author__ = "Cyril Jaquier" -__version__ = "$Revision$" -__date__ = "$Date$" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" -from server.jails import UnknownJailException -from server.jails import DuplicateJailException import logging +from common.exceptions import UnknownJailException, DuplicateJailException + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.client.config") diff --git a/common/exceptions.py b/common/exceptions.py new file mode 100644 index 00000000..7e933544 --- /dev/null +++ b/common/exceptions.py @@ -0,0 +1,36 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +"""Fail2Ban exceptions used by both client and server + +""" +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko" +__license__ = "GPL" + +# +# Jails +# +class DuplicateJailException(Exception): + pass + +class UnknownJailException(Exception): + pass + + + diff --git a/server/jails.py b/server/jails.py index 3be38f70..4bf5f971 100644 --- a/server/jails.py +++ b/server/jails.py @@ -17,16 +17,11 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# Author: Cyril Jaquier -# -# $Revision$ - -__author__ = "Cyril Jaquier" -__version__ = "$Revision$" -__date__ = "$Date$" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" +from common.exceptions import DuplicateJailException, UnknownJailException from jail import Jail from threading import Lock @@ -160,9 +155,3 @@ class Jails: finally: self.__lock.release() - -class DuplicateJailException(Exception): - pass - -class UnknownJailException(Exception): - pass From d561a4c2bbc336db70d5923cf630813bc51dc3ee Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 28 Jan 2013 09:54:08 -0500 Subject: [PATCH 036/114] BF: do not rely on scripts being under /usr -- might differ eg on Fedora -- rely on import of common.version (Closes gh-112) This is also not ideal, since if there happens to be some systemwide common.version -- we are doomed but otherwise, we cannot keep extending comparison check to /bin, /sbin whatelse --- fail2ban-client | 9 +++++---- fail2ban-regex | 9 +++++---- fail2ban-server | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fail2ban-client b/fail2ban-client index 1d8eb15e..13d018e6 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -27,12 +27,13 @@ import getopt, time, shlex, socket # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -client is also under /usr/ +try: + from common.version import version +except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version -# Now we can import our modules -from common.version import version +# Now we can import the rest of modules from common.protocol import printFormatted from client.csocket import CSocket from client.configurator import Configurator diff --git a/fail2ban-regex b/fail2ban-regex index a42ed96d..f9bc72c1 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -26,13 +26,14 @@ import getopt, sys, time, logging, os # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -regex is also under /usr/ - sys.path.insert(1, "/usr/share/fail2ban") +try: + from common.version import version +except ImportError, e: + sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError -from common.version import version from server.filter import Filter from server.failregex import RegexException diff --git a/fail2ban-server b/fail2ban-server index bd86e6cd..0f3410c9 100755 --- a/fail2ban-server +++ b/fail2ban-server @@ -26,11 +26,12 @@ import getopt, sys, logging, os # Inserts our own modules path first in the list # fix for bug #343821 -if os.path.abspath(__file__).startswith('/usr/'): - # makes sense to use system-wide library iff -server is also under /usr/ +try: + from common.version import version +except ImportError, e: sys.path.insert(1, "/usr/share/fail2ban") + from common.version import version -from common.version import version from server.server import Server # Gets the instance of the logger. From bf5f46c3d597c3914ba8fa138e172aa5db10ede2 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 30 Jan 2013 19:57:03 +0000 Subject: [PATCH 037/114] Warn if config file present but unreadable --- client/configreader.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/configreader.py b/client/configreader.py index 063484e8..e435b254 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -59,6 +59,16 @@ class ConfigReader(SafeConfigParserWithIncludes): bConf = basename + ".conf" bLocal = basename + ".local" if os.path.exists(bConf) or os.path.exists(bLocal): + if not os.access(bConf, os.R_OK) and not os.access(bLocal, os.R_OK): + logSys.warning( + "Unable to read either \"%s\" or \"%s\"" % (bConf, bLocal)) + return False + elif os.path.exists(bConf) and not os.access(bConf, os.R_OK): + logSys.warning( + "\"%s\" read, but unable to read \"%s\"" % (bLocal, bConf)) + elif os.path.exists(bLocal) and not os.access(bLocal, os.R_OK): + logSys.warning( + "\"%s\" read, but unable to read \"%s\"" % (bConf, bLocal)) SafeConfigParserWithIncludes.read(self, [bConf, bLocal]) return True else: From 9c2e0cbbc86569108fd729c9710e42b96b1c03fa Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 31 Jan 2013 18:36:23 +0000 Subject: [PATCH 038/114] Fix up for warning/error for inaccessible config files --- client/configreader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/configreader.py b/client/configreader.py index e435b254..4e783ebb 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -60,8 +60,13 @@ class ConfigReader(SafeConfigParserWithIncludes): bLocal = basename + ".local" if os.path.exists(bConf) or os.path.exists(bLocal): if not os.access(bConf, os.R_OK) and not os.access(bLocal, os.R_OK): - logSys.warning( - "Unable to read either \"%s\" or \"%s\"" % (bConf, bLocal)) + if os.path.exists(bConf) and not os.path.exists(bLocal): + logSys.error("Unable to read \"%s\" " % bConf) + elif os.path.exists(bLocal) and not os.path.exists(bConf): + logSys.error("Unable to read \"%s\" " % bLocal) + else: + logSys.error( + "Unable to read \"%s\" and \"%s\"" % (bConf, bLocal)) return False elif os.path.exists(bConf) and not os.access(bConf, os.R_OK): logSys.warning( From 5f2d3832f780ef229c06b6cb64e443393e32df1d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 31 Jan 2013 14:39:59 -0500 Subject: [PATCH 039/114] NF: roundcube-auth filter (to close Debian #699442, needing debian/jail.conf section) --- config/filter.d/roundcube-auth.conf | 22 ++++++++++++++++++++++ config/jail.conf | 9 +++++++++ testcases/files/logs/roundcube-auth | 1 + 3 files changed, 32 insertions(+) create mode 100644 config/filter.d/roundcube-auth.conf create mode 100644 testcases/files/logs/roundcube-auth diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf new file mode 100644 index 00000000..41766e31 --- /dev/null +++ b/config/filter.d/roundcube-auth.conf @@ -0,0 +1,22 @@ +# Fail2Ban configuration file for roundcube web server +# +# Author: Teodor Micu & Yaroslav Halchenko +# +# + +[Definition] + +# Option: failregex +# Notes.: regex to match the password failure messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = FAILED login for .*. from \s*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index 3f2425b4..1817ebd5 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -192,6 +192,15 @@ action = shorewall sendmail[name=Postfix, dest=you@example.com] logpath = /var/log/apache2/error_log +# Monitor roundcube server + +[roundcube-iptables] + +enabled = false +filter = roundcube-auth +action = iptables[name=RoundCube, port="http,https"] +logpath = /var/log/roundcube/userlogins + # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year # of usage in production environments. diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth new file mode 100644 index 00000000..d16f7266 --- /dev/null +++ b/testcases/files/logs/roundcube-auth @@ -0,0 +1 @@ +[22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 From f8983872ad4297ddb3017f4818edd08892dd2129 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 1 Feb 2013 16:07:00 -0500 Subject: [PATCH 040/114] BF: return str(host) to avoid spurious characters in the logs (Close gh-113) thanks to opoplawski@github --- server/failregex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/failregex.py b/server/failregex.py index 8ce9597a..b194d472 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -130,4 +130,4 @@ class FailRegex(Regex): s = self._matchCache.string r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) - return host + return str(host) From 52f952e6454d12fbe31dbd53bf2220c8b7658610 Mon Sep 17 00:00:00 2001 From: ArndRa Date: Mon, 11 Feb 2013 17:14:29 +0100 Subject: [PATCH 041/114] Update config/jail.conf Update to use the new sogo-auth filter --- config/jail.conf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index 1817ebd5..215e5b9a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -201,6 +201,20 @@ filter = roundcube-auth action = iptables[name=RoundCube, port="http,https"] logpath = /var/log/roundcube/userlogins + +# Monitor SOGo groupware server + +[sogo-iptables] + +enabled = false +filter = sogo-auth +port = http, https +# without proxy this would be: +# port = 20000 + +action = iptables[name=SOGo, port="http,https"] +logpath = /var/log/sogo/sogo.log + # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year # of usage in production environments. From 35bf84abadfc633e33a36bdcb32586f652fdadf3 Mon Sep 17 00:00:00 2001 From: ArndRa Date: Mon, 11 Feb 2013 08:19:48 -0800 Subject: [PATCH 042/114] Create sogo-auth.conf Regexp works with SOGo 2.0.5 or newer, following new feature implemented here: http://www.sogo.nu/bugs/view.php?id=2229 --- config/filter.d/sogo-auth.conf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 config/filter.d/sogo-auth.conf diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf new file mode 100644 index 00000000..e4d29c23 --- /dev/null +++ b/config/filter.d/sogo-auth.conf @@ -0,0 +1,19 @@ +# /etc/fail2ban/filter.d/sogo.conf +# +# Fail2Ban configuration file +# By Arnd Brandes +# SOGo +# + +[Definition] +# Option: failregex +# Filter Ban in /var/log/sogo/sogo.log +# Note: the error log may contain multiple hosts, whereas the first one +# is the client and all others are poxys. We match the first one, only + +failregex = Login from '.*' for user '.*' might not have worked + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# From 6004fe7a9449caf2aeb135cb3b7e76efa52b2ad7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Feb 2013 16:17:52 -0500 Subject: [PATCH 043/114] just trailing spaces in setup.py --- setup.py | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index 53b03f0f..37086227 100755 --- a/setup.py +++ b/setup.py @@ -36,33 +36,33 @@ to reject the IP address or executes user defined commands.''' setup( - name = "fail2ban", - version = version, - description = "Ban IPs that make too many password failure", - long_description = longdesc, - author = "Cyril Jaquier", - author_email = "cyril.jaquier@fail2ban.org", - url = "http://www.fail2ban.org", - license = "GPL", - platforms = "Posix", + name = "fail2ban", + version = version, + description = "Ban IPs that make too many password failure", + long_description = longdesc, + author = "Cyril Jaquier", + author_email = "cyril.jaquier@fail2ban.org", + url = "http://www.fail2ban.org", + license = "GPL", + platforms = "Posix", scripts = [ - 'fail2ban-client', - 'fail2ban-server', + 'fail2ban-client', + 'fail2ban-server', 'fail2ban-regex' - ], + ], packages = [ - 'common', - 'client', + 'common', + 'client', 'server' - ], + ], data_files = [ - ('/etc/fail2ban', + ('/etc/fail2ban', glob("config/*.conf") - ), - ('/etc/fail2ban/filter.d', + ), + ('/etc/fail2ban/filter.d', glob("config/filter.d/*.conf") - ), - ('/etc/fail2ban/action.d', + ), + ('/etc/fail2ban/action.d', glob("config/action.d/*.conf") ), ('/var/run/fail2ban', @@ -78,20 +78,20 @@ elements = { "/etc/": [ "fail2ban.conf" - ], + ], "/usr/bin/": [ "fail2ban.py" - ], + ], "/usr/lib/fail2ban/firewall/": [ - "iptables.py", - "ipfwadm.py", + "iptables.py", + "ipfwadm.py", "ipfw.py" ], "/usr/lib/fail2ban/": [ - "version.py", + "version.py", "protocol.py" ] } From 6cd358ee95dbeef5a07af25ee028df5cfe543731 Mon Sep 17 00:00:00 2001 From: ArndRa Date: Tue, 12 Feb 2013 10:45:37 +0100 Subject: [PATCH 044/114] Update config/filter.d/sogo-auth.conf Comment line in the top altered to fit file name. My local file was named differently... --- config/filter.d/sogo-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf index e4d29c23..6a1abd3f 100644 --- a/config/filter.d/sogo-auth.conf +++ b/config/filter.d/sogo-auth.conf @@ -1,4 +1,4 @@ -# /etc/fail2ban/filter.d/sogo.conf +# /etc/fail2ban/filter.d/sogo-auth.conf # # Fail2Ban configuration file # By Arnd Brandes From 8cf006827e6f7def17aef82f7a792f66747be8ca Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 12 Feb 2013 08:48:05 -0500 Subject: [PATCH 045/114] BF: remove path from grep call in sendmail-whois-lines.conf Closes: gh-118 --- config/action.d/sendmail-whois-lines.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index d1e6e40f..b84440d0 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -57,7 +57,7 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned Here are more information about :\n `/usr/bin/whois `\n\n Lines containing IP: in \n - `/bin/grep '\<\>' `\n\n + `grep '\<\>' `\n\n Regards,\n Fail2Ban" | /usr/sbin/sendmail -f From 47b1ee39d8e5e918afaea40f361b3932a2b8616e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 17 Feb 2013 12:37:34 +1100 Subject: [PATCH 046/114] add blocking type --- config/action.d/route.conf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/config/action.d/route.conf b/config/action.d/route.conf index 2d11c700..ec940b74 100644 --- a/config/action.d/route.conf +++ b/config/action.d/route.conf @@ -15,5 +15,11 @@ # - Blocking is per IP and NOT per service, but ideal as action against ssh password bruteforcing hosts [Definition] -actionban = ip route add blackhole -actionunban = ip route del blackhole +actionban = ip route add +actionunban = ip route del + +# Type of blocking +# +# Type can be blackhole, unreachable and prohibit. Unreachable and prohibit correspond to the ICMP reject messages. + +type = blackhole From 9ba27353b62bb20ae8aaeed45a274483ad67bd83 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 17 Feb 2013 17:03:23 -0500 Subject: [PATCH 047/114] NF: allow customization configuration under corresponding .d directories (Closes gh-114) Additional changes: ENH: make basedir for Config's a kwarg to the constructor ENH: improved analysis/reporting on presence/accessibility of config files. Got carried away and forgot about existing work done by Steven Hiscocks in the gh-115 -- will merge it in the next commit for the fair ack of his work Now for any X.conf configuration file we have following ways to provide customizations X.local -- read after .conf (kept for backward compatibility) X.d/ -- directory to contain additional .conf files, sorted alphabetically, e.g X.d/01_enable.conf - to enable the jail X.d/02_custom_port.conf - to change the port X could be a 'jail' or 'fail2ban' or any other configuration file in fail2ban. Mention that all files still must contain the corresponding sections (most of the time duplicating it across all of them). --- client/actionreader.py | 4 +- client/configreader.py | 70 ++++++++++++++++-------- client/fail2banreader.py | 4 +- client/filterreader.py | 4 +- client/jailreader.py | 11 ++-- client/jailsreader.py | 10 ++-- fail2ban-testcases | 4 +- testcases/clientreadertestcase.py | 89 ++++++++++++++++++++++++++++--- 8 files changed, 150 insertions(+), 46 deletions(-) diff --git a/client/actionreader.py b/client/actionreader.py index 581a1b3c..9ad1ef28 100644 --- a/client/actionreader.py +++ b/client/actionreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class ActionReader(ConfigReader): - def __init__(self, action, name): - ConfigReader.__init__(self) + def __init__(self, action, name, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__file = action[0] self.__cInfo = action[1] self.__name = name diff --git a/client/configreader.py b/client/configreader.py index 063484e8..79c5b47b 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os +import glob, logging, os from configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError @@ -35,36 +35,64 @@ from ConfigParser import NoOptionError, NoSectionError logSys = logging.getLogger("fail2ban.client.config") class ConfigReader(SafeConfigParserWithIncludes): + + DEFAULT_BASEDIR = '/etc/fail2ban' - BASE_DIRECTORY = "/etc/fail2ban/" - - def __init__(self): + def __init__(self, basedir=None): SafeConfigParserWithIncludes.__init__(self) + self.setBaseDir(basedir) self.__opts = None - #@staticmethod - def setBaseDir(folderName): - path = folderName.rstrip('/') - ConfigReader.BASE_DIRECTORY = path + '/' - setBaseDir = staticmethod(setBaseDir) - - #@staticmethod - def getBaseDir(): - return ConfigReader.BASE_DIRECTORY - getBaseDir = staticmethod(getBaseDir) + def setBaseDir(self, basedir): + if basedir is None: + basedir = ConfigReader.DEFAULT_BASEDIR # stock system location + if not (os.path.exists(basedir) and os.access(basedir, os.R_OK | os.X_OK)): + raise ValueError("Base configuration directory %s either does not exist " + "or is not accessible" % basedir) + self._basedir = basedir.rstrip('/') + + def getBaseDir(self): + return self._basedir def read(self, filename): - basename = ConfigReader.BASE_DIRECTORY + filename + basename = os.path.join(self._basedir, filename) logSys.debug("Reading " + basename) - bConf = basename + ".conf" - bLocal = basename + ".local" - if os.path.exists(bConf) or os.path.exists(bLocal): - SafeConfigParserWithIncludes.read(self, [bConf, bLocal]) + config_files = [ basename + ".conf", + basename + ".local" ] + + # choose only existing ones + config_files = filter(os.path.exists, config_files) + + # possible further customizations under a .conf.d directory + config_dir = basename + '.d' + if os.path.exists(config_dir): + if os.path.isdir(config_dir) and os.access(config_dir, os.X_OK | os.R_OK): + # files must carry .conf suffix as well + config_files += sorted(glob.glob('%s/*.conf' % config_dir)) + else: + logSys.warn("%s exists but not a directory or not accessible" + % config_dir) + + # check if files are accessible, warn if any is not accessible + # and remove it from the list + config_files_accessible = [] + for f in config_files: + if os.access(f, os.R_OK): + config_files_accessible.append(f) + else: + logSys.warn("%s exists but not accessible - skipping" % f) + + if len(config_files_accessible): + # at least one config exists and accessible + SafeConfigParserWithIncludes.read(self, config_files_accessible) return True else: - logSys.error(bConf + " and " + bLocal + " do not exist") + logSys.error("Found no accessible config files for %r " % filename + + (["", + "among existing ones: " + ', '.join(config_files)][bool(len(config_files))])) + return False - + ## # Read the options. # diff --git a/client/fail2banreader.py b/client/fail2banreader.py index ee097bd6..7115ec79 100644 --- a/client/fail2banreader.py +++ b/client/fail2banreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class Fail2banReader(ConfigReader): - def __init__(self): - ConfigReader.__init__(self) + def __init__(self, **kwargs): + ConfigReader.__init__(self, **kwargs) def read(self): ConfigReader.read(self, "fail2ban") diff --git a/client/filterreader.py b/client/filterreader.py index b7a72f9c..7dba3579 100644 --- a/client/filterreader.py +++ b/client/filterreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class FilterReader(ConfigReader): - def __init__(self, fileName, name): - ConfigReader.__init__(self) + def __init__(self, fileName, name, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__file = fileName self.__name = name diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..ec73ce46 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -40,8 +40,8 @@ class JailReader(ConfigReader): actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") - def __init__(self, name): - ConfigReader.__init__(self) + def __init__(self, name, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None self.__actions = list() @@ -53,7 +53,7 @@ class JailReader(ConfigReader): return self.__name def read(self): - ConfigReader.read(self, "jail") + return ConfigReader.read(self, "jail") def isEnabled(self): return self.__opts["enabled"] @@ -75,7 +75,8 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name) + self.__filter = FilterReader(self.__opts["filter"], self.__name, + basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -87,7 +88,7 @@ class JailReader(ConfigReader): for act in self.__opts["action"].split('\n'): try: splitAct = JailReader.splitAction(act) - action = ActionReader(splitAct, self.__name) + action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) diff --git a/client/jailsreader.py b/client/jailsreader.py index bedc5a3c..e1b8efa3 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -36,12 +36,12 @@ logSys = logging.getLogger("fail2ban.client.config") class JailsReader(ConfigReader): - def __init__(self): - ConfigReader.__init__(self) + def __init__(self, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__jails = list() def read(self): - ConfigReader.read(self, "jail") + return ConfigReader.read(self, "jail") def getOptions(self, section = None): opts = [] @@ -49,7 +49,7 @@ class JailsReader(ConfigReader): if section: # Get the options of a specific jail. - jail = JailReader(section) + jail = JailReader(section, basedir=self.getBaseDir()) jail.read() ret = jail.getOptions() if ret: @@ -62,7 +62,7 @@ class JailsReader(ConfigReader): else: # Get the options of all jails. for sec in self.sections(): - jail = JailReader(sec) + jail = JailReader(sec, basedir=self.getBaseDir()) jail.read() ret = jail.getOptions() if ret: diff --git a/fail2ban-testcases b/fail2ban-testcases index 99fefd57..ff94cfad 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -130,8 +130,10 @@ tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) -# ClientReader +# ClientReaders +tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) +tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) # Filter if not opts.no_network: diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 83121345..55fb010d 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -27,20 +27,93 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest +import os, shutil, tempfile, unittest +from client.configreader import ConfigReader from client.jailreader import JailReader +from client.jailsreader import JailsReader -class JailReaderTest(unittest.TestCase): +class ConfigReaderTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + self.d = tempfile.mkdtemp(prefix="f2b-temp") + self.c = ConfigReader(basedir=self.d) + def tearDown(self): """Call after every test case.""" + shutil.rmtree(self.d) + + def _write(self, fname, value): + # verify if we don't need to create .d directory + if os.path.sep in fname: + d = os.path.dirname(fname) + d_ = os.path.join(self.d, d) + if not os.path.exists(d_): + os.makedirs(d_) + open("%s/%s" % (self.d, fname), "w").write(""" +[section] +option = %s +""" % value) + + def _remove(self, fname): + os.unlink("%s/%s" % (self.d, fname)) + self.assertTrue(self.c.read('c')) # we still should have some + + + def _getoption(self): + self.assertTrue(self.c.read('c')) # we got some now + return self.c.getOptions('section', [("int", 'option')])['option'] + + def testOptionalDotDDir(self): + self.assertFalse(self.c.read('c')) # nothing is there yet + self._write("c.conf", "1") + self.assertEqual(self._getoption(), 1) + self._write("c.conf", "2") # overwrite + self.assertEqual(self._getoption(), 2) + self._write("c.local", "3") # add override in .local + self.assertEqual(self._getoption(), 3) + self._write("c.d/98.conf", "998") # add 1st override in .d/ + self.assertEqual(self._getoption(), 998) + self._write("c.d/90.conf", "990") # add previously sorted override in .d/ + self.assertEqual(self._getoption(), 998) # should stay the same + self._write("c.d/99.conf", "999") # now override in a way without sorting we possibly get a failure + self.assertEqual(self._getoption(), 999) + self._remove("c.d/99.conf") + self.assertEqual(self._getoption(), 998) + self._remove("c.d/98.conf") + self.assertEqual(self._getoption(), 990) + self._remove("c.d/90.conf") + self.assertEqual(self._getoption(), 3) + self._remove("c.conf") # we allow to stay without .conf + self.assertEqual(self._getoption(), 3) + self._write("c.conf", "1") + self._remove("c.local") + self.assertEqual(self._getoption(), 1) + + +class JailReaderTest(unittest.TestCase): + + def testStockSSHJail(self): + jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertFalse(jail.isEnabled()) + self.assertEqual(jail.getName(), 'ssh-iptables') + + +class JailsReaderTest(unittest.TestCase): + + def testProvidingBadBasedir(self): + if not os.path.exists('/XXX'): + self.assertRaises(ValueError, JailsReader, basedir='/XXX') + + def testReadStockJailConf(self): + jails = JailsReader(basedir='config') # we are running tests from root project dir atm + self.assertTrue(jails.read()) # opens fine + self.assertTrue(jails.getOptions()) # reads fine + comm_commands = jails.convert() + # by default None of the jails is enabled and we get no + # commands to communicate to the server + self.assertEqual(comm_commands, []) - def testSplitAction(self): - action = "mail-whois[name=SSH]" - expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitAction(action) - self.assertEquals(expected, result) - From ce3ab34dd8bc0181fbb68c7fc8b9ed18e35cf5f3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 17 Feb 2013 22:14:01 +0000 Subject: [PATCH 048/114] Added ability to specify PID file --- client/fail2banreader.py | 3 ++- config/fail2ban.conf | 7 +++++++ fail2ban-client | 18 ++++++++++++++---- fail2ban-server | 10 ++++++++-- server/server.py | 12 +++++------- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/client/fail2banreader.py b/client/fail2banreader.py index ee097bd6..6954b3bf 100644 --- a/client/fail2banreader.py +++ b/client/fail2banreader.py @@ -42,7 +42,8 @@ class Fail2banReader(ConfigReader): ConfigReader.read(self, "fail2ban") def getEarlyOptions(self): - opts = [["string", "socket", "/tmp/fail2ban.sock"]] + opts = [["string", "socket", "/tmp/fail2ban.sock"], + ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] return ConfigReader.getOptions(self, "Definition", opts) def getOptions(self): diff --git a/config/fail2ban.conf b/config/fail2ban.conf index f2f1b215..e759513b 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -40,3 +40,10 @@ logtarget = /var/log/fail2ban.log # socket = /var/run/fail2ban/fail2ban.sock +# Option: pidfile +# Notes.: Set the PID file. This is used to store the process ID of the +# fail2ban server. +# Values: FILE Default: /var/run/fail2ban/fail2ban.sock +# +pidfile = /var/run/fail2ban/fail2ban.pid + diff --git a/fail2ban-client b/fail2ban-client index 13d018e6..595144ef 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -62,6 +62,7 @@ class Fail2banClient: self.__conf["verbose"] = 1 self.__conf["interactive"] = False self.__conf["socket"] = None + self.__conf["pidfile"] = None def dispVersion(self): print "Fail2Ban v" + version @@ -84,6 +85,7 @@ class Fail2banClient: print "Options:" print " -c configuration directory" print " -s socket path" + print " -p pidfile path" print " -d dump configuration. For debugging" print " -i interactive mode" print " -v increase verbosity" @@ -119,6 +121,8 @@ class Fail2banClient: self.__conf["conf"] = opt[1] elif opt[0] == "-s": self.__conf["socket"] = opt[1] + elif opt[0] == "-p": + self.__conf["pidfile"] = opt[1] elif opt[0] == "-d": self.__conf["dump"] = True elif opt[0] == "-v": @@ -183,6 +187,7 @@ class Fail2banClient: return False # Start the server self.__startServerAsync(self.__conf["socket"], + self.__conf["pidfile"], self.__conf["force"]) try: # Wait for the server to start @@ -231,7 +236,7 @@ class Fail2banClient: # # Start the Fail2ban server in daemon mode. - def __startServerAsync(self, socket, force = False): + def __startServerAsync(self, socket, pidfile, force = False): # Forks the current process. pid = os.fork() if pid == 0: @@ -242,6 +247,9 @@ class Fail2banClient: # Set the socket path. args.append("-s") args.append(socket) + # Set the pidfile + args.append("-p") + args.append(pidfile) # Force the execution if needed. if force: args.append("-x") @@ -297,7 +305,7 @@ class Fail2banClient: # Reads the command line options. try: - cmdOpts = 'hc:s:xdviqV' + cmdOpts = 'hc:s:p:xdviqV' cmdLongOpts = ['help', 'version'] optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: @@ -328,9 +336,11 @@ class Fail2banClient: # Set socket path self.__configurator.readEarly() - socket = self.__configurator.getEarlyOptions() + conf = self.__configurator.getEarlyOptions() if self.__conf["socket"] == None: - self.__conf["socket"] = socket["socket"] + self.__conf["socket"] = conf["socket"] + if self.__conf["pidfile"] == None: + self.__conf["pidfile"] = conf["pidfile"] logSys.info("Using socket file " + self.__conf["socket"]) if self.__conf["dump"]: diff --git a/fail2ban-server b/fail2ban-server index 0f3410c9..81db58bd 100755 --- a/fail2ban-server +++ b/fail2ban-server @@ -54,6 +54,7 @@ class Fail2banServer: self.__conf["background"] = True self.__conf["force"] = False self.__conf["socket"] = "/var/run/fail2ban/fail2ban.sock" + self.__conf["pidfile"] = "/var/run/fail2ban/fail2ban.pid" def dispVersion(self): print "Fail2Ban v" + version @@ -81,6 +82,7 @@ class Fail2banServer: print " -b start in background" print " -f start in foreground" print " -s socket path" + print " -p pidfile path" print " -x force execution of the server (remove socket file)" print " -h, --help display this help message" print " -V, --version print the version" @@ -97,6 +99,8 @@ class Fail2banServer: self.__conf["background"] = False if opt[0] == "-s": self.__conf["socket"] = opt[1] + if opt[0] == "-p": + self.__conf["pidfile"] = opt[1] if opt[0] == "-x": self.__conf["force"] = True if opt[0] in ["-h", "--help"]: @@ -112,7 +116,7 @@ class Fail2banServer: # Reads the command line options. try: - cmdOpts = 'bfs:xhV' + cmdOpts = 'bfs:p:xhV' cmdLongOpts = ['help', 'version'] optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: @@ -123,7 +127,9 @@ class Fail2banServer: try: self.__server = Server(self.__conf["background"]) - self.__server.start(self.__conf["socket"], self.__conf["force"]) + self.__server.start(self.__conf["socket"], + self.__conf["pidfile"], + self.__conf["force"]) return True except Exception, e: logSys.exception(e) diff --git a/server/server.py b/server/server.py index d9532be2..3889c491 100644 --- a/server/server.py +++ b/server/server.py @@ -40,8 +40,6 @@ logSys = logging.getLogger("fail2ban.server") class Server: - PID_FILE = "/var/run/fail2ban/fail2ban.pid" - def __init__(self, daemon = False): self.__loggingLock = Lock() self.__lock = RLock() @@ -59,7 +57,7 @@ class Server: logSys.debug("Caught signal %d. Exiting" % signum) self.quit() - def start(self, sock, force = False): + def start(self, sock, pidfile, force = False): logSys.info("Starting Fail2ban v" + version.version) # Install signal handlers @@ -79,8 +77,8 @@ class Server: # Creates a PID file. try: - logSys.debug("Creating PID file %s" % Server.PID_FILE) - pidFile = open(Server.PID_FILE, 'w') + logSys.debug("Creating PID file %s" % pidfile) + pidFile = open(pidfile, 'w') pidFile.write("%s\n" % os.getpid()) pidFile.close() except IOError, e: @@ -94,8 +92,8 @@ class Server: logSys.error("Could not start server: %s", e) # Removes the PID file. try: - logSys.debug("Remove PID file %s" % Server.PID_FILE) - os.remove(Server.PID_FILE) + logSys.debug("Remove PID file %s" % pidfile) + os.remove(pidfile) except OSError, e: logSys.error("Unable to remove PID file: %s" % e) logSys.info("Exiting Fail2ban") From 2312b1d950826bebcedc0fef937afa9fb76f0602 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 17 Feb 2013 17:19:09 -0500 Subject: [PATCH 049/114] ENH: made log messages while parsing files more informative + test for inaccessible file (Closes: gh-24) --- client/configreader.py | 4 ++-- testcases/clientreadertestcase.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/client/configreader.py b/client/configreader.py index 79c5b47b..e1a2b69f 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -56,7 +56,7 @@ class ConfigReader(SafeConfigParserWithIncludes): def read(self, filename): basename = os.path.join(self._basedir, filename) - logSys.debug("Reading " + basename) + logSys.debug("Reading configs for %s under %s " % (basename, self._basedir)) config_files = [ basename + ".conf", basename + ".local" ] @@ -88,7 +88,7 @@ class ConfigReader(SafeConfigParserWithIncludes): return True else: logSys.error("Found no accessible config files for %r " % filename - + (["", + + (["under %s" % self.getBaseDir(), "among existing ones: " + ', '.join(config_files)][bool(len(config_files))])) return False diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 55fb010d..389220ee 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -39,7 +39,6 @@ class ConfigReaderTest(unittest.TestCase): self.d = tempfile.mkdtemp(prefix="f2b-temp") self.c = ConfigReader(basedir=self.d) - def tearDown(self): """Call after every test case.""" shutil.rmtree(self.d) @@ -61,10 +60,19 @@ option = %s self.assertTrue(self.c.read('c')) # we still should have some - def _getoption(self): - self.assertTrue(self.c.read('c')) # we got some now + def _getoption(self, f='c'): + self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + + def testInaccessibleFile(self): + f = os.path.join(self.d, "d.conf") # inaccessible file + self._write('d.conf', 0) + self.assertEqual(self._getoption('d'), 0) + os.chmod(f, 0) + self.assertFalse(self.c.read('d')) # should not be readable BUT present + + def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet self._write("c.conf", "1") From ec3080cba599832506723f1492de1150d19d2ca5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 17 Feb 2013 17:21:03 -0500 Subject: [PATCH 050/114] Reincarnated removed (by mistake) test for SplitAction --- testcases/clientreadertestcase.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 389220ee..d581a81f 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -109,6 +109,11 @@ class JailReaderTest(unittest.TestCase): self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'ssh-iptables') + def testSplitAction(self): + action = "mail-whois[name=SSH]" + expected = ['mail-whois', {'name': 'SSH'}] + result = JailReader.splitAction(action) + self.assertEquals(expected, result) class JailsReaderTest(unittest.TestCase): From 40c5a2d996ac16f0690c2a8b8f0b95979277d4ee Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 18 Feb 2013 23:08:44 -0500 Subject: [PATCH 051/114] ENH: adding more of diagnostic messages into -client while starting the daemon --- fail2ban-client | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/fail2ban-client b/fail2ban-client index 595144ef..7ee4a47c 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -134,11 +134,11 @@ class Fail2banClient: elif opt[0] == "-i": self.__conf["interactive"] = True elif opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) + self.dispUsage() + sys.exit(0) + elif opt[0] in ["-V", "--version"]: + self.dispVersion() + sys.exit(0) def __ping(self): return self.__processCmd([["ping"]], False) @@ -185,6 +185,18 @@ class Fail2banClient: # Do not continue if configuration is not 100% valid if not ret: return False + # verify that directory for the socket file exists + socket_dir = os.path.dirname(self.__conf["socket"]) + if not os.path.exists(socket_dir): + logSys.error( + "There is no directory %s to contain the socket file %s." + % (socket_dir, self.__conf["socket"])) + return False + if not os.access(socket_dir, os.W_OK | os.X_OK): + logSys.error( + "Directory %s exists but not accessible for writing" + % (socket_dir,)) + return False # Start the server self.__startServerAsync(self.__conf["socket"], self.__conf["pidfile"], @@ -196,10 +208,10 @@ class Fail2banClient: self.__processCmd(self.__stream, False) return True except ServerExecutionException: - logSys.error("Could not start server. Maybe an old " + - "socket file is still present. Try to " + - "remove " + self.__conf["socket"] + ". If " + - "you used fail2ban-client to start the " + + logSys.error("Could not start server. Maybe an old " + "socket file is still present. Try to " + "remove " + self.__conf["socket"] + ". If " + "you used fail2ban-client to start the " "server, adding the -x option will do it") return False elif len(cmd) == 1 and cmd[0] == "reload": @@ -256,16 +268,17 @@ class Fail2banClient: try: # Use the current directory. exe = os.path.abspath(os.path.join(sys.path[0], self.SERVER)) + logSys.debug("Starting %r with args %r" % (exe, args)) os.execv(exe, args) except OSError: try: # Use the PATH env. + logSys.warning("Initial start attempt failed. Starting %r with the same args" % (self.SERVER,)) os.execvp(self.SERVER, args) except OSError: - print "Could not find %s" % self.SERVER + logSys.error("Could not start %s" % self.SERVER) os.exit(-1) - - + def __waitOnServer(self): # Wait for the server to start cnt = 0 From 088e40c48192caed3a0c415e09211cd08c7c85d0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 20 Feb 2013 23:14:42 +0000 Subject: [PATCH 052/114] Rewrite and enable server testcase for Transmitter --- fail2ban-testcases | 2 +- testcases/servertestcase.py | 449 ++++++++++++++++++++++++++++++------ 2 files changed, 381 insertions(+), 70 deletions(-) diff --git a/fail2ban-testcases b/fail2ban-testcases index 99fefd57..d2bbfed1 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -124,7 +124,7 @@ else: # Server #tests.addTest(unittest.makeSuite(servertestcase.StartStop)) -#tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) +tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) # FailManager tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 54eac444..47d4ef0b 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest, socket, time +import unittest, socket, time, tempfile, os from server.server import Server class StartStop(unittest.TestCase): @@ -55,76 +55,387 @@ class Transmitter(unittest.TestCase): def setUp(self): """Call before every test case.""" self.__server = Server() + self.__transm = self.__server._Server__transm + self.__server.setLogTarget("/dev/null") self.__server.setLogLevel(0) - self.__server.start(False) + sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter') + os.close(sock_fd) + pidfile_fd, pidfile_name = tempfile.mkstemp( + 'fail2ban.pid', 'transmitter') + os.close(pidfile_fd) + self.__server.start(sock_name, pidfile_name, force=False) + self.jailName = "TestJail1" + self.__server.addJail(self.jailName, "auto") def tearDown(self): """Call after every test case.""" self.__server.quit() - - def testSetActionOK(self): - name = "TestCase" - cmdList = [["add", name], - ["set", name, "actionstart", "Action Start"], - ["set", name, "actionstop", "Action Stop"], - ["set", name, "actioncheck", "Action Check"], - ["set", name, "actionban", "Action Ban"], - ["set", name, "actionunban", "Action Unban"], - ["quit"]] - - outList = [(0, name), - (0, 'Action Start'), - (0, 'Action Stop'), - (0, 'Action Check'), - (0, 'Action Ban'), - (0, 'Action Unban'), - (0, None)] - - cnt = 0 - for cmd in cmdList: - self.assertEqual(self.__server.transm.proceed(cmd), outList[cnt]) - cnt += 1 - - def testSetActionNOK(self): - name = "TestCase" - cmdList = [["addd", name], - ["set", name, "test"], - ["prout prout", "Stop"], - ["fail2ban", "sucks"], - ["set"], - ["_/&%", "@*+%&"], - [" quit"]] - - outList = [1, - 1, - 1, - 1, - 1, - 1, - 1] - - cnt = 0 - for cmd in cmdList: - msg = self.__server.transm.proceed(cmd) - self.assertEqual(msg[0], outList[cnt]) - cnt += 1 - - def testJail(self): - name = "TestCase" - cmdList = [["add", name], - ["set", name, "logpath", "testcases/files/testcase01.log"], - ["set", name, "timeregex", "\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}"], - ["set", name, "timepattern", "%b %d %H:%M:%S"], - ["set", name, "failregex", "Authentication failure"], - ["start", name], - ["stop", name], - ["quit"]] - - for cmd in cmdList: - self.__server.transm.proceed(cmd) - if cmd == ["start", name]: - time.sleep(2) - jail = self.__server.jails[name] - self.assertEqual(jail.getFilter().failManager.size(), 0) - self.assertEqual(jail.getAction().banManager.size(), 2) - + + def setGetTest(self, cmd, inValue, outValue=None, jail=None): + setCmd = ["set", cmd, inValue] + getCmd = ["get", cmd] + if jail is not None: + setCmd.insert(1, jail) + getCmd.insert(1, jail) + if outValue is None: + outValue = inValue + + self.assertEqual(self.__transm.proceed(setCmd), (0, outValue)) + self.assertEqual(self.__transm.proceed(getCmd), (0, outValue)) + + def setGetTestNOK(self, cmd, inValue, jail=None): + setCmd = ["set", cmd, inValue] + getCmd = ["get", cmd] + if jail is not None: + setCmd.insert(1, jail) + getCmd.insert(1, jail) + + # Get initial value before trying invalid value + initValue = self.__transm.proceed(getCmd)[1] + self.assertEqual(self.__transm.proceed(setCmd)[0], 1) + # Check after failed set that value is same as previous + self.assertEqual(self.__transm.proceed(getCmd), (0, initValue)) + + def jailAddDelTest(self, cmd, values, jail): + cmdAdd = "add" + cmd + cmdDel = "del" + cmd + + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), (0, [])) + for n, value in enumerate(values): + self.assertEqual( + self.__transm.proceed(["set", jail, cmdAdd, value]), + (0, values[:n+1])) + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), + (0, values[:n+1])) + for n, value in enumerate(values): + self.assertEqual( + self.__transm.proceed(["set", jail, cmdDel, value]), + (0, values[n+1:])) + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), + (0, values[n+1:])) + + def jailAddDelRegexTest(self, cmd, inValues, outValues, jail): + cmdAdd = "add" + cmd + cmdDel = "del" + cmd + + if outValues is None: + outValues = inValues + + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), (0, [])) + for n, value in enumerate(inValues): + self.assertEqual( + self.__transm.proceed(["set", jail, cmdAdd, value]), + (0, outValues[:n+1])) + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), + (0, outValues[:n+1])) + for n, value in enumerate(inValues): + self.assertEqual( + self.__transm.proceed(["set", jail, cmdDel, 0]), # First item + (0, outValues[n+1:])) + self.assertEqual( + self.__transm.proceed(["get", jail, cmd]), + (0, outValues[n+1:])) + + def testPing(self): + self.assertEqual(self.__transm.proceed(["ping"]), (0, "pong")) + + def testSleep(self): + t0 = time.time() + self.assertEqual(self.__transm.proceed(["sleep", "1"]), (0, None)) + t1 = time.time() + # Approx 1 second delay + self.assertAlmostEqual(t1 - t0, 1, places=2) + + def testLogTarget(self): + logTargets = [] + for _ in xrange(3): + tmpFile = tempfile.mkstemp("fail2ban", "transmitter") + logTargets.append(tmpFile[1]) + os.close(tmpFile[0]) + for logTarget in logTargets: + self.setGetTest("logtarget", logTarget) + + # If path is invalid, do not change logtarget + value = "/this/path/should/not/exist" + self.assertEqual( + self.__transm.proceed(["set", "logtarget", value]), + (0, logTarget)) #NOTE: Shouldn't this return 1 + self.assertEqual( + self.__transm.proceed(["get", "logtarget"]), (0, logTargets[-1])) + + self.__transm.proceed(["set", "/dev/null"]) + for logTarget in logTargets: + os.remove(logTarget) + + def testLogLevel(self): + self.setGetTest("loglevel", "4", 4) + self.setGetTest("loglevel", "2", 2) + self.setGetTest("loglevel", "-1", -1) + self.setGetTestNOK("loglevel", "Bird") + + def testAddJail(self): + jail2 = "TestJail2" + jail3 = "TestJail3" + jail4 = "TestJail4" + self.assertEqual( + self.__transm.proceed(["add", jail2, "polling"]), (0, jail2)) + self.assertEqual(self.__transm.proceed(["add", jail3]), (0, jail3)) + self.assertEqual( + self.__transm.proceed(["add", jail4, "invalid backend"])[0], 1) + self.assertEqual( + self.__transm.proceed(["add", jail4, "auto"]), (0, jail4)) + # Duplicate Jail + self.assertEqual( + self.__transm.proceed(["add", self.jailName, "polling"])[0], 1) + # All name is reserved + self.assertEqual( + self.__transm.proceed(["add", "all", "polling"])[0], 1) + + def testJailIdle(self): + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "idle", "on"]), + (0, True)) + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "idle", "off"]), + (0, False)) + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "idle", "CAT"])[0], + 0) #NOTE: Should this return 1 + + def testJailFindTime(self): + self.setGetTest("findtime", "120", 120, jail=self.jailName) + self.setGetTest("findtime", "60", 60, jail=self.jailName) + self.setGetTest("findtime", "-60", -60, jail=self.jailName) + self.setGetTestNOK("findtime", "Dog", jail=self.jailName) + + def testJailBanTime(self): + self.setGetTest("bantime", "600", 600, jail=self.jailName) + self.setGetTest("bantime", "50", 50, jail=self.jailName) + self.setGetTest("bantime", "-50", -50, jail=self.jailName) + self.setGetTestNOK("bantime", "Cat", jail=self.jailName) + + def testJailUseDNS(self): + self.setGetTest("usedns", "yes", jail=self.jailName) + self.setGetTest("usedns", "warn", jail=self.jailName) + self.setGetTest("usedns", "no", jail=self.jailName) + + # Safe default should be "no" + value = "Fish" + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "usedns", value]), + (0, "no")) + + def testJailBanIP(self): + self.__server.startJail(self.jailName) # Jail must be started + + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]), + (0, "127.0.0.1")) + time.sleep(1) # Give chance to ban + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "banip", "Badger"]), + (0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done? + time.sleep(1) # Give chance to ban + # Unban IP + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "unbanip", "127.0.0.1"]), + (0, "127.0.0.1")) + # Unban IP which isn't banned + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "unbanip", "192.168.1.1"]), + (0, "None")) #NOTE: Should this return 1? + + def testJailMaxRetry(self): + self.setGetTest("maxretry", "5", 5, jail=self.jailName) + self.setGetTest("maxretry", "2", 2, jail=self.jailName) + self.setGetTest("maxretry", "-2", -2, jail=self.jailName) + self.setGetTestNOK("maxretry", "Duck", jail=self.jailName) + + def testJailLogPath(self): + self.jailAddDelTest( + "logpath", + [ + "testcases/files/testcase01.log", + "testcases/files/testcase02.log", + "testcases/files/testcase03.log", + ], + self.jailName + ) + # Try duplicates + value = "testcases/files/testcase04.log" + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "addlogpath", value]), + (0, [value])) + # Will silently ignore duplicate + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "addlogpath", value]), + (0, [value])) + self.assertEqual( + self.__transm.proceed(["get", self.jailName, "logpath"]), + (0, [value])) + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "dellogpath", value]), + (0, [])) + + # Invalid file + value = "this_file_shouldn't_exist" + result = self.__transm.proceed( + ["set", self.jailName, "addlogpath", value]) + self.assertTrue(isinstance(result[1], IOError)) + + def testJailIgnoreIP(self): + self.jailAddDelTest( + "ignoreip", + [ + "127.0.0.1", + "192.168.1.1", + "8.8.8.8", + ], + self.jailName + ) + + # Try duplicates + value = "127.0.0.1" + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "addignoreip", value]), + (0, [value])) + # Will allow duplicate + #NOTE: Should duplicates be allowed, or silent ignore like logpath? + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "addignoreip", value]), + (0, [value, value])) + self.assertEqual( + self.__transm.proceed(["get", self.jailName, "ignoreip"]), + (0, [value, value])) + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "delignoreip", value]), + (0, [value])) + + def testJailRegex(self): + self.jailAddDelRegexTest("failregex", + [ + "user john at ", + "Admin user login from ", + "failed attempt from again", + ], + [ + "user john at (?:::f{4,6}:)?(?P[\w\-.^_]+)", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", + "failed attempt from (?:::f{4,6}:)?(?P[\w\-.^_]+) again", + ], + self.jailName + ) + + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "addfailregex", "No host regex"]), + (0, [])) #NOTE: Shouldn't this return 1? + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "addfailregex", 654])[0], + 1) + + def testJailIgnoreRegex(self): + self.jailAddDelRegexTest("ignoreregex", + [ + "user john", + "Admin user login from ", + "Dont match me!", + ], + [ + "user john", + "Admin user login from (?:::f{4,6}:)?(?P[\w\-.^_]+)", + "Dont match me!", + ], + self.jailName + ) + + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "addignoreregex", 50])[0], + 1) + + def testStatus(self): + jails = [self.jailName] + self.assertEqual(self.__transm.proceed(["status"]), + (0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))])) + self.__server.addJail("TestJail2", "auto") + jails.append("TestJail2") + self.assertEqual(self.__transm.proceed(["status"]), + (0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))])) + + def testJailStatus(self): + self.assertEqual(self.__transm.proceed(["status", self.jailName]), + (0, + [ + ('filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('action', [ + ('Currently banned', 0), + ('Total banned', 0), + ('IP list', [])] + ) + ] + ) + ) + + def testAction(self): + action = "TestCaseAction" + cmdList = [ + "actionstart", + "actionstop", + "actioncheck", + "actionban", + "actionunban", + ] + cmdValueList = [ + "Action Start", + "Action Stop", + "Action Check", + "Action Ban", + "Action Unban", + ] + + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "addaction", action]), + (0, action)) + for cmd, value in zip(cmdList, cmdValueList): + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, cmd, action, value]), + (0, value)) + for cmd, value in zip(cmdList, cmdValueList): + self.assertEqual( + self.__transm.proceed(["get", self.jailName, cmd, action]), + (0, value)) + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "setcinfo", action, "KEY", "VALUE"]), + (0, "VALUE")) + #TODO: Implement below in server.transmitter? + """ + self.assertEqual( + self.__transm.proceed( + ["get", self.jailName, "cinfo", action, "KEY"]), + (0, "VALUE")) + """ + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "delcinfo", action, "KEY"]), + (0, None)) + #FIXME: This is broken in server.transmitter (value not defined) + """ + self.assertEqual( + self.__transm.proceed(["set", self.jailName, "delaction", action]), + (0, None)) + """ From b6a68f5138c12e5f1374d76ac76f8f43bc814bfc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 20 Feb 2013 23:24:46 +0000 Subject: [PATCH 053/114] Fix for missing value in transmitter delaction --- server/transmitter.py | 1 + testcases/servertestcase.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/transmitter.py b/server/transmitter.py index 23b609a1..3bd7fb14 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -183,6 +183,7 @@ class Transmitter: self.__server.addAction(name, value) return self.__server.getLastAction(name).getName() elif command[1] == "delaction": + value = command[2] self.__server.delAction(name, value) return None elif command[1] == "setcinfo": diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 47d4ef0b..0b33a96f 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -433,9 +433,10 @@ class Transmitter(unittest.TestCase): self.__transm.proceed( ["set", self.jailName, "delcinfo", action, "KEY"]), (0, None)) - #FIXME: This is broken in server.transmitter (value not defined) - """ self.assertEqual( self.__transm.proceed(["set", self.jailName, "delaction", action]), (0, None)) - """ + self.assertEqual( + self.__transm.proceed( + ["set", self.jailName, "delaction", "Doesn't exist"]), + (0, None)) #NOTE: Should this return 1? From b36835f6f061efded70c747539a1da2d66b9d270 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 20 Feb 2013 23:33:39 +0000 Subject: [PATCH 054/114] Added transmitter get cinfo option for action --- common/protocol.py | 1 + server/transmitter.py | 4 ++++ testcases/servertestcase.py | 7 ++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/common/protocol.py b/common/protocol.py index 2f8ffa6c..99a2fe09 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -90,6 +90,7 @@ protocol = [ ["get actioncheck ", "gets the check command for the action for "], ["get actionban ", "gets the ban command for the action for "], ["get actionunban ", "gets the unban command for the action for "], +["get cinfo ", "gets the value for for the action for "], ] ## diff --git a/server/transmitter.py b/server/transmitter.py index 3bd7fb14..a02b94a2 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -266,6 +266,10 @@ class Transmitter: elif command[1] == "actionunban": act = command[2] return self.__server.getActionUnban(name, act) + elif command[1] == "cinfo": + act = command[2] + key = command[3] + return self.__server.getCInfo(name, act, key) raise Exception("Invalid command (no get action or not yet implemented)") def status(self, command): diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 0b33a96f..00f56b81 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -422,13 +422,14 @@ class Transmitter(unittest.TestCase): self.__transm.proceed( ["set", self.jailName, "setcinfo", action, "KEY", "VALUE"]), (0, "VALUE")) - #TODO: Implement below in server.transmitter? - """ self.assertEqual( self.__transm.proceed( ["get", self.jailName, "cinfo", action, "KEY"]), (0, "VALUE")) - """ + self.assertEqual( + self.__transm.proceed( + ["get", self.jailName, "cinfo", action, "InvalidKey"])[0], + 1) self.assertEqual( self.__transm.proceed( ["set", self.jailName, "delcinfo", action, "KEY"]), From 012264dce169b8ae9a18a599371bcb9685ec3570 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 21 Feb 2013 20:58:27 -0500 Subject: [PATCH 055/114] BF: safeguard closing of log handlers + close in reverse order otherwise there might be "stuck" handler in the queue. and closing exceptions can occur -- even stock logging guards in recent versions --- server/server.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/server/server.py b/server/server.py index 3889c491..dea303db 100644 --- a/server/server.py +++ b/server/server.py @@ -367,11 +367,20 @@ class Server: logSys.error("Unable to log to " + target) logSys.info("Logging to previous target " + self.__logTarget) return False - # Removes previous handlers - for handler in logging.getLogger("fail2ban").handlers: - # Closes the handler. + # Removes previous handlers -- in reverse order since removeHandler + # alter the list in-place and that can confuses the iterable + for handler in logging.getLogger("fail2ban").handlers[::-1]: + # Remove the handler. logging.getLogger("fail2ban").removeHandler(handler) - handler.close() + # And try to close -- it might be closed already + try: + handler.flush() + handler.close() + except ValueError: + if sys.version_info >= (2,6): + raise + # is known to be thrown after logging was shutdown once + # with older Pythons -- seems to be safe to ignore there # tell the handler to use this format hdlr.setFormatter(formatter) logging.getLogger("fail2ban").addHandler(hdlr) From 154aa38e3f8174baee6593c3022c53f25b8aa364 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 21 Feb 2013 20:59:46 -0500 Subject: [PATCH 056/114] BF: do not shutdown logging until all jails stop -- so move into Server.quit() Together with previous commit it should resolve failures with the server tests on python < 2.6 --- server/server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/server.py b/server/server.py index dea303db..0e1b6c97 100644 --- a/server/server.py +++ b/server/server.py @@ -97,12 +97,6 @@ class Server: except OSError, e: logSys.error("Unable to remove PID file: %s" % e) logSys.info("Exiting Fail2ban") - # Shutdowns the logging. - try: - self.__loggingLock.acquire() - logging.shutdown() - finally: - self.__loggingLock.release() def quit(self): # Stop communication first because if jail's unban action @@ -112,8 +106,17 @@ class Server: # are exiting) # See https://github.com/fail2ban/fail2ban/issues/7 self.__asyncServer.stop() + # Now stop all the jails self.stopAllJail() + + # Only now shutdown the logging. + try: + self.__loggingLock.acquire() + logging.shutdown() + finally: + self.__loggingLock.release() + def addJail(self, name, backend): self.__jails.add(name, backend) From 59c35bc44a175a672e084bc30511dfa3436ff052 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 1 Mar 2013 19:57:56 -0500 Subject: [PATCH 057/114] Downgrade log rotation detection message to DEBUG level from INFO. Closes: gh-129 This message useful only when debugging problems so it is more reasonable to have it suppressed otherwise --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index fb79fcbe..5b2b85e0 100644 --- a/server/filter.py +++ b/server/filter.py @@ -550,7 +550,7 @@ class FileContainer: stats = os.fstat(self.__handler.fileno()) # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.info("Log rotation detected for %s" % self.__filename) + logSys.debug("Log rotation detected for %s" % self.__filename) self.__hash = myHash self.__ino = stats.st_ino self.__pos = 0 From d17b415371d14f00c9b24fc3672733b6666fe345 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 5 Mar 2013 00:02:39 +0100 Subject: [PATCH 058/114] invalid date recognition, irregular because of sorting template list (sometimes not reproducible by fail2ban-regex, cause will be not sorted) date in following log line (from nginx) will be wrong detected: 2012/10/11 02:37:17 [error] 18434#0: *947 user "test" was not found in "/www/...", client: 192.168.1.5, ... sometimes it is [correct] - 2012/10/11 (%Y/%m/%d) = 1349919861.71 sometimes it is [invalid] - 12/10/11 (%d/%m/%y) = 1349915838.06 and older as now - 1800 seconds (therefore will be not found) solution: regular expression fixed, cause date in log used always after non symbol (\W) character, so r"\d{2}/\d{2}/\d{2}" will be r"(? Date: Thu, 7 Mar 2013 13:03:49 -0500 Subject: [PATCH 059/114] refresh generated manpages (since 0.8.2 state) --- man/fail2ban-client.1 | 39 +++++++++++++++++++++++++-------------- man/fail2ban-regex.1 | 13 ++++++++----- man/fail2ban-server.1 | 13 ++++++++----- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 1bbddf09..431e690f 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.36. -.TH FAIL2BAN-CLIENT "1" "March 2008" "fail2ban-client v0.8.2" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.10. +.TH FAIL2BAN-CLIENT "1" "March 2013" "fail2ban-client v0.8.8" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fIOPTIONS\fR] \fI\fR .SH DESCRIPTION -Fail2Ban v0.8.2 reads log file that contains password failure report +Fail2Ban v0.8.8 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP @@ -16,6 +16,9 @@ configuration directory \fB\-s\fR socket path .TP +\fB\-p\fR +pidfile path +.TP \fB\-d\fR dump configuration. For debugging .TP @@ -110,7 +113,7 @@ adds to the monitoring list of .TP \fBset dellogpath \fR -removes to the monitoring +removes from the monitoring list of .TP \fBset addfailregex \fR @@ -140,6 +143,15 @@ back for sets the number of seconds