From aec709f4c19d4c873345e184c71c5181d15b126c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 22 Jan 2013 20:54:14 +0000 Subject: [PATCH 001/426] Initial changes and test for multi-line filtering --- client/jailreader.py | 3 +++ common/protocol.py | 2 ++ man/fail2ban-client.1 | 9 ++++++++ server/failregex.py | 2 +- server/filter.py | 30 +++++++++++++++++++++++++- server/server.py | 6 ++++++ server/transmitter.py | 6 ++++++ testcases/files/testcase-multiline.log | 12 +++++++++++ testcases/filtertestcase.py | 15 +++++++++++++ 9 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 testcases/files/testcase-multiline.log diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..ca9cc5a8 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -63,6 +63,7 @@ class JailReader(ConfigReader): ["string", "logpath", "/var/log/messages"], ["string", "backend", "auto"], ["int", "maxretry", 3], + ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], @@ -114,6 +115,8 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) + elif opt == "maxlines": + stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. diff --git a/common/protocol.py b/common/protocol.py index 2f8ffa6c..de7c84a8 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -66,6 +66,7 @@ protocol = [ ["set banip ", "manually Ban for "], ["set unbanip ", "manually Unban in "], ["set maxretry ", "sets the number of failures before banning the host for "], +["set maxlines ", "sets the number of to buffer for regex search for "], ["set addaction ", "adds a new action named for "], ["set delaction ", "removes the action from "], ["set setcinfo ", "sets for of the action for "], @@ -84,6 +85,7 @@ protocol = [ ["get bantime", "gets the time a host is banned for "], ["get usedns", "gets the usedns setting for "], ["get maxretry", "gets the number of failures allowed for "], +["get maxlines", "gets the number of lines to buffer for "], ["get addaction", "gets the last action which has been added for "], ["get actionstart ", "gets the start command for the action for "], ["get actionstop ", "gets the stop command for the action for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 1bbddf09..d60c9481 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -145,6 +145,11 @@ sets the number of failures before banning the host for .TP +\fBset maxlines \fR +sets the number of to +buffer for regex search for + +.TP \fBset addaction \fR adds a new action named for @@ -222,6 +227,10 @@ gets the time a host is banned for gets the number of failures allowed for .TP +\fBget maxlines\fR +gets the number lines to +buffer for +.TP \fBget addaction\fR gets the last action which has been added for diff --git a/server/failregex.py b/server/failregex.py index 8ce9597a..c595b4de 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -51,7 +51,7 @@ class Regex: if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: - self._regexObj = re.compile(regex) + self._regexObj = re.compile(regex, re.MULTILINE) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % diff --git a/server/filter.py b/server/filter.py index b37e37e6..3ab21b17 100644 --- a/server/filter.py +++ b/server/filter.py @@ -36,6 +36,7 @@ from mytime import MyTime from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time +from collections import deque # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -71,6 +72,10 @@ class Filter(JailThread): self.__findTime = 6000 ## The ignore IP list. self.__ignoreIpList = [] + ## Size of line buffer + self.__line_buffer_size = 1 + ## Line buffer + self.__line_buffer = deque() self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -204,6 +209,25 @@ class Filter(JailThread): def getMaxRetry(self): return self.failManager.getMaxRetry() + ## + # Set the maximum line buffer size. + # + # @param value the line buffer size + + def setMaxLines(self, value): + if value < 1: + value = 1 + self.__line_buffer_size = value + logSys.info("Set maxLines = %s" % value) + + ## + # Get the maximum line buffer size. + # + # @return the line buffer size + + def getMaxLines(self): + return self.__line_buffer_size + ## # Main loop. # @@ -305,7 +329,10 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine) + self.__line_buffer.append(logLine) + while len(self.__line_buffer) > self.__line_buffer_size: + self.__line_buffer.popleft() + return self.findFailure(timeLine, "".join(self.__line_buffer)) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -365,6 +392,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: + self.__line_buffer.clear() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/server/server.py b/server/server.py index d9532be2..32d72770 100644 --- a/server/server.py +++ b/server/server.py @@ -216,6 +216,12 @@ class Server: def getMaxRetry(self, name): return self.__jails.getFilter(name).getMaxRetry() + def setMaxLines(self, name, value): + self.__jails.getFilter(name).setMaxLines(value) + + def getMaxLines(self, name): + return self.__jails.getFilter(name).getMaxLines() + # Action def addAction(self, name, value): self.__jails.getAction(name).addAction(value) diff --git a/server/transmitter.py b/server/transmitter.py index 23b609a1..23fb3eba 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -167,6 +167,10 @@ class Transmitter: value = command[2] self.__server.setMaxRetry(name, int(value)) return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + value = command[2] + self.__server.setMaxLines(name, int(value)) + return self.__server.getMaxLines(name) # command elif command[1] == "bantime": value = command[2] @@ -245,6 +249,8 @@ class Transmitter: return self.__server.getFindTime(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + return self.__server.getMaxLines(name) # Action elif command[1] == "bantime": return self.__server.getBanTime(name) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log new file mode 100644 index 00000000..b91f2756 --- /dev/null +++ b/testcases/files/testcase-multiline.log @@ -0,0 +1,12 @@ +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index c10fa78d..31d89239 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -499,6 +499,7 @@ class GetFailures(unittest.TestCase): FILENAME_03 = "testcases/files/testcase03.log" FILENAME_04 = "testcases/files/testcase04.log" FILENAME_USEDNS = "testcases/files/testcase-usedns.log" + FILENAME_MULTILINE = "testcases/files/testcase-multiline.log" # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, @@ -604,6 +605,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLine(self): + output = ("212.41.96.185", 3, 1124013598.0) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("Invalid user .+\n.+ from $") + self.filter.addIgnoreRegex("user fuck") + + self.filter.setMaxLines(2) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From 5c7e3841e095b7d0581091c8425c92ad626f8e89 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 18:26:49 +0000 Subject: [PATCH 002/426] Simplify and change some filter line buffer Include change variable names to `fail2ban` style --- server/filter.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/server/filter.py b/server/filter.py index 3ab21b17..a66a3c06 100644 --- a/server/filter.py +++ b/server/filter.py @@ -36,7 +36,6 @@ from mytime import MyTime from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time -from collections import deque # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -73,9 +72,9 @@ class Filter(JailThread): ## The ignore IP list. self.__ignoreIpList = [] ## Size of line buffer - self.__line_buffer_size = 1 + self.__lineBufferSize = 1 ## Line buffer - self.__line_buffer = deque() + self.__lineBuffer = [] self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -215,10 +214,8 @@ class Filter(JailThread): # @param value the line buffer size def setMaxLines(self, value): - if value < 1: - value = 1 - self.__line_buffer_size = value - logSys.info("Set maxLines = %s" % value) + self.__lineBufferSize = max(1, value) + logSys.info("Set maxLines = %i" % self.__lineBufferSize) ## # Get the maximum line buffer size. @@ -226,7 +223,7 @@ class Filter(JailThread): # @return the line buffer size def getMaxLines(self): - return self.__line_buffer_size + return self.__lineBufferSize ## # Main loop. @@ -329,10 +326,9 @@ class Filter(JailThread): else: timeLine = l logLine = l - self.__line_buffer.append(logLine) - while len(self.__line_buffer) > self.__line_buffer_size: - self.__line_buffer.popleft() - return self.findFailure(timeLine, "".join(self.__line_buffer)) + self.__lineBuffer = ((self.__lineBuffer + + [logLine])[-self.__lineBufferSize:]) + return self.findFailure(timeLine, "".join(self.__lineBuffer)) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -392,7 +388,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: - self.__line_buffer.clear() + self.__lineBuffer = [] try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) From 055aeeb227e129fdf56bacb3e1ee006264cd2350 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 18:42:25 +0000 Subject: [PATCH 003/426] Filter for multi-line now stores last time match This is useful for log files which dont contain a date/time on every line --- server/filter.py | 5 ++++- testcases/files/testcase-multiline.log | 20 ++++++++++---------- testcases/filtertestcase.py | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/server/filter.py b/server/filter.py index a66a3c06..4d99ae33 100644 --- a/server/filter.py +++ b/server/filter.py @@ -75,6 +75,8 @@ class Filter(JailThread): self.__lineBufferSize = 1 ## Line buffer self.__lineBuffer = [] + ## Store last time stamp, applicable for multi-line + self.__lastTimeLine = "" self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -319,12 +321,13 @@ class Filter(JailThread): if timeMatch: # Lets split into time part and log part of the line timeLine = timeMatch.group() + self.__lastTimeLine = timeLine # Lets leave the beginning in as well, so if there is no # anchore at the beginning of the time regexp, we don't # at least allow injection. Should be harmless otherwise logLine = l[:timeMatch.start()] + l[timeMatch.end():] else: - timeLine = l + timeLine = self.__lastTimeLine or l logLine = l self.__lineBuffer = ((self.__lineBuffer + [logLine])[-self.__lineBufferSize:]) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index b91f2756..12132920 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,12 +1,12 @@ -Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user duck + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user duck... Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] Invalid user duck... Aug 14 11:59:58 [sshd] from 212.41.96.185 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 31d89239..4083d231 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -608,8 +608,8 @@ class GetFailures(unittest.TestCase): def testGetFailuresMultiLine(self): output = ("212.41.96.185", 3, 1124013598.0) self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("Invalid user .+\n.+ from $") - self.filter.addIgnoreRegex("user fuck") + self.filter.addFailRegex("Invalid user .+\n.* from $") + self.filter.addIgnoreRegex("user duck") self.filter.setMaxLines(2) From 00ab42549228cca2034dd0f47ff5ece34b885c5e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 19:10:27 +0000 Subject: [PATCH 004/426] Changed multi-line test to provided example --- testcases/files/testcase-multiline.log | 36 +++++++++++++++++--------- testcases/filtertestcase.py | 9 +++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 12132920..70bd99ad 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,12 +1,24 @@ -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10) +Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10) +Aug 14 11:59:58 yyyy rsyncd[23864]: building file list +Aug 14 11:59:58 yyyy rsyncd[28101]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[28101]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[28101]: building file list +Aug 14 11:59:58 yyyy rsyncd[28101]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes received 65115 bytes total size 29221543998 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115 +Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[1762]: building file list +Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 115 to 116 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error +Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[7788]: building file list +Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 4083d231..7cd4c5ec 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,12 +606,11 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = ("212.41.96.185", 3, 1124013598.0) + output = ("192.0.43.10", 1, 1124013598.0) self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("Invalid user .+\n.* from $") - self.filter.addIgnoreRegex("user duck") - - self.filter.setMaxLines(2) + self.filter.addFailRegex("rsyncd\[(?P\d+)\]: connect from .+ \(\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) From 5952819a580c9e9824a19cd7a123b8b696edfe9a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 19:32:55 +0000 Subject: [PATCH 005/426] Sanitise testcase log 04 --- testcases/files/testcase04.log | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testcases/files/testcase04.log b/testcases/files/testcase04.log index c0304d06..987abec6 100644 --- a/testcases/files/testcase04.log +++ b/testcases/files/testcase04.log @@ -1,15 +1,15 @@ Sep 21 22:03:07 [sshd] Invalid user toto from 212.41.96.185 -1124012400 [sshd] Invalid user fuck from 212.41.96.185 +1124012400 [sshd] Invalid user duck from 212.41.96.185 Sep 21 21:03:38 [sshd] Invalid user toto from 212.41.96.185 -1124012500 [sshd] Invalid user fuck from 212.41.96.185 +1124012500 [sshd] Invalid user duck from 212.41.96.185 Sep 21 21:03:46 [sshd] Invalid user toto from 212.41.96.185 -Aug 14 11:58:48 [sshd] Invalid user fuck from 212.41.96.185 +Aug 14 11:58:48 [sshd] Invalid user duck from 212.41.96.185 Aug 14 11:59:58 [sshd] Invalid user toto from 212.41.96.185 -Sep 21 21:04:03 [sshd] Invalid user fuck from 212.41.96.185 +Sep 21 21:04:03 [sshd] Invalid user duck from 212.41.96.185 - Last output repeated twice - 2005/08/14 11:57:00 [sshd] Invalid user toto from 212.41.96.186 -2005/08/14 11:58:00 [sshd] Invalid user fuck from 212.41.96.186 +2005/08/14 11:58:00 [sshd] Invalid user duck from 212.41.96.186 2005/08/14 11:59:00 [sshd] Invalid user toto from 212.41.96.186 -2005/08/14 12:00:00 [sshd] Invalid user fuck from 212.41.96.186 +2005/08/14 12:00:00 [sshd] Invalid user duck from 212.41.96.186 - Last output repeated twice - Sep 21 21:09:01 [sshd] Invalid user toto from 212.41.96.185 From 9b4806bfd31e286e8b3271e7765c53f1c3c6445c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 24 Jan 2013 18:16:17 +0000 Subject: [PATCH 006/426] Added regex applicable for multi-line This allows lines captured by regex to remain in the line buffer in Filter --- server/failregex.py | 23 +++++++++++++++++++++++ server/filter.py | 2 +- testcases/files/testcase-multiline.log | 6 +++++- testcases/filtertestcase.py | 8 +++++--- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index c595b4de..84b237c7 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -48,6 +48,11 @@ class Regex: # Perform shortcuts expansions. # Replace "" with default regular expression for host. regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]+)") + # Replace "" with regular expression for multiple lines. + regexSplit = regex.split("") + regex = regexSplit[0] + for n, regexLine in enumerate(regexSplit[1:]): + regex += "\n(?P(?:(.*\n)*?))" % n + regexLine if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: @@ -131,3 +136,21 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host + + ## + # Returns unmatched lines. + # + # This returns unmatched lines inlcuding captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + unmatchedLines = self._matchCache.string[:self._matchCache.start()] + n = 0 + while True: + try: + unmatchedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + unmatchedLines += self._matchCache.string[self._matchCache.end():] + return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 4d99ae33..0b0e0535 100644 --- a/server/filter.py +++ b/server/filter.py @@ -391,7 +391,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: - self.__lineBuffer = [] + self.__lineBuffer = failRegex.getUnmatchedLines() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 70bd99ad..c0151d34 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -19,6 +19,10 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test co Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: building file list +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 7cd4c5ec..262074e6 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,15 +606,17 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = ("192.0.43.10", 1, 1124013598.0) + output = [("192.0.43.10", 1, 1124013598.0), + ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("rsyncd\[(?P\d+)\]: connect from .+ \(\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error") + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output) + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) From 28f68a693f8f042497cd6a18de49bd9c4ff85a54 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 24 Jan 2013 21:12:45 +0000 Subject: [PATCH 007/426] Minor typo in server/failregex.py --- server/failregex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/failregex.py b/server/failregex.py index 84b237c7..b7084ce2 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -140,7 +140,7 @@ class FailRegex(Regex): ## # Returns unmatched lines. # - # This returns unmatched lines inlcuding captured by the tag. + # This returns unmatched lines including captured by the tag. # @return list of unmatched lines def getUnmatchedLines(self): From ea466d59f478c18e4a3695032499b3c390d93222 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:11:40 +0000 Subject: [PATCH 008/426] ignoreregex now functions correctly with multiline Ignore regexs are now only compared to lines that match the failregex. Supporting test also added for multiline regex and overlapping multiline regex matches. --- server/failregex.py | 66 +++++++++++++++++++------- server/filter.py | 9 ++-- testcases/files/testcase-multiline.log | 7 ++- testcases/filtertestcase.py | 16 ++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index b7084ce2..d98ffa27 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -93,6 +93,54 @@ class Regex: else: return False + ## + # Returns skipped lines. + # + # This returns skipped lines captured by the tag. + # @return list of skipped lines + + def getSkippedLines(self): + if not self._matchCache: + return [] + skippedLines = "" + n = 0 + while True: + try: + skippedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + return skippedLines.splitlines(True) + + ## + # Returns unmatched lines. + # + # This returns unmatched lines including captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + if not self._matchCache: + return [] + unmatchedLines = ( + self._matchCache.string[:self._matchCache.start()].splitlines(True) + + self.getSkippedLines() + + self._matchCache.string[self._matchCache.end():].splitlines(True)) + return unmatchedLines + + ## + # Returns matched lines. + # + # This returns matched lines by excluding those captured + # by the tag. + # @return list of matched lines + + def getMatchedLines(self): + if not self._matchCache: + return [] + matchedLines = self._matchCache.string[ + self._matchCache.start():self._matchCache.end()].splitlines(True) + return [line for line in matchedLines + if line not in self.getSkippedLines()] ## # Exception dedicated to the class Regex. @@ -136,21 +184,3 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host - - ## - # Returns unmatched lines. - # - # This returns unmatched lines including captured by the tag. - # @return list of unmatched lines - - def getUnmatchedLines(self): - unmatchedLines = self._matchCache.string[:self._matchCache.start()] - n = 0 - while True: - try: - unmatchedLines += self._matchCache.group("skiplines%i" % n) - n += 1 - except IndexError: - break - unmatchedLines += self._matchCache.string[self._matchCache.end():] - return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 0b0e0535..5f2f0f2b 100644 --- a/server/filter.py +++ b/server/filter.py @@ -374,14 +374,15 @@ class Filter(JailThread): def findFailure(self, timeLine, logLine): failList = list() - # Checks if we must ignore this line. - if self.ignoreLine(logLine): - # The ignoreregex matched. Return. - return failList # Iterates over all the regular expressions. for failRegex in self.__failRegex: failRegex.search(logLine) if failRegex.hasMatched(): + # Checks if we must ignore this match. + if self.ignoreLine("".join(failRegex.getMatchedLines())): + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer = failRegex.getUnmatchedLines() + continue # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) if date == None: diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index c0151d34..69dac361 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,3 +1,4 @@ +Aug 14 11:58:58 yyyy rsyncd[9874]: connect from example.com (192.0.43.10) Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: building file list @@ -9,6 +10,7 @@ Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes received 65115 bytes Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115 Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) + Aug 14 11:59:58 yyyy rsyncd[1762]: building file list Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. @@ -16,6 +18,8 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error + + Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list @@ -24,5 +28,6 @@ Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (c Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: Received SIGINT Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:59 yyyy rsyncd[9874]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 262074e6..4e60c242 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,7 +606,7 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 1, 1124013598.0), + output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -620,6 +620,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLineIgnoreRegex(self): + output = [("192.0.43.10", 2, 1124013599.0)] + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addIgnoreRegex("rsync error: Received SIGINT") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output.pop()) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From 7234c2a3aa07faa8dea755b973e835b13e8a37d6 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:16:55 +0000 Subject: [PATCH 009/426] Added multiregex test for multi-line filter --- testcases/files/testcase-multiline.log | 2 ++ testcases/filtertestcase.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 69dac361..a8d977ab 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -13,6 +13,7 @@ Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[1762]: building file list Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy sendmail[30222]: r0NNNlC0030222: from=, size=6420, class=0, nrcpts=1, msgid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=[192.0.43.15] (may be forged) Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 115 to 116 Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. @@ -25,6 +26,7 @@ Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy spamd[19119]: spamd: result: Y 11 - AWL,BAYES_50,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE,RCVD_IN_BRBL_LASTEXT,RCVD_IN_PSBL,RCVD_IN_RP_RNBL,RDNS_NONE,URIBL_BLACK,URIBL_DBL_SPAM scantime=1.2,size=6910,user=sa-milt,uid=499,required_score=5.0,rhost=localhost,raddr=127.0.0.1,rport=57429,mid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>,bayes=0.536244,autolearn=no Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: building file list diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 4e60c242..f61abacb 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -634,6 +634,24 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLineMultiRegex(self): + output = [("192.0.43.10", 2, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0), + ("192.0.43.15", 1, 1124013598.0)] + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From d05f42075811f4299954f1d74ee913b42fed465e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:28:48 +0000 Subject: [PATCH 010/426] Added FilterReader test --- fail2ban-testcases | 1 + testcases/clientreadertestcase.py | 37 +++++++++++++++ testcases/files/filter.d/testcase-common.conf | 47 +++++++++++++++++++ testcases/files/filter.d/testcase01.conf | 34 ++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 testcases/files/filter.d/testcase-common.conf create mode 100644 testcases/files/filter.d/testcase01.conf diff --git a/fail2ban-testcases b/fail2ban-testcases index aaf78525..3ea3f413 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -115,6 +115,7 @@ tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) # ClientReader tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) +tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) # Filter tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 83121345..058ed5dd 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -29,6 +29,8 @@ __license__ = "GPL" import unittest from client.jailreader import JailReader +from client.configreader import ConfigReader +from client.filterreader import FilterReader class JailReaderTest(unittest.TestCase): @@ -44,3 +46,38 @@ class JailReaderTest(unittest.TestCase): result = JailReader.splitAction(action) self.assertEquals(expected, result) +class FilterReaderTest(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + ConfigReader.setBaseDir("testcases/files/") + + def tearDown(self): + """Call after every test case.""" + + def testConvert(self): + output = [['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?Authentication failure for .* from \\s*$"], + ['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?User not known to the underlying authentication mo" + "dule for .* from \\s*$"], + ['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?User not known to the\\nunderlying authentication." + "+$^.+ module for .* from \\s*$"], + ['set', 'testcase01', 'addignoreregex', + "^.+ john from host 192.168.1.1\\s*$"]] + filterReader = FilterReader("testcase01", "testcase01") + filterReader.read() + #filterReader.getOptions(["failregex", "ignoreregex"]) + filterReader.getOptions(None) + + self.assertEquals(filterReader.convert(), output) diff --git a/testcases/files/filter.d/testcase-common.conf b/testcases/files/filter.d/testcase-common.conf new file mode 100644 index 00000000..18bf41c5 --- /dev/null +++ b/testcases/files/filter.d/testcase-common.conf @@ -0,0 +1,47 @@ +# Generic configuration items (to be used as interpolations) in other +# filters or actions configurations +# +# Author: Yaroslav Halchenko +# +# $Revision$ +# + +[INCLUDES] + +# Load customizations if any available +after = common.local + + +[DEFAULT] + +# Daemon definition is to be specialized (if needed) in .conf file +_daemon = \S* + +# +# Shortcuts for easier comprehension of the failregex +# +# PID. +# EXAMPLES: [123] +__pid_re = (?:\[\d+\]) + +# Daemon name (with optional source_file:line or whatever) +# EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix) +__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? + +# Combinations of daemon name and PID +# EXAMPLES: sshd[31607], pop(pam_unix)[4920] +__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:) + +# Some messages have a kernel prefix with a timestamp +# EXAMPLES: kernel: [769570.846956] +__kernel_prefix = kernel: \[\d+\.\d+\] + +__hostname = \S+ + +# +# Common line prefixes (beginnings) which could be used in filters +# +# [hostname] [vserver tag] daemon_id spaces +# this can be optional (for instance if we match named native log files) +__prefix_line = \s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s* + diff --git a/testcases/files/filter.d/testcase01.conf b/testcases/files/filter.d/testcase01.conf new file mode 100644 index 00000000..4a3a95e9 --- /dev/null +++ b/testcases/files/filter.d/testcase01.conf @@ -0,0 +1,34 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# +# $Revision$ +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + + +[Definition] + +_daemon = sshd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from \s*$ + ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ + ^%(__prefix_line)s(?:error: PAM: )?User not known to the\nunderlying authentication.+$^.+ module for .* from \s*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = ^.+ john from host 192.168.1.1\s*$ From 99914ac0f3d7af67f88b4ffb1c4413bb4c72871d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Jan 2013 09:17:48 +0000 Subject: [PATCH 011/426] Regex get(Un)MatchedLines now returns whole lines only Fix issue where for regexs not anchored at start/end of line, that getMatchedLines and getUnmatchedLines returned partial lines --- server/failregex.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index d98ffa27..608d5652 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -81,6 +81,19 @@ class Regex: def search(self, value): self._matchCache = self._regexObj.search(value) + if self.hasMatched(): + # Find start of the first line where the match was found + try: + self._matchLineStart = self._matchCache.string.rindex( + "\n", 0, self._matchCache.start() +1 ) + 1 + except ValueError: + self._matchLineStart = 0 + # Find end of the last line where the match was found + try: + self._matchLineEnd = self._matchCache.string.index( + "\n", self._matchCache.end() - 1) + 1 + except ValueError: + self._matchLineEnd = len(self._matchCache.string) ## # Checks if the previous call to search() matched. @@ -119,12 +132,12 @@ class Regex: # @return list of unmatched lines def getUnmatchedLines(self): - if not self._matchCache: + if not self.hasMatched(): return [] unmatchedLines = ( - self._matchCache.string[:self._matchCache.start()].splitlines(True) + self._matchCache.string[:self._matchLineStart].splitlines(True) + self.getSkippedLines() - + self._matchCache.string[self._matchCache.end():].splitlines(True)) + + self._matchCache.string[self._matchLineEnd:].splitlines(True)) return unmatchedLines ## @@ -135,10 +148,10 @@ class Regex: # @return list of matched lines def getMatchedLines(self): - if not self._matchCache: + if not self.hasMatched(): return [] matchedLines = self._matchCache.string[ - self._matchCache.start():self._matchCache.end()].splitlines(True) + self._matchLineStart:self._matchLineEnd].splitlines(True) return [line for line in matchedLines if line not in self.getSkippedLines()] From b48c17b8c49aed67f0979b462d7c5e0dead211ef Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Jan 2013 10:41:58 +0000 Subject: [PATCH 012/426] Added 'maxlines' option to fail2ban-regex This allows multi-line regex to be tested --- fail2ban-regex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 3900c909..4e5698ab 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -105,6 +105,7 @@ class Fail2banRegex: print " -h, --help display this help message" print " -V, --version print the version" print " -v, --verbose verbose output" + print " -l INT, --maxlines=INT set maxlines for multi-line regex default: 1" print print "Log:" print " string a string representing a log line" @@ -133,6 +134,14 @@ class Fail2banRegex: sys.exit(0) elif opt[0] in ["-v", "--verbose"]: self.__verbose = True + elif opt[0] in ["-l", "--maxlines"]: + try: + self.__filter.setMaxLines(int(opt[1])) + except ValueError: + print "Invlaid value for maxlines: %s" % ( + opt[1]) + fail2banRegex.dispUsage() + sys.exit(-1) #@staticmethod def logIsFile(value): @@ -310,8 +319,8 @@ if __name__ == "__main__": fail2banRegex = Fail2banRegex() # Reads the command line options. try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] + cmdOpts = 'hVcvl:' + cmdLongOpts = ['help', 'version', 'verbose', 'maxlines='] optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: fail2banRegex.dispUsage() From 02218294bc0d012e22cec7d4c33c810df187a2be Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 28 Jan 2013 18:41:12 +0000 Subject: [PATCH 013/426] Removed "common.local" include for FilterReader test --- testcases/files/filter.d/testcase-common.conf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/testcases/files/filter.d/testcase-common.conf b/testcases/files/filter.d/testcase-common.conf index 18bf41c5..af7df1f7 100644 --- a/testcases/files/filter.d/testcase-common.conf +++ b/testcases/files/filter.d/testcase-common.conf @@ -6,12 +6,6 @@ # $Revision$ # -[INCLUDES] - -# Load customizations if any available -after = common.local - - [DEFAULT] # Daemon definition is to be specialized (if needed) in .conf file From efea62e03f9da6e277d9a60a5af9e3df87f692ec Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 28 Jan 2013 20:47:32 +0000 Subject: [PATCH 014/426] Revert changes to man/fail2ban-client.1 --- man/fail2ban-client.1 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index d60c9481..1bbddf09 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -145,11 +145,6 @@ sets the number of failures before banning the host for .TP -\fBset maxlines \fR -sets the number of to -buffer for regex search for - -.TP \fBset addaction \fR adds a new action named for @@ -227,10 +222,6 @@ gets the time a host is banned for gets the number of failures allowed for .TP -\fBget maxlines\fR -gets the number lines to -buffer for -.TP \fBget addaction\fR gets the last action which has been added for From 9dc662af2794f11ab3d1dc1e68c14b87038f486e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Feb 2013 16:00:05 -0500 Subject: [PATCH 015/426] Introducing 0.9.x series with 0.9.0a0 0.9.0a0 is chosen so that StrictVersion works within python 2.x --- ChangeLog | 12 +++++++++++- README | 2 +- common/version.py | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index eabd6e81..59533c64 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,19 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.8) 2012/12/06 +Fail2Ban (version 0.9.0a) 20??/??/?? ================================================================================ + +ver. 0.9.0 (20??/??/??) - alpha +---------- + +Will carry all fixes in 0.8.x series and new features and enhancements + +- Fixes: +- New features: +- Enhancements: + ver. 0.8.8 (2012/12/06) - stable ---------- - Fixes: diff --git a/README b/README index db97aa8b..0a3d5117 100644 --- a/README +++ b/README @@ -4,7 +4,7 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.8) 2012/07/31 +Fail2Ban (version 0.9.0a0) 20??/??/?? ================================================================================ Fail2Ban scans log files like /var/log/pwdfail and bans IP that makes too many diff --git a/common/version.py b/common/version.py index 2a1c0d0b..a4499b21 100644 --- a/common/version.py +++ b/common/version.py @@ -22,7 +22,7 @@ # $Revision$ __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -version = "0.8.8" +version = "0.9.0a0" From 4d4c2d7e0209bd1d7c93ac936df2f19441f228d9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Feb 2013 16:04:44 -0500 Subject: [PATCH 016/426] Brief changelog entry for multiline failregex. With this Close gh-54 --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 59533c64..e5979331 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ Will carry all fixes in 0.8.x series and new features and enhancements - Fixes: - New features: + Steven Hiscocks + * Multiline failregex. Close gh-54 - Enhancements: ver. 0.8.8 (2012/12/06) - stable From c926c88ea13577151d30e88ed32a610dee205ca0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 18:42:50 +0000 Subject: [PATCH 017/426] Remove spurious space at start of line --- server/action.py | 2 +- testcases/filtertestcase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/action.py b/server/action.py index 35974c70..65263751 100644 --- a/server/action.py +++ b/server/action.py @@ -342,7 +342,7 @@ class Action: return True else: msg = _RETCODE_HINTS.get(retcode, None) - logSys.error("%s returned %x" % (realCmd, retcode)) + logSys.error("%s returned %x" % (realCmd, retcode)) if msg: logSys.info("HINT on %x: %s" % (retcode, msg % locals())) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index c10fa78d..3b764729 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -409,7 +409,7 @@ def get_monitor_failures_testcase(Filter_): #return # just for fun let's copy all of them again and see if that results # in a new ban - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) self.assert_correct_last_attempt(GetFailures.FAILURES_01) def test_rewrite_file(self): From 99c92a56ca602bad7936966843d6ca20e70ee37f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 18:47:56 +0000 Subject: [PATCH 018/426] Check for unicode and encode before md5sum in filter --- server/filter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/filter.py b/server/filter.py index fb79fcbe..dab8cb8a 100644 --- a/server/filter.py +++ b/server/filter.py @@ -526,7 +526,10 @@ class FileContainer: try: firstLine = handler.readline() # Computes the MD5 of the first line. - self.__hash = md5sum(firstLine).digest() + if isinstance(firstLine, unicode): + self.__hash = md5sum(firstLine.encode('utf-8')).digest() + else: + self.__hash = md5sum(firstLine).digest() # Start at the beginning of file if tail mode is off. if tail: handler.seek(0, 2) @@ -546,7 +549,10 @@ class FileContainer: fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC) firstLine = self.__handler.readline() # Computes the MD5 of the first line. - myHash = md5sum(firstLine).digest() + if isinstance(firstLine, unicode): + myHash = md5sum(firstLine.encode('utf-8')).digest() + else: + myHash = md5sum(firstLine).digest() stats = os.fstat(self.__handler.fileno()) # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: From 2ecf34540e601e01381bcd41ac6308c3f9299109 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 19:33:49 +0000 Subject: [PATCH 019/426] Change sort in datedetector to use key based comparison --- server/datedetector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/datedetector.py b/server/datedetector.py index c013d551..d31667fb 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -204,7 +204,7 @@ class DateDetector: self.__lock.acquire() try: logSys.debug("Sorting the template list") - self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits()), reverse=True) + self.__templates.sort(key=lambda x: x.getHits(), reverse=True) t = self.__templates[0] logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits())) finally: From ad7119a3fe4fb3ad33716911f1f62c3e2a5670ad Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 21:01:39 +0000 Subject: [PATCH 020/426] time.mktime argument should be tuple --- server/datedetector.py | 2 +- server/datetemplate.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/datedetector.py b/server/datedetector.py index d31667fb..7876580d 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -194,7 +194,7 @@ class DateDetector: if date == None: return None else: - return time.mktime(date) + return time.mktime(tuple(date)) ## # Sort the template lists using the hits score. This method is not called diff --git a/server/datetemplate.py b/server/datetemplate.py index f663862e..14076030 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -162,10 +162,10 @@ class DateStrptime(DateTemplate): # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before - if time.mktime(date) > MyTime.time(): + if time.mktime(tuple(date)) > MyTime.time(): logSys.debug( - u"Correcting deduced year from %d to %d since %f > %f" % - (date[0], date[0]-1, time.mktime(date), MyTime.time())) + "Correcting deduced year from %d to %d since %f > %f" % + (date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time())) date[0] -= 1 elif date[1] == 1 and date[2] == 1: # If it is Jan 1st, it is either really Jan 1st or there From 2bb346964432080899d1bf17223841f4b5b1cbae Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 21:03:51 +0000 Subject: [PATCH 021/426] Change filter to ignore unicode errors in python3 Also do not try to convert unicode to unicode again for python3 and python2 --- server/filter.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/server/filter.py b/server/filter.py index dab8cb8a..e47458de 100644 --- a/server/filter.py +++ b/server/filter.py @@ -35,7 +35,7 @@ from datedetector import DateDetector from mytime import MyTime from failregex import FailRegex, Regex, RegexException -import logging, re, os, fcntl, time +import logging, re, os, fcntl, time, sys # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -289,22 +289,24 @@ class Filter(JailThread): def processLine(self, line): """Split the time portion from log msg and return findFailures on them """ - try: - # Decode line to UTF-8 - l = line.decode('utf-8') - except UnicodeDecodeError: - l = line - timeMatch = self.dateDetector.matchTime(l) + if (sys.version_info >= (3,) and isinstance(line, bytes)) or \ + (sys.version_info < (3,) and isinstance(line, str)): + try: + # Decode line to UTF-8 + line = line.decode('utf-8') + except UnicodeDecodeError: + line = line + timeMatch = self.dateDetector.matchTime(line) if timeMatch: # Lets split into time part and log part of the line timeLine = timeMatch.group() # Lets leave the beginning in as well, so if there is no # anchore at the beginning of the time regexp, we don't # at least allow injection. Should be harmless otherwise - logLine = l[:timeMatch.start()] + l[timeMatch.end():] + logLine = line[:timeMatch.start()] + line[timeMatch.end():] else: - timeLine = l - logLine = l + timeLine = line + logLine = line return self.findFailure(timeLine, logLine) def processLineAndAdd(self, line): @@ -520,7 +522,10 @@ class FileContainer: self.__tail = tail self.__handler = None # Try to open the file. Raises an exception if an error occured. - handler = open(filename) + if sys.version_info >= (3,): + handler = open(filename, errors='ignore') + else: + handler = open(filename) stats = os.fstat(handler.fileno()) self.__ino = stats.st_ino try: From 53be2ade863d0ead9ca53566fad2ef8684f80399 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 21:05:23 +0000 Subject: [PATCH 022/426] Wrap open method to allow python3 to ignore unicode errors --- testcases/filtertestcase.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 3b764729..8ab3fca8 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -27,6 +27,7 @@ import os import sys import time import tempfile +import functools from server.jail import Jail from server.filterpoll import FilterPoll @@ -38,6 +39,11 @@ from server.failmanager import FailManagerEmpty # Useful helpers # +if sys.version_info >= (3,): + open_ = functools.partial(open, errors='ignore') +else: + open_ = open + def _killfile(f, name): try: f.close() @@ -104,9 +110,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # polling filter could detect the change time.sleep(1) if isinstance(fin, str): - fin = open(fin, 'r') + fin = open_(fin, 'r') if isinstance(fout, str): - fout = open(fout, mode) + fout = open_(fout, mode) # Skip for i in xrange(skip): _ = fin.readline() @@ -184,7 +190,7 @@ class LogFileMonitor(unittest.TestCase): """Call before every test case.""" self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') - self.file = open(self.name, 'a') + self.file = open_(self.name, 'a') self.filter = FilterPoll(None) self.filter.addLogPath(self.name) self.filter.setActive(True) @@ -226,7 +232,7 @@ class LogFileMonitor(unittest.TestCase): # we are not signaling as modified whenever # it gets away self.assertTrue(self.notModified()) - f = open(self.name, 'a') + f = open_(self.name, 'a') self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) _sleep_4_poll() @@ -329,7 +335,7 @@ def get_monitor_failures_testcase(Filter_): """Call before every test case.""" self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') - self.file = open(self.name, 'a') + self.file = open_(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) self.filter.addLogPath(self.name) @@ -454,7 +460,7 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01) # create a bogus file in the same directory and see if that doesn't affect - open(self.name + '.bak2', 'w').write('') + open_(self.name + '.bak2', 'w').write('') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) From 5d0d362e3ff039467149e69ae17c7dcdf8b2a952 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 22:06:16 +0000 Subject: [PATCH 023/426] Ensure jail names return in alphabetical order in status Primarily to support testing as order can switch --- server/server.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/server.py b/server/server.py index 0e1b6c97..4a086078 100644 --- a/server/server.py +++ b/server/server.py @@ -282,12 +282,9 @@ class Server: def status(self): try: self.__lock.acquire() - jailList = '' - for jail in self.__jails.getAll(): - jailList += jail + ', ' - length = len(jailList) - if not length == 0: - jailList = jailList[:length-2] + jails = list(self.__jails.getAll()) + jails.sort() + jailList = ", ".join(jails) ret = [("Number of jail", self.__jails.size()), ("Jail list", jailList)] return ret From df255063ae3527edf29ec741528a1d434ad042d1 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Feb 2013 22:19:26 +0000 Subject: [PATCH 024/426] python3: Default open encoding for files to UTF-8 Also tidy up unicode check in processLine, which is handled by `2to3` --- server/filter.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/filter.py b/server/filter.py index e47458de..b3b99e30 100644 --- a/server/filter.py +++ b/server/filter.py @@ -289,8 +289,7 @@ class Filter(JailThread): def processLine(self, line): """Split the time portion from log msg and return findFailures on them """ - if (sys.version_info >= (3,) and isinstance(line, bytes)) or \ - (sys.version_info < (3,) and isinstance(line, str)): + if not isinstance(line, unicode): try: # Decode line to UTF-8 line = line.decode('utf-8') @@ -523,7 +522,7 @@ class FileContainer: self.__handler = None # Try to open the file. Raises an exception if an error occured. if sys.version_info >= (3,): - handler = open(filename, errors='ignore') + handler = open(filename, encoding='utf-8', errors='ignore') else: handler = open(filename) stats = os.fstat(handler.fileno()) @@ -548,7 +547,11 @@ class FileContainer: return self.__filename def open(self): - self.__handler = open(self.__filename) + if sys.version_info >= (3,): + self.__handler = open( + self.__filename, encoding='utf-8', errors='ignore') + else: + self.__handler = open(self.__filename) # Set the file descriptor to be FD_CLOEXEC fd = self.__handler.fileno() fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC) From 6f4da8f3c4ceb771c785df55cbb4ca0d3b4fc5cd Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 10:04:36 +0000 Subject: [PATCH 025/426] Remove spurious space in fail2ban-server --- fail2ban-server | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban-server b/fail2ban-server index 81db58bd..87ddd042 100755 --- a/fail2ban-server +++ b/fail2ban-server @@ -104,10 +104,10 @@ class Fail2banServer: if opt[0] == "-x": self.__conf["force"] = True if opt[0] in ["-h", "--help"]: - self.dispUsage() + self.dispUsage() sys.exit(0) if opt[0] in ["-V", "--version"]: - self.dispVersion() + self.dispVersion() sys.exit(0) def start(self, argv): From 46a2b6e428c1e54293182b343ecb83eb2b5314d1 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 10:10:15 +0000 Subject: [PATCH 026/426] Fix up for client/server socket for python3 --- client/csocket.py | 12 +++++++++--- server/asyncserver.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/client/csocket.py b/client/csocket.py index 6e014e23..3963bbe5 100644 --- a/client/csocket.py +++ b/client/csocket.py @@ -29,11 +29,14 @@ __license__ = "GPL" #from cPickle import dumps, loads, HIGHEST_PROTOCOL from pickle import dumps, loads, HIGHEST_PROTOCOL -import socket +import socket, sys class CSocket: - END_STRING = "" + if sys.version_info >= (3,): + END_STRING = b"" + else: + END_STRING = "" def __init__(self, sock = "/var/run/fail2ban/fail2ban.sock"): # Create an INET, STREAMing socket @@ -52,7 +55,10 @@ class CSocket: #@staticmethod def receive(sock): - msg = '' + if sys.version_info >= (3,): + msg = b'' + else: + msg = '' while msg.rfind(CSocket.END_STRING) == -1: chunk = sock.recv(6) if chunk == '': diff --git a/server/asyncserver.py b/server/asyncserver.py index 64ec8f39..0e06a790 100644 --- a/server/asyncserver.py +++ b/server/asyncserver.py @@ -42,7 +42,10 @@ logSys = logging.getLogger("fail2ban.server") class RequestHandler(asynchat.async_chat): - END_STRING = "" + if sys.version_info >= (3,): + END_STRING = b"" + else: + END_STRING = "" def __init__(self, conn, transmitter): asynchat.async_chat.__init__(self, conn) @@ -62,7 +65,10 @@ class RequestHandler(asynchat.async_chat): def found_terminator(self): # Joins the buffer items. - message = loads("".join(self.__buffer)) + if sys.version_info >= (3,): + message = loads(b"".join(self.__buffer)) + else: + message = loads("".join(self.__buffer)) # Gives the message to the transmitter. message = self.__transmitter.proceed(message) # Serializes the response. From e28a698c0eb7976c4ab51c44b3f5f4e94e90f6d3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 10:11:34 +0000 Subject: [PATCH 027/426] Change filter testcases python3 open wrapper to utf-8 --- testcases/filtertestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 8ab3fca8..e8f19260 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -40,7 +40,7 @@ from server.failmanager import FailManagerEmpty # if sys.version_info >= (3,): - open_ = functools.partial(open, errors='ignore') + open_ = functools.partial(open, encoding='utf-8', errors='ignore') else: open_ = open From 0dd3a81ba276bb61a9332f9612bd13a276d4689e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 18:04:40 +0000 Subject: [PATCH 028/426] Fix use of python3 bytes in client/server socket for python2.5 --- client/csocket.py | 4 ++-- server/asyncserver.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/csocket.py b/client/csocket.py index 3963bbe5..283d6357 100644 --- a/client/csocket.py +++ b/client/csocket.py @@ -34,7 +34,7 @@ import socket, sys class CSocket: if sys.version_info >= (3,): - END_STRING = b"" + END_STRING = bytes("", encoding='ascii') else: END_STRING = "" @@ -56,7 +56,7 @@ class CSocket: #@staticmethod def receive(sock): if sys.version_info >= (3,): - msg = b'' + msg = bytes("", encoding='ascii') else: msg = '' while msg.rfind(CSocket.END_STRING) == -1: diff --git a/server/asyncserver.py b/server/asyncserver.py index 0e06a790..a671d877 100644 --- a/server/asyncserver.py +++ b/server/asyncserver.py @@ -43,7 +43,7 @@ logSys = logging.getLogger("fail2ban.server") class RequestHandler(asynchat.async_chat): if sys.version_info >= (3,): - END_STRING = b"" + END_STRING = bytes("", encoding="ascii") else: END_STRING = "" @@ -66,7 +66,7 @@ class RequestHandler(asynchat.async_chat): def found_terminator(self): # Joins the buffer items. if sys.version_info >= (3,): - message = loads(b"".join(self.__buffer)) + message = loads(bytes("", encoding="ascii").join(self.__buffer)) else: message = loads("".join(self.__buffer)) # Gives the message to the transmitter. From c8c9ed4dfcf5526defc84a18279f38d79fb7b2ca Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 18:13:48 +0000 Subject: [PATCH 029/426] Fix DNSUtils dnsToIp not catching general socket.error exception --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index b3b99e30..2669cc7f 100644 --- a/server/filter.py +++ b/server/filter.py @@ -605,7 +605,7 @@ class DNSUtils: """ try: return socket.gethostbyname_ex(dns)[2] - except socket.gaierror: + except socket.error: logSys.warn("Unable to find a corresponding IP address for %s" % dns) return list() From dbc40d6210f1b67e18834cf9c84d822013c52897 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 18:27:40 +0000 Subject: [PATCH 030/426] Added helper scripts to carry out 2to3 and testcases on python3 --- fail2ban-2to3 | 17 +++++++++++++++++ fail2ban-testcases-all-python3 | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100755 fail2ban-2to3 create mode 100755 fail2ban-testcases-all-python3 diff --git a/fail2ban-2to3 b/fail2ban-2to3 new file mode 100755 index 00000000..f0e292d6 --- /dev/null +++ b/fail2ban-2to3 @@ -0,0 +1,17 @@ +#!/bin/bash +# This script carries out conversion of fail2ban to python3 +# A backup of any converted files are created with ".bak" +# extension + +set -eu + +nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases" + +find . \ + -name "*.py" \ + -exec 2to3 -w --no-diffs $nonPyFiles {} + \ + || echo "Fail!" >&2 && exit 1 + +echo "Success!" >&2 + +exit 0 diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3 new file mode 100755 index 00000000..094f2938 --- /dev/null +++ b/fail2ban-testcases-all-python3 @@ -0,0 +1,18 @@ +#!/bin/bash +# Simple helper script to exercise unittests using all available +# (under /usr/bin and /usr/local/bin python2.*) + +set -eu + +failed= +for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg} +do + [ -e "$python" ] || continue + echo "Testing using $python" + $python ./fail2ban-testcases "$@" || failed+=" $python" +done + +if [ ! -z "$failed" ]; then + echo "E: Failed with $failed" + exit 1 +fi From 418d845f9b5579bc11e9f3a0bcba8b9ea62a7fac Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 18:59:07 +0000 Subject: [PATCH 031/426] fail2ban-regex open files as utf-8 for python3 --- fail2ban-regex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index f9bc72c1..8bd2ff63 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -346,7 +346,10 @@ if __name__ == "__main__": if fail2banRegex.logIsFile(cmd_log): try: - hdlr = open(cmd_log) + if sys.version_info >= (3,): + hdlr = open(cmd_log, encoding='utf-8', errors='ignore') + else: + hdlr = open(cmd_log) print "Use log file : " + cmd_log print for line in hdlr: From 31b173f0320f471899b9329ef941d6b92ba6fa74 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 19:13:36 +0000 Subject: [PATCH 032/426] Remove functools from filter testcases for python2.4 compatibility --- testcases/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index e8f19260..d9e859a3 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -27,7 +27,6 @@ import os import sys import time import tempfile -import functools from server.jail import Jail from server.filterpoll import FilterPoll @@ -40,7 +39,8 @@ from server.failmanager import FailManagerEmpty # if sys.version_info >= (3,): - open_ = functools.partial(open, encoding='utf-8', errors='ignore') + def open_(filename, mode): + return open(filename, mode, encoding='utf-8', errors='ignore') else: open_ = open From 3a3d07ef39d7cf8fefefb373e9acab879b59bd77 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 19:22:50 +0000 Subject: [PATCH 033/426] Undo removal of unicode prefix in server/datetemplate.py --- server/datetemplate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/datetemplate.py b/server/datetemplate.py index 14076030..ba4477f6 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -164,7 +164,7 @@ class DateStrptime(DateTemplate): # that the log is not from this year but from the year before if time.mktime(tuple(date)) > MyTime.time(): logSys.debug( - "Correcting deduced year from %d to %d since %f > %f" % + u"Correcting deduced year from %d to %d since %f > %f" % (date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time())) date[0] -= 1 elif date[1] == 1 and date[2] == 1: From 78d86bc38dbd4ca3d70ece3a268506c307d7444a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 19:24:07 +0000 Subject: [PATCH 034/426] Minor typo in fail2ban-testcases-all-python3 --- fail2ban-testcases-all-python3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3 index 094f2938..fd3ff871 100755 --- a/fail2ban-testcases-all-python3 +++ b/fail2ban-testcases-all-python3 @@ -1,6 +1,6 @@ #!/bin/bash # Simple helper script to exercise unittests using all available -# (under /usr/bin and /usr/local/bin python2.*) +# (under /usr/bin and /usr/local/bin python3.*) set -eu From 184e0eccb6b054799f51faa63c05a1661e083e21 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 19:28:41 +0000 Subject: [PATCH 035/426] Remove redundant reassignment of variable --- server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/filter.py b/server/filter.py index 2669cc7f..6a2cf342 100644 --- a/server/filter.py +++ b/server/filter.py @@ -294,7 +294,7 @@ class Filter(JailThread): # Decode line to UTF-8 line = line.decode('utf-8') except UnicodeDecodeError: - line = line + pass timeMatch = self.dateDetector.matchTime(line) if timeMatch: # Lets split into time part and log part of the line From 7e1819ed65954da7f56f77e8835d6eef759c9338 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 24 Feb 2013 19:34:05 +0000 Subject: [PATCH 036/426] Fix incorrect exit code from fail2ban-2to3 --- fail2ban-2to3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban-2to3 b/fail2ban-2to3 index f0e292d6..a6a61090 100755 --- a/fail2ban-2to3 +++ b/fail2ban-2to3 @@ -10,7 +10,7 @@ nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases" find . \ -name "*.py" \ -exec 2to3 -w --no-diffs $nonPyFiles {} + \ - || echo "Fail!" >&2 && exit 1 + || (echo "Fail!" >&2 && exit 1) echo "Success!" >&2 From d23d365be2b2027ab745e2222d3ae48550256339 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 25 Feb 2013 22:45:16 +0000 Subject: [PATCH 037/426] Move handling of unicode decoding to FileContainer readline Also print warning for unicode decode failure, leaving as str in python2 and ignoring erroneous characters in python3 --- server/filter.py | 38 +++++++++++++------------------------ testcases/filtertestcase.py | 10 +++++----- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/server/filter.py b/server/filter.py index 6a2cf342..3f507cd1 100644 --- a/server/filter.py +++ b/server/filter.py @@ -289,12 +289,6 @@ class Filter(JailThread): def processLine(self, line): """Split the time portion from log msg and return findFailures on them """ - if not isinstance(line, unicode): - try: - # Decode line to UTF-8 - line = line.decode('utf-8') - except UnicodeDecodeError: - pass timeMatch = self.dateDetector.matchTime(line) if timeMatch: # Lets split into time part and log part of the line @@ -485,7 +479,7 @@ class FileFilter(Filter): while True: line = container.readline() - if (line == "") or not self._isActive(): + if not line or not self._isActive(): # The jail reached the bottom or has been stopped break self.processLineAndAdd(line) @@ -521,19 +515,13 @@ class FileContainer: self.__tail = tail self.__handler = None # Try to open the file. Raises an exception if an error occured. - if sys.version_info >= (3,): - handler = open(filename, encoding='utf-8', errors='ignore') - else: - handler = open(filename) + handler = open(filename, 'rb') stats = os.fstat(handler.fileno()) self.__ino = stats.st_ino try: firstLine = handler.readline() # Computes the MD5 of the first line. - if isinstance(firstLine, unicode): - self.__hash = md5sum(firstLine.encode('utf-8')).digest() - else: - self.__hash = md5sum(firstLine).digest() + self.__hash = md5sum(firstLine).digest() # Start at the beginning of file if tail mode is off. if tail: handler.seek(0, 2) @@ -547,20 +535,13 @@ class FileContainer: return self.__filename def open(self): - if sys.version_info >= (3,): - self.__handler = open( - self.__filename, encoding='utf-8', errors='ignore') - else: - self.__handler = open(self.__filename) + self.__handler = open(self.__filename, 'rb') # Set the file descriptor to be FD_CLOEXEC fd = self.__handler.fileno() fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC) firstLine = self.__handler.readline() # Computes the MD5 of the first line. - if isinstance(firstLine, unicode): - myHash = md5sum(firstLine.encode('utf-8')).digest() - else: - myHash = md5sum(firstLine).digest() + myHash = md5sum(firstLine).digest() stats = os.fstat(self.__handler.fileno()) # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: @@ -574,7 +555,14 @@ class FileContainer: def readline(self): if self.__handler == None: return "" - return self.__handler.readline() + line = self.__handler.readline() + try: + line = line.decode('utf-8', 'strict') + except UnicodeDecodeError: + logSys.warn("Error decoding line to utf-8: %s" % `line`) + if sys.version_info >= (3,): # In python3, must be unicode + line = line.decode('utf-8', 'ignore') + return line def close(self): if not self.__handler == None: diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index d9e859a3..58384449 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -508,7 +508,7 @@ class GetFailures(unittest.TestCase): # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, - ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) + [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) def setUp(self): """Call before every test case.""" @@ -532,7 +532,7 @@ class GetFailures(unittest.TestCase): def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, - ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' + [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' % m for m in 53, 54, 57, 58]) self.filter.addLogPath(GetFailures.FILENAME_02) @@ -565,11 +565,11 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ('192.0.43.10', 2, 1124013539.0, - ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', - 'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', + u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) output_no = ('192.0.43.10', 1, 1124013539.0, - ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n']) # Actually no exception would be raised -- it will be just set to 'no' #self.assertRaises(ValueError, From 66367876bbc780688c5755e3654bb5b13ea58390 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 27 Feb 2013 18:09:55 +0000 Subject: [PATCH 038/426] Add ability to set log encoding for jail --- client/beautifier.py | 3 +++ client/jailreader.py | 3 +++ common/protocol.py | 2 ++ config/jail.conf | 7 ++++++ server/filter.py | 46 +++++++++++++++++++++++++++++++------ server/server.py | 6 +++++ server/transmitter.py | 6 +++++ testcases/servertestcase.py | 9 +++++++- 8 files changed, 74 insertions(+), 8 deletions(-) diff --git a/client/beautifier.py b/client/beautifier.py index 7e48016c..1653ff63 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -110,6 +110,9 @@ class Beautifier: for path in response[:-1]: msg = msg + "|- " + path + "\n" msg = msg + "`- " + response[len(response)-1] + elif inC[2] == "logencoding": + msg = "Current log encoding is set to:\n" + msg = msg + response elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"): if len(response) == 0: msg = "No IP address/network is ignored" diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..be22a78f 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -61,6 +61,7 @@ class JailReader(ConfigReader): def getOptions(self): opts = [["bool", "enabled", "false"], ["string", "logpath", "/var/log/messages"], + ["string", "logencoding", "auto"], ["string", "backend", "auto"], ["int", "maxretry", 3], ["int", "findtime", 600], @@ -110,6 +111,8 @@ class JailReader(ConfigReader): logSys.error("No file found for " + path) for p in pathList: stream.append(["set", self.__name, "addlogpath", p]) + elif opt == "logencoding": + stream.append(["set", self.__name, "logencoding", self.__opts[opt]]) elif opt == "backend": backend = self.__opts[opt] elif opt == "maxretry": diff --git a/common/protocol.py b/common/protocol.py index 99a2fe09..99d608a8 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -56,6 +56,7 @@ protocol = [ ["set delignoreip ", "removes from the ignore list of "], ["set addlogpath ", "adds to the monitoring list of "], ["set dellogpath ", "removes from the monitoring list of "], +["set logencoding ", "sets the of the log files for "], ["set addfailregex ", "adds the regular expression which must match failures for "], ["set delfailregex ", "removes the regular expression at for failregex"], ["set addignoreregex ", "adds the regular expression which should match pattern to exclude for "], @@ -77,6 +78,7 @@ protocol = [ ["set actionunban ", "sets the unban command of the action for "], ['', "JAIL INFORMATION", ""], ["get logpath", "gets the list of the monitored files for "], +["get logencoding ", "gets the of the log files for "], ["get ignoreip", "gets the list of ignored IP addresses for "], ["get failregex", "gets the list of regular expressions which matches the failures for "], ["get ignoreregex", "gets the list of regular expressions which matches patterns to ignore for "], diff --git a/config/jail.conf b/config/jail.conf index a0093f68..e56023d7 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -55,6 +55,13 @@ backend = auto # but it will be logged as info. usedns = warn +# "logencoding" specifies the encoding of the log files handled by the jail +# This is used to decode the lines from the log file. +# Typical examples: "ascii", "utf-8" +# +# auto: will use the system locale setting +logencoding = auto + # This jail corresponds to the standard configuration in Fail2ban 0.6. # The mail-whois action send a notification e-mail with a whois request diff --git a/server/filter.py b/server/filter.py index 3f507cd1..57305431 100644 --- a/server/filter.py +++ b/server/filter.py @@ -35,7 +35,7 @@ from datedetector import DateDetector from mytime import MyTime from failregex import FailRegex, Regex, RegexException -import logging, re, os, fcntl, time, sys +import logging, re, os, fcntl, time, sys, locale, codecs # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -392,6 +392,7 @@ class FileFilter(Filter): Filter.__init__(self, jail, **kwargs) ## The log file path. self.__logPath = [] + self.setLogEncoding("auto") ## # Add a log file path @@ -402,7 +403,7 @@ class FileFilter(Filter): if self.containsLogPath(path): logSys.error(path + " already exists") else: - container = FileContainer(path, tail) + container = FileContainer(path, self.getLogEncoding(), tail) self.__logPath.append(container) logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -451,6 +452,28 @@ class FileFilter(Filter): return True return False + ## + # Set the log file encoding + # + # @param encoding the encoding used with log files + + def setLogEncoding(self, encoding): + if encoding.lower() == "auto": + encoding = locale.getpreferredencoding() + codecs.lookup(encoding) # Raise LookupError if invalid codec + for log in self.getLogPath(): + log.setEncoding(encoding) + self.__encoding = encoding + logSys.info("Set jail log file encoding to %s" % encoding) + + ## + # Get the log file encoding + # + # @return log encoding value + + def getLogEncoding(self): + return self.__encoding + def getFileContainer(self, path): for log in self.__logPath: if log.getFileName() == path: @@ -510,8 +533,9 @@ except ImportError: class FileContainer: - def __init__(self, filename, tail = False): + def __init__(self, filename, encoding, tail = False): self.__filename = filename + self.setEncoding(encoding) self.__tail = tail self.__handler = None # Try to open the file. Raises an exception if an error occured. @@ -534,6 +558,13 @@ class FileContainer: def getFileName(self): return self.__filename + def setEncoding(self, encoding): + codecs.lookup(encoding) # Raises LookupError if invalid + self.__encoding = encoding + + def getEncoding(self): + return self.__encoding + def open(self): self.__handler = open(self.__filename, 'rb') # Set the file descriptor to be FD_CLOEXEC @@ -557,11 +588,12 @@ class FileContainer: return "" line = self.__handler.readline() try: - line = line.decode('utf-8', 'strict') + line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warn("Error decoding line to utf-8: %s" % `line`) - if sys.version_info >= (3,): # In python3, must be unicode - line = line.decode('utf-8', 'ignore') + logSys.warn("Error decoding line from '%s' with '%s': %s" % + (self.getFileName(), self.getEncoding(), `line`)) + if sys.version_info >= (3,): # In python3, must be decoded + line = line.decode(self.getEncoding(), 'ignore') return line def close(self): diff --git a/server/server.py b/server/server.py index 4a086078..0db03d09 100644 --- a/server/server.py +++ b/server/server.py @@ -181,6 +181,12 @@ class Server: return [m.getFileName() for m in self.__jails.getFilter(name).getLogPath()] + def setLogEncoding(self, name, encoding): + return self.__jails.getFilter(name).setLogEncoding(encoding) + + def getLogEncoding(self, name): + return self.__jails.getFilter(name).getLogEncoding() + def setFindTime(self, name, value): self.__jails.getFilter(name).setFindTime(value) diff --git a/server/transmitter.py b/server/transmitter.py index a02b94a2..1c1b1e2c 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -139,6 +139,10 @@ class Transmitter: value = command[2] self.__server.delLogPath(name, value) return self.__server.getLogPath(name) + elif command[1] == "logencoding": + value = command[2] + self.__server.setLogEncoding(name, value) + return self.__server.getLogEncoding(name) elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value) @@ -234,6 +238,8 @@ class Transmitter: # Filter elif command[1] == "logpath": return self.__server.getLogPath(name) + elif command[1] == "logencoding": + return self.__server.getLogEncoding(name) elif command[1] == "ignoreip": return self.__server.getIgnoreIP(name) elif command[1] == "failregex": diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 00f56b81..f52fecca 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest, socket, time, tempfile, os +import unittest, socket, time, tempfile, os, locale from server.server import Server class StartStop(unittest.TestCase): @@ -258,6 +258,13 @@ class Transmitter(unittest.TestCase): self.setGetTest("maxretry", "-2", -2, jail=self.jailName) self.setGetTestNOK("maxretry", "Duck", jail=self.jailName) + def testJailLogEncoding(self): + self.setGetTest("logencoding", "UTF-8", jail=self.jailName) + self.setGetTest("logencoding", "ascii", jail=self.jailName) + self.setGetTest("logencoding", "auto", locale.getpreferredencoding(), + jail=self.jailName) + self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName) + def testJailLogPath(self): self.jailAddDelTest( "logpath", From 578d9bed1bdb965a34c32a8a73f4471ff7e3c6b9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 27 Feb 2013 18:13:22 +0000 Subject: [PATCH 039/426] Added ability to set log file encoding with fail2ban-regex --- fail2ban-regex | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 8bd2ff63..20a67975 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -22,7 +22,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" __license__ = "GPL" -import getopt, sys, time, logging, os +import getopt, sys, time, logging, os, locale # Inserts our own modules path first in the list # fix for bug #343821 @@ -77,6 +77,7 @@ class Fail2banRegex: self.__ignoreregex = list() self.__failregex = list() self.__verbose = False + self.encoding = locale.getpreferredencoding() # Setup logging logging.getLogger("fail2ban").handlers = [] self.__hdlr = logging.StreamHandler(Fail2banRegex.test) @@ -110,6 +111,7 @@ class Fail2banRegex: print "This tools can test regular expressions for \"fail2ban\"." print print "Options:" + print " -e, --encoding set the file encoding" print " -h, --help display this help message" print " -V, --version print the version" print " -v, --verbose verbose output" @@ -141,6 +143,8 @@ class Fail2banRegex: sys.exit(0) elif opt[0] in ["-v", "--verbose"]: self.__verbose = True + elif opt[0] in ["-e", "--encoding"]: + self.encoding = opt[1] #@staticmethod def logIsFile(value): @@ -318,8 +322,8 @@ if __name__ == "__main__": fail2banRegex = Fail2banRegex() # Reads the command line options. try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] + cmdOpts = 'e:hVcv' + cmdLongOpts = ['encoding=', 'help', 'version', 'verbose'] optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: fail2banRegex.dispUsage() @@ -346,13 +350,15 @@ if __name__ == "__main__": if fail2banRegex.logIsFile(cmd_log): try: - if sys.version_info >= (3,): - hdlr = open(cmd_log, encoding='utf-8', errors='ignore') - else: - hdlr = open(cmd_log) + hdlr = open(cmd_log, 'rb') print "Use log file : " + cmd_log print for line in hdlr: + try: + line = line.decode(fail2banRegex.encoding, 'strict') + except UnicodeDecodeError: + if sys.version_info >= (3,): # Python 3 must be decoded + line = line.decode(fail2banRegex.encoding, 'ignore') fail2banRegex.testIgnoreRegex(line) fail2banRegex.testRegex(line) except IOError, e: From a4a24048d4ab49620a6f98c168939272650471a7 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 27 Feb 2013 18:31:47 +0000 Subject: [PATCH 040/426] Minor tweaks to fail2ban-regex for encoding --- fail2ban-regex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index 20a67975..f9746ce4 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -111,7 +111,8 @@ class Fail2banRegex: print "This tools can test regular expressions for \"fail2ban\"." print print "Options:" - print " -e, --encoding set the file encoding" + print " -e ENCODING, --encoding=ENCODING" + print " set the file encoding. default:system locale" print " -h, --help display this help message" print " -V, --version print the version" print " -v, --verbose verbose output" @@ -352,6 +353,7 @@ if __name__ == "__main__": try: hdlr = open(cmd_log, 'rb') print "Use log file : " + cmd_log + print "Use encoding : " + fail2banRegex.encoding print for line in hdlr: try: From 8ddc9de928a812665ef774ffa1c7713d5f9ba17d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Mar 2013 19:04:58 +0000 Subject: [PATCH 041/426] BF: Handle expected errors for python3.{0,1} when changing log target --- server/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/server.py b/server/server.py index 0db03d09..de837b86 100644 --- a/server/server.py +++ b/server/server.py @@ -383,7 +383,8 @@ class Server: handler.flush() handler.close() except ValueError: - if sys.version_info >= (2,6): + if (2,6) <= sys.version_info < (3,) or \ + (3,2) <= sys.version_info: raise # is known to be thrown after logging was shutdown once # with older Pythons -- seems to be safe to ignore there From 21888dfe69ef03d1efe1e453ea72c1b9229e8848 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 23 Mar 2013 19:31:29 +0000 Subject: [PATCH 042/426] ENH: Add python3 versions to Travis CI config --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index e655e7c1..b309255b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,13 @@ python: - "2.5" - "2.6" - "2.7" + - "3.0" + - "3.1" + - "3.2" + - "3.3" install: - "pip install pyinotify" +before_script: + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi script: - python ./fail2ban-testcases From d30f6a2d66201e8d37d04fbc78262d02b84753a0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 30 Mar 2013 21:40:42 +0000 Subject: [PATCH 043/426] add fail2ban-2to3 to MANIFEST file --- MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST b/MANIFEST index eef145b6..785edd5e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -7,6 +7,7 @@ fail2ban-client fail2ban-server fail2ban-testcases fail2ban-regex +fail2ban-2to3 client/configreader.py client/configparserinc.py client/jailreader.py From 5acd035f7224f2dfa564916d969619dcba04884f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 30 Mar 2013 21:51:22 +0000 Subject: [PATCH 044/426] TST: Remove Travis CI unsupported versions of python from Travis config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b309255b..b68b6c3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ python: - "2.5" - "2.6" - "2.7" - - "3.0" - - "3.1" - "3.2" - "3.3" install: From e43fcc80dbfb1c9b70ccbb200b691894b22328ca Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 30 Mar 2013 18:30:23 -0400 Subject: [PATCH 045/426] BF: setBaseDir is not static method now -- so set it for the filterReader in question --- testcases/clientreadertestcase.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 21dd4745..8eccb2b9 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -113,13 +113,6 @@ class JailReaderTest(unittest.TestCase): class FilterReaderTest(unittest.TestCase): - def setUp(self): - """Call before every test case.""" - ConfigReader.setBaseDir("testcases/files/") - - def tearDown(self): - """Call after every test case.""" - def testConvert(self): output = [['set', 'testcase01', 'addfailregex', "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" @@ -141,6 +134,7 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"]] filterReader = FilterReader("testcase01", "testcase01") + filterReader.setBaseDir("testcases/files/") filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) From dba88e842f3583ce7f213d06892db7f68d890209 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 31 Mar 2013 18:18:21 +0100 Subject: [PATCH 046/426] ENH+BF+TST+DOC: Make fail2ban a python module --- DEVELOP | 4 +- MANIFEST | 76 +++++++++---------- fail2ban-client | 18 ++--- fail2ban-regex | 17 ++--- fail2ban-server | 11 +-- fail2ban-testcases | 10 +-- {client => fail2ban}/__init__.py | 0 {common => fail2ban/client}/__init__.py | 0 {client => fail2ban/client}/actionreader.py | 0 {client => fail2ban/client}/beautifier.py | 2 +- .../client}/configparserinc.py | 0 {client => fail2ban/client}/configreader.py | 0 {client => fail2ban/client}/configurator.py | 0 {client => fail2ban/client}/csocket.py | 0 {client => fail2ban/client}/fail2banreader.py | 0 {client => fail2ban/client}/filterreader.py | 0 {client => fail2ban/client}/jailreader.py | 0 {client => fail2ban/client}/jailsreader.py | 0 {common => fail2ban}/exceptions.py | 0 {common => fail2ban}/helpers.py | 0 {common => fail2ban}/protocol.py | 0 {server => fail2ban/server}/__init__.py | 0 {server => fail2ban/server}/action.py | 0 {server => fail2ban/server}/actions.py | 0 {server => fail2ban/server}/asyncserver.py | 3 +- {server => fail2ban/server}/banmanager.py | 0 {server => fail2ban/server}/datedetector.py | 0 {server => fail2ban/server}/datetemplate.py | 0 {server => fail2ban/server}/faildata.py | 0 {server => fail2ban/server}/failmanager.py | 0 {server => fail2ban/server}/failregex.py | 0 {server => fail2ban/server}/filter.py | 0 {server => fail2ban/server}/filtergamin.py | 0 {server => fail2ban/server}/filterpoll.py | 0 .../server}/filterpyinotify.py | 0 {server => fail2ban/server}/iso8601.py | 0 {server => fail2ban/server}/jail.py | 0 {server => fail2ban/server}/jails.py | 2 +- {server => fail2ban/server}/jailthread.py | 0 {server => fail2ban/server}/mytime.py | 0 {server => fail2ban/server}/server.py | 2 +- {server => fail2ban/server}/ticket.py | 0 {server => fail2ban/server}/transmitter.py | 0 {common => fail2ban}/version.py | 0 setup.cfg | 3 - setup.py | 15 ++-- testcases/actiontestcase.py | 3 +- testcases/banmanagertestcase.py | 5 +- testcases/clientreadertestcase.py | 9 ++- testcases/datedetectortestcase.py | 5 +- testcases/failmanagertestcase.py | 5 +- testcases/filtertestcase.py | 10 +-- testcases/servertestcase.py | 5 +- testcases/sockettestcase.py | 5 +- 54 files changed, 97 insertions(+), 113 deletions(-) rename {client => fail2ban}/__init__.py (100%) rename {common => fail2ban/client}/__init__.py (100%) rename {client => fail2ban/client}/actionreader.py (100%) rename {client => fail2ban/client}/beautifier.py (98%) rename {client => fail2ban/client}/configparserinc.py (100%) rename {client => fail2ban/client}/configreader.py (100%) rename {client => fail2ban/client}/configurator.py (100%) rename {client => fail2ban/client}/csocket.py (100%) rename {client => fail2ban/client}/fail2banreader.py (100%) rename {client => fail2ban/client}/filterreader.py (100%) rename {client => fail2ban/client}/jailreader.py (100%) rename {client => fail2ban/client}/jailsreader.py (100%) rename {common => fail2ban}/exceptions.py (100%) rename {common => fail2ban}/helpers.py (100%) rename {common => fail2ban}/protocol.py (100%) rename {server => fail2ban/server}/__init__.py (100%) rename {server => fail2ban/server}/action.py (100%) rename {server => fail2ban/server}/actions.py (100%) rename {server => fail2ban/server}/asyncserver.py (99%) rename {server => fail2ban/server}/banmanager.py (100%) rename {server => fail2ban/server}/datedetector.py (100%) rename {server => fail2ban/server}/datetemplate.py (100%) rename {server => fail2ban/server}/faildata.py (100%) rename {server => fail2ban/server}/failmanager.py (100%) rename {server => fail2ban/server}/failregex.py (100%) rename {server => fail2ban/server}/filter.py (100%) rename {server => fail2ban/server}/filtergamin.py (100%) rename {server => fail2ban/server}/filterpoll.py (100%) rename {server => fail2ban/server}/filterpyinotify.py (100%) rename {server => fail2ban/server}/iso8601.py (100%) rename {server => fail2ban/server}/jail.py (100%) rename {server => fail2ban/server}/jails.py (98%) rename {server => fail2ban/server}/jailthread.py (100%) rename {server => fail2ban/server}/mytime.py (100%) rename {server => fail2ban/server}/server.py (99%) rename {server => fail2ban/server}/ticket.py (100%) rename {server => fail2ban/server}/transmitter.py (100%) rename {common => fail2ban}/version.py (100%) diff --git a/DEVELOP b/DEVELOP index 623aee12..158fb2b2 100644 --- a/DEVELOP +++ b/DEVELOP @@ -249,7 +249,7 @@ Takes care about executing start/check/ban/unban/stop commands Releasing ========= -# Ensure the version is correct in ./common/version.py +# Ensure the version is correct in ./fail2ban/version.py # Add/finalize the corresponding entry in the ChangeLog @@ -271,7 +271,7 @@ Releasing # Run the following and update the wiki with output: - python -c 'import common.protocol; common.protocol.printWiki()' + python -c 'import fail2ban.protocol; fail2ban.protocol.printWiki()' # Email users and development list of release diff --git a/MANIFEST b/MANIFEST index 28063b83..fc56356a 100644 --- a/MANIFEST +++ b/MANIFEST @@ -9,39 +9,39 @@ fail2ban-client fail2ban-server fail2ban-testcases fail2ban-regex -client/configreader.py -client/configparserinc.py -client/jailreader.py -client/fail2banreader.py -client/jailsreader.py -client/beautifier.py -client/filterreader.py -client/actionreader.py -client/__init__.py -client/configurator.py -client/csocket.py -server/asyncserver.py -server/filter.py -server/filterpyinotify.py -server/filtergamin.py -server/filterpoll.py -server/iso8601.py -server/server.py -server/actions.py -server/faildata.py -server/failmanager.py -server/datedetector.py -server/jailthread.py -server/transmitter.py -server/action.py -server/ticket.py -server/jail.py -server/jails.py -server/__init__.py -server/banmanager.py -server/datetemplate.py -server/mytime.py -server/failregex.py +fail2ban/client/configreader.py +fail2ban/client/configparserinc.py +fail2ban/client/jailreader.py +fail2ban/client/fail2banreader.py +fail2ban/client/jailsreader.py +fail2ban/client/beautifier.py +fail2ban/client/filterreader.py +fail2ban/client/actionreader.py +fail2ban/client/__init__.py +fail2ban/client/configurator.py +fail2ban/client/csocket.py +fail2ban/server/asyncserver.py +fail2ban/server/filter.py +fail2ban/server/filterpyinotify.py +fail2ban/server/filtergamin.py +fail2ban/server/filterpoll.py +fail2ban/server/iso8601.py +fail2ban/server/server.py +fail2ban/server/actions.py +fail2ban/server/faildata.py +fail2ban/server/failmanager.py +fail2ban/server/datedetector.py +fail2ban/server/jailthread.py +fail2ban/server/transmitter.py +fail2ban/server/action.py +fail2ban/server/ticket.py +fail2ban/server/jail.py +fail2ban/server/jails.py +fail2ban/server/__init__.py +fail2ban/server/banmanager.py +fail2ban/server/datetemplate.py +fail2ban/server/mytime.py +fail2ban/server/failregex.py testcases/files/testcase-usedns.log testcases/banmanagertestcase.py testcases/failmanagertestcase.py @@ -58,11 +58,11 @@ testcases/files/testcase03.log testcases/files/testcase04.log setup.py setup.cfg -common/__init__.py -common/exceptions.py -common/helpers.py -common/version.py -common/protocol.py +fail2ban/__init__.py +fail2ban/exceptions.py +fail2ban/helpers.py +fail2ban/version.py +fail2ban/protocol.py config/jail.conf config/filter.d/common.conf config/filter.d/apache-auth.conf diff --git a/fail2ban-client b/fail2ban-client index d8147f02..8068d60f 100755 --- a/fail2ban-client +++ b/fail2ban-client @@ -25,19 +25,11 @@ __license__ = "GPL" import sys, string, os, pickle, re, logging, signal import getopt, time, shlex, socket -# Inserts our own modules path first in the list -# fix for bug #343821 -try: - from common.version import version -except ImportError, e: - sys.path.insert(1, "/usr/share/fail2ban") - from common.version import version - -# Now we can import the rest of modules -from common.protocol import printFormatted -from client.csocket import CSocket -from client.configurator import Configurator -from client.beautifier import Beautifier +from fail2ban.version import version +from fail2ban.protocol import printFormatted +from fail2ban.client.csocket import CSocket +from fail2ban.client.configurator import Configurator +from fail2ban.client.beautifier import Beautifier # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.client") diff --git a/fail2ban-regex b/fail2ban-regex index a0a90b05..6bff21de 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -23,19 +23,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" __license__ = "GPL" import getopt, sys, time, logging, os - -# Inserts our own modules path first in the list -# fix for bug #343821 -try: - from common.version import version -except ImportError, e: - sys.path.insert(1, "/usr/share/fail2ban") - from common.version import version - -from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError -from server.filter import Filter -from server.failregex import RegexException + +from fail2ban.version import version +from fail2ban.client.configparserinc import SafeConfigParserWithIncludes +from fail2ban.server.filter import Filter +from fail2ban.server.failregex import RegexException # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.regex") diff --git a/fail2ban-server b/fail2ban-server index 404a1ced..3a1686d3 100755 --- a/fail2ban-server +++ b/fail2ban-server @@ -24,15 +24,8 @@ __license__ = "GPL" import getopt, sys, logging, os -# Inserts our own modules path first in the list -# fix for bug #343821 -try: - from common.version import version -except ImportError, e: - sys.path.insert(1, "/usr/share/fail2ban") - from common.version import version - -from server.server import Server +from fail2ban.version import version +from fail2ban.server.server import Server # Gets the instance of the logger. logSys = logging.getLogger("fail2ban") diff --git a/fail2ban-testcases b/fail2ban-testcases index e00cc908..c10856ec 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -27,7 +27,7 @@ __license__ = "GPL" import unittest, logging, sys, time, os -from common.version import version +from fail2ban.version import version from testcases import banmanagertestcase from testcases import clientreadertestcase from testcases import failmanagertestcase @@ -38,7 +38,7 @@ from testcases import actiontestcase from testcases import sockettestcase from testcases.utils import FormatterWithTraceBack -from server.mytime import MyTime +from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option @@ -168,20 +168,20 @@ tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) # Extensive use-tests of different available filters backends # -from server.filterpoll import FilterPoll +from fail2ban.server.filterpoll import FilterPoll filters = [FilterPoll] # always available # Additional filters available only if external modules are available # yoh: Since I do not know better way for parametric tests # with good old unittest try: - from server.filtergamin import FilterGamin + from fail2ban.server.filtergamin import FilterGamin filters.append(FilterGamin) except Exception, e: # pragma: no cover print "I: Skipping gamin backend testing. Got exception '%s'" % e try: - from server.filterpyinotify import FilterPyinotify + from fail2ban.server.filterpyinotify import FilterPyinotify filters.append(FilterPyinotify) except Exception, e: # pragma: no cover print "I: Skipping pyinotify backend testing. Got exception '%s'" % e diff --git a/client/__init__.py b/fail2ban/__init__.py similarity index 100% rename from client/__init__.py rename to fail2ban/__init__.py diff --git a/common/__init__.py b/fail2ban/client/__init__.py similarity index 100% rename from common/__init__.py rename to fail2ban/client/__init__.py diff --git a/client/actionreader.py b/fail2ban/client/actionreader.py similarity index 100% rename from client/actionreader.py rename to fail2ban/client/actionreader.py diff --git a/client/beautifier.py b/fail2ban/client/beautifier.py similarity index 98% rename from client/beautifier.py rename to fail2ban/client/beautifier.py index 7e48016c..1403bb08 100644 --- a/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -23,7 +23,7 @@ __license__ = "GPL" import logging -from common.exceptions import UnknownJailException, DuplicateJailException +from fail2ban.exceptions import UnknownJailException, DuplicateJailException # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.client.config") diff --git a/client/configparserinc.py b/fail2ban/client/configparserinc.py similarity index 100% rename from client/configparserinc.py rename to fail2ban/client/configparserinc.py diff --git a/client/configreader.py b/fail2ban/client/configreader.py similarity index 100% rename from client/configreader.py rename to fail2ban/client/configreader.py diff --git a/client/configurator.py b/fail2ban/client/configurator.py similarity index 100% rename from client/configurator.py rename to fail2ban/client/configurator.py diff --git a/client/csocket.py b/fail2ban/client/csocket.py similarity index 100% rename from client/csocket.py rename to fail2ban/client/csocket.py diff --git a/client/fail2banreader.py b/fail2ban/client/fail2banreader.py similarity index 100% rename from client/fail2banreader.py rename to fail2ban/client/fail2banreader.py diff --git a/client/filterreader.py b/fail2ban/client/filterreader.py similarity index 100% rename from client/filterreader.py rename to fail2ban/client/filterreader.py diff --git a/client/jailreader.py b/fail2ban/client/jailreader.py similarity index 100% rename from client/jailreader.py rename to fail2ban/client/jailreader.py diff --git a/client/jailsreader.py b/fail2ban/client/jailsreader.py similarity index 100% rename from client/jailsreader.py rename to fail2ban/client/jailsreader.py diff --git a/common/exceptions.py b/fail2ban/exceptions.py similarity index 100% rename from common/exceptions.py rename to fail2ban/exceptions.py diff --git a/common/helpers.py b/fail2ban/helpers.py similarity index 100% rename from common/helpers.py rename to fail2ban/helpers.py diff --git a/common/protocol.py b/fail2ban/protocol.py similarity index 100% rename from common/protocol.py rename to fail2ban/protocol.py diff --git a/server/__init__.py b/fail2ban/server/__init__.py similarity index 100% rename from server/__init__.py rename to fail2ban/server/__init__.py diff --git a/server/action.py b/fail2ban/server/action.py similarity index 100% rename from server/action.py rename to fail2ban/server/action.py diff --git a/server/actions.py b/fail2ban/server/actions.py similarity index 100% rename from server/actions.py rename to fail2ban/server/actions.py diff --git a/server/asyncserver.py b/fail2ban/server/asyncserver.py similarity index 99% rename from server/asyncserver.py rename to fail2ban/server/asyncserver.py index 66b2b53f..d5fb791c 100644 --- a/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -28,9 +28,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" from pickle import dumps, loads, HIGHEST_PROTOCOL -from common import helpers import asyncore, asynchat, socket, os, logging, sys, traceback +from fail2ban import helpers + # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.server") diff --git a/server/banmanager.py b/fail2ban/server/banmanager.py similarity index 100% rename from server/banmanager.py rename to fail2ban/server/banmanager.py diff --git a/server/datedetector.py b/fail2ban/server/datedetector.py similarity index 100% rename from server/datedetector.py rename to fail2ban/server/datedetector.py diff --git a/server/datetemplate.py b/fail2ban/server/datetemplate.py similarity index 100% rename from server/datetemplate.py rename to fail2ban/server/datetemplate.py diff --git a/server/faildata.py b/fail2ban/server/faildata.py similarity index 100% rename from server/faildata.py rename to fail2ban/server/faildata.py diff --git a/server/failmanager.py b/fail2ban/server/failmanager.py similarity index 100% rename from server/failmanager.py rename to fail2ban/server/failmanager.py diff --git a/server/failregex.py b/fail2ban/server/failregex.py similarity index 100% rename from server/failregex.py rename to fail2ban/server/failregex.py diff --git a/server/filter.py b/fail2ban/server/filter.py similarity index 100% rename from server/filter.py rename to fail2ban/server/filter.py diff --git a/server/filtergamin.py b/fail2ban/server/filtergamin.py similarity index 100% rename from server/filtergamin.py rename to fail2ban/server/filtergamin.py diff --git a/server/filterpoll.py b/fail2ban/server/filterpoll.py similarity index 100% rename from server/filterpoll.py rename to fail2ban/server/filterpoll.py diff --git a/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py similarity index 100% rename from server/filterpyinotify.py rename to fail2ban/server/filterpyinotify.py diff --git a/server/iso8601.py b/fail2ban/server/iso8601.py similarity index 100% rename from server/iso8601.py rename to fail2ban/server/iso8601.py diff --git a/server/jail.py b/fail2ban/server/jail.py similarity index 100% rename from server/jail.py rename to fail2ban/server/jail.py diff --git a/server/jails.py b/fail2ban/server/jails.py similarity index 98% rename from server/jails.py rename to fail2ban/server/jails.py index 4bf5f971..7ea1dde0 100644 --- a/server/jails.py +++ b/fail2ban/server/jails.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" -from common.exceptions import DuplicateJailException, UnknownJailException +from fail2ban.exceptions import DuplicateJailException, UnknownJailException from jail import Jail from threading import Lock diff --git a/server/jailthread.py b/fail2ban/server/jailthread.py similarity index 100% rename from server/jailthread.py rename to fail2ban/server/jailthread.py diff --git a/server/mytime.py b/fail2ban/server/mytime.py similarity index 100% rename from server/mytime.py rename to fail2ban/server/mytime.py diff --git a/server/server.py b/fail2ban/server/server.py similarity index 99% rename from server/server.py rename to fail2ban/server/server.py index a0824f1d..e8696b36 100644 --- a/server/server.py +++ b/fail2ban/server/server.py @@ -32,7 +32,7 @@ from jails import Jails from transmitter import Transmitter from asyncserver import AsyncServer from asyncserver import AsyncServerException -from common import version +from fail2ban import version import logging, logging.handlers, sys, os, signal # Gets the instance of the logger. diff --git a/server/ticket.py b/fail2ban/server/ticket.py similarity index 100% rename from server/ticket.py rename to fail2ban/server/ticket.py diff --git a/server/transmitter.py b/fail2ban/server/transmitter.py similarity index 100% rename from server/transmitter.py rename to fail2ban/server/transmitter.py diff --git a/common/version.py b/fail2ban/version.py similarity index 100% rename from common/version.py rename to fail2ban/version.py diff --git a/setup.cfg b/setup.cfg index 74c22b25..bb016599 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[install] -install-purelib=/usr/share/fail2ban - [sdist] formats=bztar diff --git a/setup.py b/setup.py index 784999a2..004f2f84 100755 --- a/setup.py +++ b/setup.py @@ -23,11 +23,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" from distutils.core import setup -from common.version import version from os.path import isfile, join, isdir -from sys import argv +import sys from glob import glob +from fail2ban.version import version + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -38,7 +39,7 @@ commands.''' setup( name = "fail2ban", version = version, - description = "Ban IPs that make too many password failure", + description = "Ban IPs that make too many password failures", long_description = longdesc, author = "Cyril Jaquier", author_email = "cyril.jaquier@fail2ban.org", @@ -51,9 +52,9 @@ setup( 'fail2ban-regex' ], packages = [ - 'common', - 'client', - 'server' + 'fail2ban', + 'fail2ban.client', + 'fail2ban.server' ], data_files = [ ('/etc/fail2ban', @@ -117,7 +118,7 @@ if isdir("/usr/lib/fail2ban"): print # Update config file -if argv[1] == "install": +if sys.argv[1] == "install": print print "Please do not forget to update your configuration files." print "They are in /etc/fail2ban/." diff --git a/testcases/actiontestcase.py b/testcases/actiontestcase.py index b8292c27..e0ea3a9b 100644 --- a/testcases/actiontestcase.py +++ b/testcases/actiontestcase.py @@ -29,9 +29,10 @@ __license__ = "GPL" import unittest, time import logging, sys -from server.action import Action from StringIO import StringIO +from fail2ban.server.action import Action + class ExecuteAction(unittest.TestCase): def setUp(self): diff --git a/testcases/banmanagertestcase.py b/testcases/banmanagertestcase.py index 6d0ce55a..8d0f1929 100644 --- a/testcases/banmanagertestcase.py +++ b/testcases/banmanagertestcase.py @@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest -from server.banmanager import BanManager -from server.ticket import BanTicket + +from fail2ban.server.banmanager import BanManager +from fail2ban.server.ticket import BanTicket class AddFailure(unittest.TestCase): diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index fad16f04..f54df659 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -22,10 +22,11 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" import os, shutil, tempfile, unittest -from client.configreader import ConfigReader -from client.jailreader import JailReader -from client.jailsreader import JailsReader -from client.configurator import Configurator + +from fail2ban.client.configreader import ConfigReader +from fail2ban.client.jailreader import JailReader +from fail2ban.client.jailsreader import JailsReader +from fail2ban.client.configurator import Configurator class ConfigReaderTest(unittest.TestCase): diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index 64af1fab..e9cabca0 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest -from server.datedetector import DateDetector -from server.datetemplate import DateTemplate + +from fail2ban.server.datedetector import DateDetector +from fail2ban.server.datetemplate import DateTemplate class DateDetectorTest(unittest.TestCase): diff --git a/testcases/failmanagertestcase.py b/testcases/failmanagertestcase.py index ffee4ff1..7a714122 100644 --- a/testcases/failmanagertestcase.py +++ b/testcases/failmanagertestcase.py @@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest, socket, time, pickle -from server.failmanager import FailManager, FailManagerEmpty -from server.ticket import FailTicket + +from fail2ban.server.failmanager import FailManager, FailManagerEmpty +from fail2ban.server.ticket import FailTicket class AddFailure(unittest.TestCase): diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 927cb2fe..75b72c05 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -29,11 +29,11 @@ import sys import time import tempfile -from server.jail import Jail -from server.filterpoll import FilterPoll -from server.filter import FileFilter, DNSUtils -from server.failmanager import FailManager -from server.failmanager import FailManagerEmpty +from fail2ban.server.jail import Jail +from fail2ban.server.filterpoll import FilterPoll +from fail2ban.server.filter import FileFilter, DNSUtils +from fail2ban.server.failmanager import FailManager +from fail2ban.server.failmanager import FailManagerEmpty # # Useful helpers diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index ffb057a9..0cdf0422 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest, socket, time, tempfile, os -from server.server import Server -from common.exceptions import UnknownJailException + +from fail2ban.server.server import Server +from fail2ban.exceptions import UnknownJailException class StartStop(unittest.TestCase): diff --git a/testcases/sockettestcase.py b/testcases/sockettestcase.py index 4cd5a687..bbca8dde 100644 --- a/testcases/sockettestcase.py +++ b/testcases/sockettestcase.py @@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import unittest, time, tempfile, os, threading -from server.asyncserver import AsyncServer, AsyncServerException -from client.csocket import CSocket + +from fail2ban.server.asyncserver import AsyncServer, AsyncServerException +from fail2ban.client.csocket import CSocket class Socket(unittest.TestCase): From e53bfafd6afaa07b7049163a3363ab7cead789e9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 31 Mar 2013 19:36:52 +0100 Subject: [PATCH 047/426] TST: Update Travis CI coverage config for python module structure --- .travis_coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis_coveragerc b/.travis_coveragerc index 4d4b7ebd..3f6404ff 100644 --- a/.travis_coveragerc +++ b/.travis_coveragerc @@ -4,4 +4,4 @@ branch = True omit = /usr/* /home/travis/virtualenv/* - server/filtergamin.py + fail2ban/server/filtergamin.py From e3bd2042ebcdf891eef7d04632b22ce82e1eeffd Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 1 Apr 2013 11:22:40 +0100 Subject: [PATCH 048/426] TST+ENH: Move testcases to part of fail2ban module This allows fail2ban-testcases to be run on an installed fail2ban instance. TODO: Fix tests requiring config files --- MANIFEST | 29 ++++++++++--------- fail2ban-testcases | 18 ++++++------ {testcases => fail2ban/tests}/__init__.py | 0 .../tests}/actiontestcase.py | 0 .../tests}/banmanagertestcase.py | 0 .../tests}/clientreadertestcase.py | 0 .../tests}/datedetectortestcase.py | 0 .../tests}/failmanagertestcase.py | 0 .../tests}/files/logs/apache-overflows | 0 .../tests}/files/logs/asterisk | 0 .../tests}/files/logs/dovecot | 0 {testcases => fail2ban/tests}/files/logs/exim | 0 .../tests}/files/logs/lighttpd | 0 .../tests}/files/logs/named-refused | 0 .../tests}/files/logs/pam-generic | 0 .../tests}/files/logs/postfix | 0 .../tests}/files/logs/proftpd | 0 .../tests}/files/logs/pure-ftpd | 0 .../tests}/files/logs/roundcube-auth | 0 {testcases => fail2ban/tests}/files/logs/sasl | 0 .../tests}/files/logs/sogo-auth | 0 {testcases => fail2ban/tests}/files/logs/sshd | 0 .../tests}/files/logs/sshd-ddos | 0 .../tests}/files/logs/vsftpd | 0 .../tests}/files/logs/webmin-auth | 0 .../tests}/files/logs/wu-ftpd | 0 .../tests}/files/testcase-usedns.log | 0 .../tests}/files/testcase01.log | 0 .../tests}/files/testcase02.log | 0 .../tests}/files/testcase03.log | 0 .../tests}/files/testcase04.log | 0 .../tests}/filtertestcase.py | 14 +++++---- .../tests}/servertestcase.py | 10 ++++--- .../tests}/sockettestcase.py | 0 {testcases => fail2ban/tests}/utils.py | 0 setup.py | 9 ++++-- 36 files changed, 45 insertions(+), 35 deletions(-) rename {testcases => fail2ban/tests}/__init__.py (100%) rename {testcases => fail2ban/tests}/actiontestcase.py (100%) rename {testcases => fail2ban/tests}/banmanagertestcase.py (100%) rename {testcases => fail2ban/tests}/clientreadertestcase.py (100%) rename {testcases => fail2ban/tests}/datedetectortestcase.py (100%) rename {testcases => fail2ban/tests}/failmanagertestcase.py (100%) rename {testcases => fail2ban/tests}/files/logs/apache-overflows (100%) rename {testcases => fail2ban/tests}/files/logs/asterisk (100%) rename {testcases => fail2ban/tests}/files/logs/dovecot (100%) rename {testcases => fail2ban/tests}/files/logs/exim (100%) rename {testcases => fail2ban/tests}/files/logs/lighttpd (100%) rename {testcases => fail2ban/tests}/files/logs/named-refused (100%) rename {testcases => fail2ban/tests}/files/logs/pam-generic (100%) rename {testcases => fail2ban/tests}/files/logs/postfix (100%) rename {testcases => fail2ban/tests}/files/logs/proftpd (100%) rename {testcases => fail2ban/tests}/files/logs/pure-ftpd (100%) rename {testcases => fail2ban/tests}/files/logs/roundcube-auth (100%) rename {testcases => fail2ban/tests}/files/logs/sasl (100%) rename {testcases => fail2ban/tests}/files/logs/sogo-auth (100%) rename {testcases => fail2ban/tests}/files/logs/sshd (100%) rename {testcases => fail2ban/tests}/files/logs/sshd-ddos (100%) rename {testcases => fail2ban/tests}/files/logs/vsftpd (100%) rename {testcases => fail2ban/tests}/files/logs/webmin-auth (100%) rename {testcases => fail2ban/tests}/files/logs/wu-ftpd (100%) rename {testcases => fail2ban/tests}/files/testcase-usedns.log (100%) rename {testcases => fail2ban/tests}/files/testcase01.log (100%) rename {testcases => fail2ban/tests}/files/testcase02.log (100%) rename {testcases => fail2ban/tests}/files/testcase03.log (100%) rename {testcases => fail2ban/tests}/files/testcase04.log (100%) rename {testcases => fail2ban/tests}/filtertestcase.py (98%) rename {testcases => fail2ban/tests}/servertestcase.py (98%) rename {testcases => fail2ban/tests}/sockettestcase.py (100%) rename {testcases => fail2ban/tests}/utils.py (100%) diff --git a/MANIFEST b/MANIFEST index fc56356a..93698e95 100644 --- a/MANIFEST +++ b/MANIFEST @@ -42,20 +42,21 @@ fail2ban/server/banmanager.py fail2ban/server/datetemplate.py fail2ban/server/mytime.py fail2ban/server/failregex.py -testcases/files/testcase-usedns.log -testcases/banmanagertestcase.py -testcases/failmanagertestcase.py -testcases/clientreadertestcase.py -testcases/filtertestcase.py -testcases/__init__.py -testcases/datedetectortestcase.py -testcases/actiontestcase.py -testcases/servertestcase.py -testcases/sockettestcase.py -testcases/files/testcase01.log -testcases/files/testcase02.log -testcases/files/testcase03.log -testcases/files/testcase04.log +fail2ban/tests/banmanagertestcase.py +fail2ban/tests/failmanagertestcase.py +fail2ban/tests/clientreadertestcase.py +fail2ban/tests/filtertestcase.py +fail2ban/tests/__init__.py +fail2ban/tests/datedetectortestcase.py +fail2ban/tests/actiontestcase.py +fail2ban/tests/servertestcase.py +fail2ban/tests/sockettestcase.py +fail2ban/tests/utils.py +fail2ban/tests/files/testcase01.log +fail2ban/tests/files/testcase02.log +fail2ban/tests/files/testcase03.log +fail2ban/tests/files/testcase04.log +fail2ban/tests/files/testcase-usedns.log setup.py setup.cfg fail2ban/__init__.py diff --git a/fail2ban-testcases b/fail2ban-testcases index c10856ec..45ba03ad 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -28,16 +28,16 @@ __license__ = "GPL" import unittest, logging, sys, time, os from fail2ban.version import version -from testcases import banmanagertestcase -from testcases import clientreadertestcase -from testcases import failmanagertestcase -from testcases import filtertestcase -from testcases import servertestcase -from testcases import datedetectortestcase -from testcases import actiontestcase -from testcases import sockettestcase +from fail2ban.tests import banmanagertestcase +from fail2ban.tests import clientreadertestcase +from fail2ban.tests import failmanagertestcase +from fail2ban.tests import filtertestcase +from fail2ban.tests import servertestcase +from fail2ban.tests import datedetectortestcase +from fail2ban.tests import actiontestcase +from fail2ban.tests import sockettestcase -from testcases.utils import FormatterWithTraceBack +from fail2ban.tests.utils import FormatterWithTraceBack from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option diff --git a/testcases/__init__.py b/fail2ban/tests/__init__.py similarity index 100% rename from testcases/__init__.py rename to fail2ban/tests/__init__.py diff --git a/testcases/actiontestcase.py b/fail2ban/tests/actiontestcase.py similarity index 100% rename from testcases/actiontestcase.py rename to fail2ban/tests/actiontestcase.py diff --git a/testcases/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py similarity index 100% rename from testcases/banmanagertestcase.py rename to fail2ban/tests/banmanagertestcase.py diff --git a/testcases/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py similarity index 100% rename from testcases/clientreadertestcase.py rename to fail2ban/tests/clientreadertestcase.py diff --git a/testcases/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py similarity index 100% rename from testcases/datedetectortestcase.py rename to fail2ban/tests/datedetectortestcase.py diff --git a/testcases/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py similarity index 100% rename from testcases/failmanagertestcase.py rename to fail2ban/tests/failmanagertestcase.py diff --git a/testcases/files/logs/apache-overflows b/fail2ban/tests/files/logs/apache-overflows similarity index 100% rename from testcases/files/logs/apache-overflows rename to fail2ban/tests/files/logs/apache-overflows diff --git a/testcases/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk similarity index 100% rename from testcases/files/logs/asterisk rename to fail2ban/tests/files/logs/asterisk diff --git a/testcases/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot similarity index 100% rename from testcases/files/logs/dovecot rename to fail2ban/tests/files/logs/dovecot diff --git a/testcases/files/logs/exim b/fail2ban/tests/files/logs/exim similarity index 100% rename from testcases/files/logs/exim rename to fail2ban/tests/files/logs/exim diff --git a/testcases/files/logs/lighttpd b/fail2ban/tests/files/logs/lighttpd similarity index 100% rename from testcases/files/logs/lighttpd rename to fail2ban/tests/files/logs/lighttpd diff --git a/testcases/files/logs/named-refused b/fail2ban/tests/files/logs/named-refused similarity index 100% rename from testcases/files/logs/named-refused rename to fail2ban/tests/files/logs/named-refused diff --git a/testcases/files/logs/pam-generic b/fail2ban/tests/files/logs/pam-generic similarity index 100% rename from testcases/files/logs/pam-generic rename to fail2ban/tests/files/logs/pam-generic diff --git a/testcases/files/logs/postfix b/fail2ban/tests/files/logs/postfix similarity index 100% rename from testcases/files/logs/postfix rename to fail2ban/tests/files/logs/postfix diff --git a/testcases/files/logs/proftpd b/fail2ban/tests/files/logs/proftpd similarity index 100% rename from testcases/files/logs/proftpd rename to fail2ban/tests/files/logs/proftpd diff --git a/testcases/files/logs/pure-ftpd b/fail2ban/tests/files/logs/pure-ftpd similarity index 100% rename from testcases/files/logs/pure-ftpd rename to fail2ban/tests/files/logs/pure-ftpd diff --git a/testcases/files/logs/roundcube-auth b/fail2ban/tests/files/logs/roundcube-auth similarity index 100% rename from testcases/files/logs/roundcube-auth rename to fail2ban/tests/files/logs/roundcube-auth diff --git a/testcases/files/logs/sasl b/fail2ban/tests/files/logs/sasl similarity index 100% rename from testcases/files/logs/sasl rename to fail2ban/tests/files/logs/sasl diff --git a/testcases/files/logs/sogo-auth b/fail2ban/tests/files/logs/sogo-auth similarity index 100% rename from testcases/files/logs/sogo-auth rename to fail2ban/tests/files/logs/sogo-auth diff --git a/testcases/files/logs/sshd b/fail2ban/tests/files/logs/sshd similarity index 100% rename from testcases/files/logs/sshd rename to fail2ban/tests/files/logs/sshd diff --git a/testcases/files/logs/sshd-ddos b/fail2ban/tests/files/logs/sshd-ddos similarity index 100% rename from testcases/files/logs/sshd-ddos rename to fail2ban/tests/files/logs/sshd-ddos diff --git a/testcases/files/logs/vsftpd b/fail2ban/tests/files/logs/vsftpd similarity index 100% rename from testcases/files/logs/vsftpd rename to fail2ban/tests/files/logs/vsftpd diff --git a/testcases/files/logs/webmin-auth b/fail2ban/tests/files/logs/webmin-auth similarity index 100% rename from testcases/files/logs/webmin-auth rename to fail2ban/tests/files/logs/webmin-auth diff --git a/testcases/files/logs/wu-ftpd b/fail2ban/tests/files/logs/wu-ftpd similarity index 100% rename from testcases/files/logs/wu-ftpd rename to fail2ban/tests/files/logs/wu-ftpd diff --git a/testcases/files/testcase-usedns.log b/fail2ban/tests/files/testcase-usedns.log similarity index 100% rename from testcases/files/testcase-usedns.log rename to fail2ban/tests/files/testcase-usedns.log diff --git a/testcases/files/testcase01.log b/fail2ban/tests/files/testcase01.log similarity index 100% rename from testcases/files/testcase01.log rename to fail2ban/tests/files/testcase01.log diff --git a/testcases/files/testcase02.log b/fail2ban/tests/files/testcase02.log similarity index 100% rename from testcases/files/testcase02.log rename to fail2ban/tests/files/testcase02.log diff --git a/testcases/files/testcase03.log b/fail2ban/tests/files/testcase03.log similarity index 100% rename from testcases/files/testcase03.log rename to fail2ban/tests/files/testcase03.log diff --git a/testcases/files/testcase04.log b/fail2ban/tests/files/testcase04.log similarity index 100% rename from testcases/files/testcase04.log rename to fail2ban/tests/files/testcase04.log diff --git a/testcases/filtertestcase.py b/fail2ban/tests/filtertestcase.py similarity index 98% rename from testcases/filtertestcase.py rename to fail2ban/tests/filtertestcase.py index 75b72c05..27a1510d 100644 --- a/testcases/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -35,6 +35,8 @@ from fail2ban.server.filter import FileFilter, DNSUtils from fail2ban.server.failmanager import FailManager from fail2ban.server.failmanager import FailManagerEmpty +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + # # Useful helpers # @@ -182,7 +184,7 @@ class IgnoreIP(unittest.TestCase): class LogFile(unittest.TestCase): - FILENAME = "testcases/files/testcase01.log" + FILENAME = os.path.join(TEST_FILES_DIR, "testcase01.log") def setUp(self): """Call before every test case.""" @@ -522,11 +524,11 @@ def get_monitor_failures_testcase(Filter_): class GetFailures(unittest.TestCase): - FILENAME_01 = "testcases/files/testcase01.log" - FILENAME_02 = "testcases/files/testcase02.log" - FILENAME_03 = "testcases/files/testcase03.log" - FILENAME_04 = "testcases/files/testcase04.log" - FILENAME_USEDNS = "testcases/files/testcase-usedns.log" + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") + FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") + FILENAME_03 = os.path.join(TEST_FILES_DIR, "testcase03.log") + FILENAME_04 = os.path.join(TEST_FILES_DIR, "testcase04.log") + FILENAME_USEDNS = os.path.join(TEST_FILES_DIR, "testcase-usedns.log") # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, diff --git a/testcases/servertestcase.py b/fail2ban/tests/servertestcase.py similarity index 98% rename from testcases/servertestcase.py rename to fail2ban/tests/servertestcase.py index 0cdf0422..4a3435b4 100644 --- a/testcases/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -32,6 +32,8 @@ import unittest, socket, time, tempfile, os from fail2ban.server.server import Server from fail2ban.exceptions import UnknownJailException +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + class StartStop(unittest.TestCase): def setUp(self): @@ -273,14 +275,14 @@ class Transmitter(TransmitterBase): self.jailAddDelTest( "logpath", [ - "testcases/files/testcase01.log", - "testcases/files/testcase02.log", - "testcases/files/testcase03.log", + os.path.join(TEST_FILES_DIR, "testcase01.log"), + os.path.join(TEST_FILES_DIR, "testcase02.log"), + os.path.join(TEST_FILES_DIR, "testcase03.log"), ], self.jailName ) # Try duplicates - value = "testcases/files/testcase04.log" + value = os.path.join(TEST_FILES_DIR, "testcase04.log") self.assertEqual( self.transm.proceed(["set", self.jailName, "addlogpath", value]), (0, [value])) diff --git a/testcases/sockettestcase.py b/fail2ban/tests/sockettestcase.py similarity index 100% rename from testcases/sockettestcase.py rename to fail2ban/tests/sockettestcase.py diff --git a/testcases/utils.py b/fail2ban/tests/utils.py similarity index 100% rename from testcases/utils.py rename to fail2ban/tests/utils.py diff --git a/setup.py b/setup.py index 004f2f84..87159635 100755 --- a/setup.py +++ b/setup.py @@ -49,13 +49,18 @@ setup( scripts = [ 'fail2ban-client', 'fail2ban-server', - 'fail2ban-regex' + 'fail2ban-regex', + 'fail2ban-testcases', ], packages = [ 'fail2ban', 'fail2ban.client', - 'fail2ban.server' + 'fail2ban.server', + 'fail2ban.tests', ], + package_data = { + 'fail2ban.tests': ['files/*.log'], + }, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From a153653a2709dc032d814c9bacdbde50b1d0a171 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 1 Apr 2013 19:06:13 +0100 Subject: [PATCH 049/426] ENH+TST: Move fail2ban-* scripts to bin/ --- .travis.yml | 2 +- DEVELOP | 8 ++++---- MANIFEST | 8 ++++---- fail2ban-client => bin/fail2ban-client | 0 fail2ban-regex => bin/fail2ban-regex | 0 fail2ban-server => bin/fail2ban-server | 0 fail2ban-testcases => bin/fail2ban-testcases | 5 +++++ fail2ban-testcases-all | 2 +- setup.py | 8 ++++---- 9 files changed, 19 insertions(+), 14 deletions(-) rename fail2ban-client => bin/fail2ban-client (100%) rename fail2ban-regex => bin/fail2ban-regex (100%) rename fail2ban-server => bin/fail2ban-server (100%) rename fail2ban-testcases => bin/fail2ban-testcases (97%) diff --git a/.travis.yml b/.travis.yml index 773c5f0b..93082658 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,6 @@ install: - pip install pyinotify - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then pip install -q coveralls; fi script: - - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi after_script: - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coveralls; fi diff --git a/DEVELOP b/DEVELOP index 158fb2b2..0fe80596 100644 --- a/DEVELOP +++ b/DEVELOP @@ -24,9 +24,9 @@ Request feature. You can find more details on the Fail2Ban wiki Testing ======= -Existing tests can be run by executing `fail2ban-testcases`. This has options -like --log-level that will probably be useful. `fail2ban-testcases --help` for -full options. +Existing tests can be run by executing `bin/fail2ban-testcases`. This has +options like --log-level that will probably be useful. +`bin/fail2ban-testcases --help` forfull options. Test cases should cover all usual cases, all exception cases and all inside / outside boundary conditions. @@ -39,7 +39,7 @@ Install the package python-coverage to visualise your test coverage. Run the following (note: on Debian-based systems, the script is called `python-coverage`): -coverage run fail2ban-testcases +coverage run bin/fail2ban-testcases coverage html Then look at htmlcov/index.html and see how much coverage your test cases diff --git a/MANIFEST b/MANIFEST index 93698e95..12ba3435 100644 --- a/MANIFEST +++ b/MANIFEST @@ -5,10 +5,10 @@ THANKS COPYING DEVELOP doc/run-rootless.txt -fail2ban-client -fail2ban-server -fail2ban-testcases -fail2ban-regex +bin/fail2ban-client +bin/fail2ban-server +bin/fail2ban-testcases +bin/fail2ban-regex fail2ban/client/configreader.py fail2ban/client/configparserinc.py fail2ban/client/jailreader.py diff --git a/fail2ban-client b/bin/fail2ban-client similarity index 100% rename from fail2ban-client rename to bin/fail2ban-client diff --git a/fail2ban-regex b/bin/fail2ban-regex similarity index 100% rename from fail2ban-regex rename to bin/fail2ban-regex diff --git a/fail2ban-server b/bin/fail2ban-server similarity index 100% rename from fail2ban-server rename to bin/fail2ban-server diff --git a/fail2ban-testcases b/bin/fail2ban-testcases similarity index 97% rename from fail2ban-testcases rename to bin/fail2ban-testcases index 45ba03ad..5faaa75e 100755 --- a/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -27,6 +27,11 @@ __license__ = "GPL" import unittest, logging, sys, time, os +# Check if local fail2ban module exists, and use if it exists by +# modifying the path. This is such that tests can be used in dev +# environment. +if os.path.exists("fail2ban/__init__.py"): + sys.path.insert(0, ".") from fail2ban.version import version from fail2ban.tests import banmanagertestcase from fail2ban.tests import clientreadertestcase diff --git a/fail2ban-testcases-all b/fail2ban-testcases-all index fd33dce4..6b399337 100755 --- a/fail2ban-testcases-all +++ b/fail2ban-testcases-all @@ -9,7 +9,7 @@ for python in /usr/{,local/}bin/python2.[0-9]{,.*}{,-dbg} do [ -e "$python" ] || continue echo "Testing using $python" - $python ./fail2ban-testcases "$@" || failed+=" $python" + $python bin/fail2ban-testcases "$@" || failed+=" $python" done if [ ! -z "$failed" ]; then diff --git a/setup.py b/setup.py index 87159635..43658b47 100755 --- a/setup.py +++ b/setup.py @@ -47,10 +47,10 @@ setup( license = "GPL", platforms = "Posix", scripts = [ - 'fail2ban-client', - 'fail2ban-server', - 'fail2ban-regex', - 'fail2ban-testcases', + 'bin/fail2ban-client', + 'bin/fail2ban-server', + 'bin/fail2ban-regex', + 'bin/fail2ban-testcases', ], packages = [ 'fail2ban', From 0ce046ec477447d34c3721b953dc498835da681c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 1 Apr 2013 19:06:58 +0100 Subject: [PATCH 050/426] TST: clientreader test now use /etc/fail2ban/ if no local config/ --- fail2ban/tests/clientreadertestcase.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index f54df659..8bc462d4 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -28,6 +28,11 @@ from fail2ban.client.jailreader import JailReader from fail2ban.client.jailsreader import JailsReader from fail2ban.client.configurator import Configurator +if os.path.exists('config/fail2ban.conf'): + CONFIG_DIR='config' +else: + CONFIG_DIR='/etc/fail2ban' + class ConfigReaderTest(unittest.TestCase): def setUp(self): @@ -99,7 +104,7 @@ option = %s class JailReaderTest(unittest.TestCase): def testStockSSHJail(self): - jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm + jail = JailReader('ssh-iptables', basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -119,7 +124,7 @@ class JailsReaderTest(unittest.TestCase): self.assertRaises(ValueError, reader.read) def testReadStockJailConf(self): - jails = JailsReader(basedir='config') # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert() @@ -130,7 +135,7 @@ class JailsReaderTest(unittest.TestCase): def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises # on users' systems when enabling shipped jails - jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm + jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) # we are running tests from root project dir atm self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.getOptions()) # reads fine comm_commands = jails.convert() @@ -152,8 +157,8 @@ class JailsReaderTest(unittest.TestCase): def testConfigurator(self): configurator = Configurator() - configurator.setBaseDir('config') - self.assertEqual(configurator.getBaseDir(), 'config') + configurator.setBaseDir(CONFIG_DIR) + self.assertEqual(configurator.getBaseDir(), CONFIG_DIR) configurator.readEarly() opts = configurator.getEarlyOptions() @@ -166,4 +171,4 @@ class JailsReaderTest(unittest.TestCase): # otherwise just a code smoke test) configurator._Configurator__jails.setBaseDir('/tmp') self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp') - self.assertEqual(configurator.getBaseDir(), 'config') + self.assertEqual(configurator.getBaseDir(), CONFIG_DIR) From c4bdc48edbbf7641ec0385e0796e31c74c86820b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 6 Apr 2013 10:15:43 +0100 Subject: [PATCH 051/426] TST: Fix up tests from multiline elements broken in previous merge --- fail2ban/tests/clientreadertestcase.py | 3 ++- .../tests}/files/filter.d/testcase-common.conf | 0 {testcases => fail2ban/tests}/files/filter.d/testcase01.conf | 0 {testcases => fail2ban/tests}/files/testcase-multiline.log | 0 4 files changed, 2 insertions(+), 1 deletion(-) rename {testcases => fail2ban/tests}/files/filter.d/testcase-common.conf (100%) rename {testcases => fail2ban/tests}/files/filter.d/testcase01.conf (100%) rename {testcases => fail2ban/tests}/files/testcase-multiline.log (100%) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 340c9c15..99d9a50e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -29,6 +29,7 @@ from fail2ban.client.filterreader import FilterReader from fail2ban.client.jailsreader import JailsReader from fail2ban.client.configurator import Configurator +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") if os.path.exists('config/fail2ban.conf'): CONFIG_DIR='config' else: @@ -140,7 +141,7 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"]] filterReader = FilterReader("testcase01", "testcase01") - filterReader.setBaseDir("testcases/files/") + filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) diff --git a/testcases/files/filter.d/testcase-common.conf b/fail2ban/tests/files/filter.d/testcase-common.conf similarity index 100% rename from testcases/files/filter.d/testcase-common.conf rename to fail2ban/tests/files/filter.d/testcase-common.conf diff --git a/testcases/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf similarity index 100% rename from testcases/files/filter.d/testcase01.conf rename to fail2ban/tests/files/filter.d/testcase01.conf diff --git a/testcases/files/testcase-multiline.log b/fail2ban/tests/files/testcase-multiline.log similarity index 100% rename from testcases/files/testcase-multiline.log rename to fail2ban/tests/files/testcase-multiline.log From 3a16ceed0aa937c9ceeedff77be0b8b33e2d3d34 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 6 Apr 2013 10:20:53 +0100 Subject: [PATCH 052/426] BF: Added test filter.d files to setup.py package data --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43658b47..4fea8c29 100755 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ setup( 'fail2ban.tests', ], package_data = { - 'fail2ban.tests': ['files/*.log'], + 'fail2ban.tests': + ['files/*.log', 'files/filter.d/*.conf'], }, data_files = [ ('/etc/fail2ban', From a33bf5baca961a2b13d330fb905de11802fbc50b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 9 Apr 2013 19:40:54 +0100 Subject: [PATCH 053/426] ENH: setup.py now automatically runs 2to3 for python3.x --- setup.py | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index 784999a2..8aca6852 100755 --- a/setup.py +++ b/setup.py @@ -23,9 +23,18 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" from distutils.core import setup +try: + # python 3.x + from distutils.command.build_py import build_py_2to3 as build_py + from distutils.command.build_scripts \ + import build_scripts_2to3 as build_scripts +except ImportError: + # python 2.x + from distutils.command.build_py import build_py + from distutils.command.build_scripts import build_scripts from common.version import version from os.path import isfile, join, isdir -from sys import argv +import sys from glob import glob longdesc = ''' @@ -45,6 +54,7 @@ setup( url = "http://www.fail2ban.org", license = "GPL", platforms = "Posix", + cmdclass = {'build_py': build_py, 'build_scripts': build_scripts}, scripts = [ 'fail2ban-client', 'fail2ban-server', @@ -100,25 +110,26 @@ for directory in elements: obsoleteFiles.append(path) if obsoleteFiles: - print - print "Obsolete files from previous Fail2Ban versions were found on " \ - "your system." - print "Please delete them:" - print + sys.stdout.write("\n") + sys.stdout.write("Obsolete files from previous Fail2Ban versions " \ + "were found on your system.\n") + sys.stdout.write("Please delete them:\n") + sys.stdout.write("\n") for f in obsoleteFiles: - print "\t" + f - print + sys.stdout.write("\t" + f) + sys.stdout.write("\n") if isdir("/usr/lib/fail2ban"): - print - print "Fail2ban is not installed under /usr/lib anymore. The new " \ - "location is under /usr/share. Please remove the directory " \ - "/usr/lib/fail2ban and everything under this directory." - print + sys.stdout.write("\n") + sys.stdout.write("Fail2ban is not installed under /usr/lib anymore. " \ + "The new location is under /usr/share. Please remove the " \ + "directory /usr/lib/fail2ban and everything under this directory.\n") + sys.stdout.write("\n") # Update config file -if argv[1] == "install": - print - print "Please do not forget to update your configuration files." - print "They are in /etc/fail2ban/." - print +if sys.argv[1] == "install": + sys.stdout.write("\n") + sys.stdout.write("Please do not forget to update your configuration " + "files.\n") + sys.stdout.write("They are in /etc/fail2ban/.\n") + sys.stdout.write("\n") From 7a385fd442d540fe614ba886fda5395cf81019c0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Apr 2013 19:46:24 -0400 Subject: [PATCH 054/426] BF: Move mysqld.log into a new location under fail2ban module --- {testcases => fail2ban/tests}/files/logs/mysqld.log | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {testcases => fail2ban/tests}/files/logs/mysqld.log (100%) diff --git a/testcases/files/logs/mysqld.log b/fail2ban/tests/files/logs/mysqld.log similarity index 100% rename from testcases/files/logs/mysqld.log rename to fail2ban/tests/files/logs/mysqld.log From fe1c3fbdd9d0cc98fddec87f36dfb202f1807907 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Apr 2013 20:24:54 -0400 Subject: [PATCH 055/426] BF: fixing incorrect merge conflict -- run coverage only for 2.7 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a92df23a..f685ee38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,6 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi after_script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi From a3d82e2ab9174d088f883120a61b967fe0543bf7 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 10 Apr 2013 21:33:55 +0100 Subject: [PATCH 056/426] ENH: fail2ban logging uses __name__ for logger names --- bin/fail2ban-testcases | 2 +- fail2ban/client/actionreader.py | 2 +- fail2ban/client/beautifier.py | 2 +- fail2ban/client/configparserinc.py | 2 +- fail2ban/client/configreader.py | 2 +- fail2ban/client/configurator.py | 2 +- fail2ban/client/fail2banreader.py | 2 +- fail2ban/client/filterreader.py | 2 +- fail2ban/client/jailreader.py | 2 +- fail2ban/client/jailsreader.py | 2 +- fail2ban/server/action.py | 2 +- fail2ban/server/actions.py | 2 +- fail2ban/server/asyncserver.py | 2 +- fail2ban/server/banmanager.py | 2 +- fail2ban/server/datedetector.py | 2 +- fail2ban/server/datetemplate.py | 2 +- fail2ban/server/faildata.py | 2 +- fail2ban/server/failmanager.py | 2 +- fail2ban/server/filter.py | 2 +- fail2ban/server/filtergamin.py | 2 +- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 2 +- fail2ban/server/jail.py | 2 +- fail2ban/server/jailthread.py | 2 +- fail2ban/server/server.py | 11 ++++++----- fail2ban/server/ticket.py | 2 +- fail2ban/server/transmitter.py | 2 +- 27 files changed, 32 insertions(+), 31 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 68a31786..d276be1d 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -109,7 +109,7 @@ else: # Custom log format for the verbose tests runs if verbosity > 1: # pragma: no cover - stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s %(name)s' + fmt)) else: # pragma: no cover # just prefix with the space stdout.setFormatter(Formatter(fmt)) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 9ad1ef28..787a41c7 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -31,7 +31,7 @@ import logging from configreader import ConfigReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class ActionReader(ConfigReader): diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 1403bb08..fe58ccf3 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -26,7 +26,7 @@ import logging from fail2ban.exceptions import UnknownJailException, DuplicateJailException # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) ## # Beautify the output of the client. diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7ac8b4a5..0ffa0728 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -31,7 +31,7 @@ import logging, os from ConfigParser import SafeConfigParser # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class SafeConfigParserWithIncludes(SafeConfigParser): """ diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 243c843c..6f1e7740 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -32,7 +32,7 @@ from configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class ConfigReader(SafeConfigParserWithIncludes): diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 2097fd54..23ee88c6 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,7 +33,7 @@ from fail2banreader import Fail2banReader from jailsreader import JailsReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class Configurator: diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c8f42976..e388f9ae 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -31,7 +31,7 @@ import logging from configreader import ConfigReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class Fail2banReader(ConfigReader): diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 7dba3579..8b00446e 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -31,7 +31,7 @@ import logging from configreader import ConfigReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class FilterReader(ConfigReader): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 55e512f3..e4175f4f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -34,7 +34,7 @@ from filterreader import FilterReader from actionreader import ActionReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class JailReader(ConfigReader): diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 91e178d6..345a225f 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -32,7 +32,7 @@ from configreader import ConfigReader from jailreader import JailReader # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client.config") +logSys = logging.getLogger(__name__) class JailsReader(ConfigReader): diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 5fde3ae1..d2a8cfde 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -32,7 +32,7 @@ import threading #from subprocess import call # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.actions.action") +logSys = logging.getLogger(__name__) # Create a lock for running system commands _cmd_lock = threading.Lock() diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index ddcc83d6..c0373239 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -34,7 +34,7 @@ from mytime import MyTime import time, logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.actions") +logSys = logging.getLogger(__name__) ## # Execute commands. diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index d5fb791c..5f729e11 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -33,7 +33,7 @@ import asyncore, asynchat, socket, os, logging, sys, traceback from fail2ban import helpers # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.server") +logSys = logging.getLogger(__name__) ## # Request handler class. diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 1143f791..9c81f252 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -33,7 +33,7 @@ from mytime import MyTime import logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.action") +logSys = logging.getLogger(__name__) ## # Banning Manager. diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index c013d551..2f57294a 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -33,7 +33,7 @@ from datetemplate import DateStrptime, DateTai64n, DateEpoch, DateISO8601 from threading import Lock # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter.datedetector") +logSys = logging.getLogger(__name__) class DateDetector: diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 51b8bb1e..c75a02f2 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -33,7 +33,7 @@ from mytime import MyTime import iso8601 import logging -logSys = logging.getLogger("fail2ban.datetemplate") +logSys = logging.getLogger(__name__) class DateTemplate: diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py index 1f0bda04..efda51e1 100644 --- a/fail2ban/server/faildata.py +++ b/fail2ban/server/faildata.py @@ -30,7 +30,7 @@ __license__ = "GPL" import logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban") +logSys = logging.getLogger(__name__) class FailData: diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 02f16ce3..60e71c7b 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -33,7 +33,7 @@ from threading import Lock import logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = logging.getLogger(__name__) class FailManager: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 4fa22bd4..fe990578 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,7 +38,7 @@ from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = logging.getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index cff5aa54..e324b677 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -30,7 +30,7 @@ from mytime import MyTime import time, logging, gamin # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = logging.getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index f0e23ac1..3217e958 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -33,7 +33,7 @@ from mytime import MyTime import time, logging, os # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = logging.getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index e86498b0..786c6dfa 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -38,7 +38,7 @@ if not hasattr(pyinotify, '__version__') \ from os.path import dirname, sep as pathsep # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = logging.getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index dee64e7f..fa2a8fa5 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -28,7 +28,7 @@ import Queue, logging from actions import Actions # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.jail") +logSys = logging.getLogger(__name__) class Jail: diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 343ea7e2..98fd4066 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -31,7 +31,7 @@ from threading import Thread import logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.server") +logSys = logging.getLogger(__name__) class JailThread(Thread): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 8038a0c1..a5d2d5d2 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -36,7 +36,7 @@ from fail2ban import version import logging, logging.handlers, sys, os, signal # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.server") +logSys = logging.getLogger(__name__) class Server: @@ -329,7 +329,7 @@ class Server: logLevel = logging.WARNING elif value == 3: logLevel = logging.INFO - logging.getLogger("fail2ban").setLevel(logLevel) + logging.getLogger(__name__).parent.parent.setLevel(logLevel) finally: self.__loggingLock.release() @@ -378,9 +378,10 @@ class Server: return False # Removes previous handlers -- in reverse order since removeHandler # alter the list in-place and that can confuses the iterable - for handler in logging.getLogger("fail2ban").handlers[::-1]: + logger = logging.getLogger(__name__).parent.parent + for handler in logger.handlers[::-1]: # Remove the handler. - logging.getLogger("fail2ban").removeHandler(handler) + logger.removeHandler(handler) # And try to close -- it might be closed already try: handler.flush() @@ -392,7 +393,7 @@ class Server: # with older Pythons -- seems to be safe to ignore there # tell the handler to use this format hdlr.setFormatter(formatter) - logging.getLogger("fail2ban").addHandler(hdlr) + logger.addHandler(hdlr) # Does not display this message at startup. if not self.__logTarget == None: logSys.info("Changed logging target to %s for Fail2ban v%s" % diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index c03761c1..95643d15 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -30,7 +30,7 @@ __license__ = "GPL" import logging # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban") +logSys = logging.getLogger(__name__) class Ticket: diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 010be268..d209441d 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -30,7 +30,7 @@ __license__ = "GPL" import logging, time # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.comm") +logSys = logging.getLogger(__name__) class Transmitter: From 0ea9904440e1a6d27fd20806c68658c4c4976204 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 11 Apr 2013 18:19:44 +0100 Subject: [PATCH 057/426] TST: revert change of log format for testcases in commit a3d82e2 --- bin/fail2ban-testcases | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index d276be1d..68a31786 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -109,7 +109,7 @@ else: # Custom log format for the verbose tests runs if verbosity > 1: # pragma: no cover - stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s %(name)s' + fmt)) + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) else: # pragma: no cover # just prefix with the space stdout.setFormatter(Formatter(fmt)) From d061b2b549445164ad4c3b197e482bf10f220b22 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 13 Apr 2013 16:55:22 +0100 Subject: [PATCH 058/426] TST: Fix issues in tests which assumed dictionary's order --- fail2ban/tests/clientreadertestcase.py | 4 ++- fail2ban/tests/filtertestcase.py | 44 ++++++++++++++++---------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 99d9a50e..d721ef00 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -146,7 +146,9 @@ class FilterReaderTest(unittest.TestCase): #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - self.assertEquals(filterReader.convert(), output) + # Add sort as configreader uses dictionary and therefore order + # is unreliable + self.assertEquals(sorted(filterReader.convert()), sorted(output)) class JailsReaderTest(unittest.TestCase): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 460f2b45..6578a261 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -94,22 +94,25 @@ def _assert_equal_entries(utest, found, output, count=None): # do not check if custom count (e.g. going through them twice) utest.assertEqual(repr(found[3]), repr(output[3])) +def _ticket_tuple(ticket): + """Create a tuple for easy comparison from fail ticket + """ + attempts = ticket.getAttempt() + date = ticket.getTime() + ip = ticket.getIP() + matches = ticket.getMatches() + return (ip, attempts, date, matches) + def _assert_correct_last_attempt(utest, filter_, output, count=None): """Additional helper to wrap most common test case Test filter to contain target ticket """ if isinstance(filter_, DummyJail): - ticket = filter_.getFailTicket() + found = _ticket_tuple(filter_.getFailTicket()) else: # when we are testing without jails - ticket = filter_.failManager.toBan() - - attempts = ticket.getAttempt() - date = ticket.getTime() - ip = ticket.getIP() - matches = ticket.getMatches() - found = (ip, attempts, date, matches) + found = _ticket_tuple(filter_.failManager.toBan()) _assert_equal_entries(utest, found, output, count) @@ -642,10 +645,14 @@ class GetFailures(unittest.TestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) - _assert_correct_last_attempt(self, self.filter, output.pop()) - - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + foundList = [] + while True: + try: + foundList.append( + _ticket_tuple(self.filter.failManager.toBan())[0:3]) + except FailManagerEmpty: + break + self.assertEqual(sorted(foundList), sorted(output)) def testGetFailuresMultiLineIgnoreRegex(self): output = [("192.0.43.10", 2, 1124013599.0)] @@ -673,11 +680,14 @@ class GetFailures(unittest.TestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) - _assert_correct_last_attempt(self, self.filter, output.pop()) - _assert_correct_last_attempt(self, self.filter, output.pop()) - - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + foundList = [] + while True: + try: + foundList.append( + _ticket_tuple(self.filter.failManager.toBan())[0:3]) + except FailManagerEmpty: + break + self.assertEqual(sorted(foundList), sorted(output)) class DNSUtilsTests(unittest.TestCase): From 9241ded37663988412091b0cb2622eb408bfb61c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 13 Apr 2013 16:58:05 +0100 Subject: [PATCH 059/426] TST: Fix up fail2ban python3 scripts --- fail2ban-2to3 | 17 +++++++---------- fail2ban-testcases-all-python3 | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/fail2ban-2to3 b/fail2ban-2to3 index a6a61090..2015ed5b 100755 --- a/fail2ban-2to3 +++ b/fail2ban-2to3 @@ -5,13 +5,10 @@ set -eu -nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases" - -find . \ - -name "*.py" \ - -exec 2to3 -w --no-diffs $nonPyFiles {} + \ - || (echo "Fail!" >&2 && exit 1) - -echo "Success!" >&2 - -exit 0 +if 2to3 -w --no-diffs bin/* fail2ban;then + echo "Success!" >&2 + exit 0 +else + echo "Fail!" >&2 + exit 1 +fi diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3 index fd3ff871..9445f396 100755 --- a/fail2ban-testcases-all-python3 +++ b/fail2ban-testcases-all-python3 @@ -9,7 +9,7 @@ for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg} do [ -e "$python" ] || continue echo "Testing using $python" - $python ./fail2ban-testcases "$@" || failed+=" $python" + $python bin/fail2ban-testcases "$@" || failed+=" $python" done if [ ! -z "$failed" ]; then From 70bcb0e32ff912e440a9e9a78523b66329b3191b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 13 Apr 2013 17:05:19 +0100 Subject: [PATCH 060/426] Add *.bak files generated by 2to3 to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c2e979e5..b697c3dc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ htmlcov .coverage *.orig *.rej +*.bak From 36097ffc3e6bd434300dae4778e114efdb8010ea Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 14 Apr 2013 10:18:22 +0100 Subject: [PATCH 061/426] DOC: Revert setup.py messages to use print statement --- setup.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 38392406..57b75225 100755 --- a/setup.py +++ b/setup.py @@ -117,26 +117,25 @@ for directory in elements: obsoleteFiles.append(path) if obsoleteFiles: - sys.stdout.write("\n") - sys.stdout.write("Obsolete files from previous Fail2Ban versions " \ - "were found on your system.\n") - sys.stdout.write("Please delete them:\n") - sys.stdout.write("\n") + print("") + print("Obsolete files from previous Fail2Ban versions were found on " + "your system.") + print("Please delete them:") + print("") for f in obsoleteFiles: - sys.stdout.write("\t" + f) - sys.stdout.write("\n") + print("\t" + f) + print("") if isdir("/usr/lib/fail2ban"): - sys.stdout.write("\n") - sys.stdout.write("Fail2ban is not installed under /usr/lib anymore. " \ - "The new location is under /usr/share. Please remove the " \ - "directory /usr/lib/fail2ban and everything under this directory.\n") - sys.stdout.write("\n") + print("") + print("Fail2ban is not installed under /usr/lib anymore. The new " + "location is under /usr/share. Please remove the directory " + "/usr/lib/fail2ban and everything under this directory.") + print("") # Update config file if sys.argv[1] == "install": - sys.stdout.write("\n") - sys.stdout.write("Please do not forget to update your configuration " - "files.\n") - sys.stdout.write("They are in /etc/fail2ban/.\n") - sys.stdout.write("\n") + print("") + print("Please do not forget to update your configuration files.") + print("They are in /etc/fail2ban/.") + print("") From d28788c87b4f02f5ec6ab2f7d928a58da0e06ff6 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 14 Apr 2013 10:24:44 +0100 Subject: [PATCH 062/426] TST: Revert changes for filter testcase open statement Also merging python3 support --- fail2ban/tests/filtertestcase.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 6578a261..bc043373 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -22,6 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko" __license__ = "GPL" +from __builtin__ import open as fopen import unittest import os import sys @@ -44,13 +45,18 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 # adding a sufficiently large buffer might help to guarantee that # writes happen atomically. -# Overload also for python3 to use utf-8 encoding by default -if sys.version_info >= (3,): - def open_(filename, mode): - return open(filename, mode, 50000, encoding='utf-8', errors='ignore') -else: - def open_(filename, mode): - return open(filename, mode, 50000) +def open(*args): + """Overload built in open so we could assure sufficiently large buffer + + Explicit .flush would be needed to assure that changes leave the buffer + """ + if len(args) == 2: + # ~50kB buffer should be sufficient for all tests here. + args = args + (50000,) + if sys.version_info >= (3,): + return fopen(*args, encoding='utf-8', errors='ignore') + else: + return fopen(*args) def _killfile(f, name): try: @@ -126,7 +132,7 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # polling filter could detect the change time.sleep(1) if isinstance(fin, str): # pragma: no branch - only used with str in test cases - fin = open_(fin, 'r') + fin = open(fin, 'r') # Skip for i in xrange(skip): _ = fin.readline() @@ -141,7 +147,7 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line i += 1 # Write: all at once and flush if isinstance(fout, str): - fout = open_(fout, mode) + fout = open(fout, mode) fout.write('\n'.join(lines)) fout.flush() # to give other threads possibly some time to crunch @@ -209,7 +215,7 @@ class LogFileMonitor(unittest.TestCase): """Call before every test case.""" self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') - self.file = open_(self.name, 'a') + self.file = open(self.name, 'a') self.filter = FilterPoll(None) self.filter.addLogPath(self.name) self.filter.setActive(True) @@ -251,7 +257,7 @@ class LogFileMonitor(unittest.TestCase): # we are not signaling as modified whenever # it gets away self.assertTrue(self.notModified()) - f = open_(self.name, 'a') + f = open(self.name, 'a') self.assertTrue(self.isModified()) self.assertTrue(self.notModified()) _sleep_4_poll() @@ -360,7 +366,7 @@ def get_monitor_failures_testcase(Filter_): self.filter = self.name = 'NA' self.name = '%s-%d' % (testclass_name, self.count) MonitorFailures.count += 1 # so we have unique filenames across tests - self.file = open_(self.name, 'a') + self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) self.filter.addLogPath(self.name) @@ -483,7 +489,7 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01) # create a bogus file in the same directory and see if that doesn't affect - open_(self.name + '.bak2', 'w').write('') + open(self.name + '.bak2', 'w').write('') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) From 88187fc1612300510fdcdaead289b533d0617b7b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 14 Apr 2013 11:00:08 +0100 Subject: [PATCH 063/426] TST: Tweak python3 open statement to resolve python2.5 SyntaxError --- fail2ban/tests/filtertestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index bc043373..fa9340ca 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -54,7 +54,7 @@ def open(*args): # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) if sys.version_info >= (3,): - return fopen(*args, encoding='utf-8', errors='ignore') + return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) else: return fopen(*args) From a8f2e02eea89885390b894624bd4a7d0b5618da9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 20:44:13 +0100 Subject: [PATCH 064/426] DOC: Revert dnsToIp error change, seperate log message for socket.error --- fail2ban/server/filter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b5f910f3..5d6a569c 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -664,10 +664,14 @@ class DNSUtils: """ try: return socket.gethostbyname_ex(dns)[2] - except socket.error: + except socket.gaierror: logSys.warn("Unable to find a corresponding IP address for %s" % dns) return list() + except socket.error, e: + logSys.warn("Socket error raised trying to resolve hostname %s: %s" + % (dns, e)) + return list() dnsToIp = staticmethod(dnsToIp) #@staticmethod From 4db77ebf41dd4f1991094d3f568e9b9ab4dfa0ab Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 20:47:37 +0100 Subject: [PATCH 065/426] ENH: Clarify use of bytes in csocket and asyncserver for python3 --- fail2ban/client/csocket.py | 12 ++++++++---- fail2ban/server/asyncserver.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index 283d6357..346c7a90 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -31,6 +31,13 @@ __license__ = "GPL" from pickle import dumps, loads, HIGHEST_PROTOCOL import socket, sys +if sys.version_info >= (3,): + # b"" causes SyntaxError in python <= 2.5, so below implements equivalent + EMPTY_BYTES = bytes("", encoding="ascii") +else: + # python 2.x, string type is equivalent to bytes. + EMPTY_BYTES = "" + class CSocket: if sys.version_info >= (3,): @@ -55,10 +62,7 @@ class CSocket: #@staticmethod def receive(sock): - if sys.version_info >= (3,): - msg = bytes("", encoding='ascii') - else: - msg = '' + msg = EMPTY_BYTES while msg.rfind(CSocket.END_STRING) == -1: chunk = sock.recv(6) if chunk == '': diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 1f46d936..9d506a66 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -35,6 +35,13 @@ from fail2ban import helpers # Gets the instance of the logger. logSys = logging.getLogger(__name__) +if sys.version_info >= (3,): + # b"" causes SyntaxError in python <= 2.5, so below implements equivalent + EMPTY_BYTES = bytes("", encoding="ascii") +else: + # python 2.x, string type is equivalent to bytes. + EMPTY_BYTES = "" + ## # Request handler class. # @@ -66,10 +73,7 @@ class RequestHandler(asynchat.async_chat): def found_terminator(self): # Joins the buffer items. - if sys.version_info >= (3,): - message = loads(bytes("", encoding="ascii").join(self.__buffer)) - else: - message = loads("".join(self.__buffer)) + message = loads(EMPTY_BYTES.join(self.__buffer)) # Gives the message to the transmitter. message = self.__transmitter.proceed(message) # Serializes the response. From f14cb7302a7682ffc9bb491640807fed3f3e5801 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 21:00:45 +0100 Subject: [PATCH 066/426] DOC: Add python3 to requirements --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 4a6460d5..e8c0dcd2 100644 --- a/README +++ b/README @@ -19,7 +19,7 @@ Installation: ------------- Required: - >=python-2.3 (http://www.python.org) + >=python-2.3 or >=python-3.0 (http://www.python.org) Optional: pyinotify: From be0e8faa4eab4e0b444374647360ad3d1fd9e41b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 21:14:12 +0100 Subject: [PATCH 067/426] TST: Add maxlines to transmitter testcases --- fail2ban/tests/servertestcase.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 4a3435b4..bc08c061 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -271,6 +271,12 @@ class Transmitter(TransmitterBase): self.setGetTest("maxretry", "-2", -2, jail=self.jailName) self.setGetTestNOK("maxretry", "Duck", jail=self.jailName) + def testJailMaxLines(self): + self.setGetTest("maxlines", "5", 5, jail=self.jailName) + self.setGetTest("maxlines", "2", 2, jail=self.jailName) + self.setGetTestNOK("maxlines", "-2", jail=self.jailName) + self.setGetTestNOK("maxlines", "Duck", jail=self.jailName) + def testJailLogPath(self): self.jailAddDelTest( "logpath", From 183cfa6e0062159b4053c4b56eabe991ea89aef0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 21:21:19 +0100 Subject: [PATCH 068/426] ENH: Default maxlines value in jail.conf, and verify value is int >0 --- bin/fail2ban-regex | 8 +------- config/jail.conf | 3 +++ fail2ban/server/filter.py | 6 ++++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index e05a4687..b80affab 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -136,13 +136,7 @@ class Fail2banRegex: elif opt[0] in ["-v", "--verbose"]: self.__verbose = True elif opt[0] in ["-l", "--maxlines"]: - try: - self.__filter.setMaxLines(int(opt[1])) - except ValueError: - print "Invlaid value for maxlines: %s" % ( - opt[1]) - fail2banRegex.dispUsage() - sys.exit(-1) + self.__filter.setMaxLines(opt[1]) #@staticmethod def logIsFile(value): diff --git a/config/jail.conf b/config/jail.conf index 4399d0bd..b7ba25d4 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -32,6 +32,9 @@ findtime = 600 # "maxretry" is the number of failures before a host get banned. maxretry = 3 +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 1 + # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling" and "auto". # This option can be overridden in each jail as well. diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index fe990578..f028b44d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -218,8 +218,10 @@ class Filter(JailThread): # @param value the line buffer size def setMaxLines(self, value): - self.__lineBufferSize = max(1, value) - logSys.info("Set maxLines = %i" % self.__lineBufferSize) + if int(value) <= 0: + raise ValueError("maxlines must be integer greater than zero") + self.__lineBufferSize = int(value) + logSys.info("Set maxlines = %i" % self.__lineBufferSize) ## # Get the maximum line buffer size. From 1756f709da34d37336b158bdaf33ea1bb196df1b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 15 Apr 2013 22:14:00 +0100 Subject: [PATCH 069/426] DOC: Revert change to fail2ban-refex from 183cfa6 --- bin/fail2ban-regex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index b80affab..e05a4687 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -136,7 +136,13 @@ class Fail2banRegex: elif opt[0] in ["-v", "--verbose"]: self.__verbose = True elif opt[0] in ["-l", "--maxlines"]: - self.__filter.setMaxLines(opt[1]) + try: + self.__filter.setMaxLines(int(opt[1])) + except ValueError: + print "Invlaid value for maxlines: %s" % ( + opt[1]) + fail2banRegex.dispUsage() + sys.exit(-1) #@staticmethod def logIsFile(value): From c6bd8fc80749417238280c84310f39fc1c4d103b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 16 Apr 2013 21:09:53 +0100 Subject: [PATCH 070/426] ENH+TST: Add Apache Tomcat date format --- fail2ban/server/datedetector.py | 6 ++++++ fail2ban/tests/datedetectortestcase.py | 1 + 2 files changed, 7 insertions(+) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index f58ab4e0..88f0f228 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -161,6 +161,12 @@ class DateDetector: template.setRegex("^\d{2}\d{2}\d{2} +\d{1,2}:\d{2}:\d{2}") template.setPattern("%y%m%d %H:%M:%S") self._appendTemplate(template) + # Apache Tomcat + template = DateStrptime() + template.setName("MONTH Day, Year 12hour:Minute:Second AM/PM") + template.setRegex("\S{3}\s{1,2}\d{1,2}, \d{4} \d{1,2}:\d{2}:\d{2} [AP]M") + template.setPattern("%b %d, %Y %I:%M:%S %p") + self._appendTemplate(template) finally: self.__lock.release() diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index e30027e1..23f7a174 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -86,6 +86,7 @@ class DateDetectorTest(unittest.TestCase): "2005-01-23T21:59:59-05:00Z", #ISO 8601 with TZ "<01/23/05@21:59:59>", "050123 21:59:59", # MySQL + "Jan 23, 2005 9:59:59 PM", # Apache Tomcat ): log = sdate + "[sshd] error: PAM: Authentication failure" # exclude From 4d80fad87445f669a1517174d06e2c66c05616bd Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 16 Apr 2013 21:13:31 +0100 Subject: [PATCH 071/426] ENH+DOC: Add Guacamole filter, example log and jail --- config/filter.d/guacamole.conf | 18 ++++++++++++++++++ config/jail.conf | 10 ++++++++++ fail2ban/tests/files/logs/guacamole | 6 ++++++ 3 files changed, 34 insertions(+) create mode 100644 config/filter.d/guacamole.conf create mode 100644 fail2ban/tests/files/logs/guacamole diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf new file mode 100644 index 00000000..272460e3 --- /dev/null +++ b/config/filter.d/guacamole.conf @@ -0,0 +1,18 @@ +# Fail2Ban configuration file for guacamole +# +# Author: Steven Hiscocks +# + +[Definition] + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. +# Values: TEXT +# +failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" failed\.$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index 4399d0bd..c331e914 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -357,6 +357,16 @@ action = iptables[name=mysql, port=3306, protocol=tcp] logpath = /var/log/mysqld.log maxretry = 5 +[guacamole-iptables] + +enabled = false +filter = guacamole +action = iptables-multiport[name=Guacmole, port="http,https"] + sendmail-whois[name=Guacamole, dest=root, sender=fail2ban@example.com] +logpath = /var/log/tomcat*/catalina.out +maxretry = 5 +maxlines = 2 + # Jail for more extended banning of persistent abusers # !!! WARNING !!! diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole new file mode 100644 index 00000000..b0f82f30 --- /dev/null +++ b/fail2ban/tests/files/logs/guacamole @@ -0,0 +1,6 @@ +apr 15, 2013 8:34:08 PM org.slf4j.impl.JCLLoggerAdapter warn +WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. +apr 16, 2013 8:32:13 AM org.slf4j.impl.JCLLoggerAdapter warn +WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. +apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn +WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. From 4665ac6b270236d43f96f3cf9b3d93ef53c84006 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Apr 2013 01:05:04 -0400 Subject: [PATCH 072/426] RF: jail.conf with entries from Debian's copy and changing existing ones to conform the "template" our unittests fail now -- will BF later --- config/jail.conf | 458 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 309 insertions(+), 149 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7ed1bbb6..337fe0b0 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -1,9 +1,13 @@ # Fail2Ban jail specifications file # +# WARNING: heavily refactored in 0.9.0 release. Please review and +# customize settings for your setup. +# # Comments: use '#' for comment lines and ';' for inline comments # # Changes: in most of the cases you should not modify this -# file, but provide customizations in jail.local file, e.g.: +# file, but provide customizations in jail.local file, +# or separate .conf files under jail.d/ directory, e.g.: # # [DEFAULT] # bantime = 3600 @@ -11,12 +15,17 @@ # [ssh-iptables] # enabled = true # +# See jail.conf(5) man page for more information # The DEFAULT allows a global definition of the options. They can be overridden # in each jail afterwards. [DEFAULT] +# +# MISCELANEOUS OPTIONS +# + # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not # ban a host which matches an address in this list. Several addresses can be # defined using space separator. @@ -30,7 +39,7 @@ bantime = 600 findtime = 600 # "maxretry" is the number of failures before a host get banned. -maxretry = 3 +maxretry = 5 # "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 1 @@ -52,7 +61,7 @@ backend = auto # warn when reverse DNS lookups are performed, or ignore all hostnames in logs # # yes: if a hostname is encountered, a reverse DNS lookup will be performed. -# warn: if a hostname is encountered, a reverse DNS lookup will be performed, +# warn: if a hostname is encountered, a reverse DNS lookup will be performed, # but it will be logged as a warning. # no: if a hostname is encountered, will not be used for banning, # but it will be logged as info. @@ -66,43 +75,113 @@ usedns = warn logencoding = auto -# This jail corresponds to the standard configuration in Fail2ban 0.6. -# The mail-whois action send a notification e-mail with a whois request -# in the body. +# +# ACTIONS +# -[ssh-iptables] +# +# Destination email address used solely for the interpolations in +# jail.{conf,local} configuration files. +destemail = root@localhost -enabled = false +# Default banning action (e.g. iptables, iptables-new, +# iptables-multiport, shorewall, etc) It is used to define +# action_* variables. Can be overridden globally or per +# section within jail.local file +banaction = iptables-multiport + +# E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the +# mailing. Change mta configuration parameter to mail if you want to +# revert to conventional 'mail'. +mta = sendmail + +# Default protocol +protocol = tcp + +# Specify chain where jumps would need to be added in iptables-* actions +chain = INPUT + +# +# Action shortcuts. To be used to define action parameter + +# The simplest action to take: ban only +action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + +# ban & send an e-mail with whois report to the destemail. +action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + +# ban & send an e-mail with whois report and relevant log lines +# to the destemail. +action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + +# Choose default action. To change, just override value of 'action' with the +# interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local +# globally (section [DEFAULT]) or per specific section +action = %(action_)s + + +# +# JAILS +# + +# +# SSH servers +# + +[sshd] + +enabled = true +port = ssh filter = sshd -action = iptables[name=SSH, port=ssh, protocol=tcp] - sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com] -logpath = /var/log/sshd.log -maxretry = 5 +logpath = /var/log/auth.log + /var/log/sshd.log -[proftpd-iptables] +[sshd-ddos] enabled = false -filter = proftpd -action = iptables[name=ProFTPD, port=ftp, protocol=tcp] - sendmail-whois[name=ProFTPD, dest=you@example.com] -logpath = /var/log/proftpd/proftpd.log -maxretry = 6 +port = ssh +filter = sshd-ddos +logpath = /var/log/auth.log + /var/log/sshd.log -# This jail forces the backend to "polling". - -[sasl-iptables] +[dropbear] enabled = false -filter = sasl -backend = polling -action = iptables[name=sasl, port=smtp, protocol=tcp] - sendmail-whois[name=sasl, dest=you@example.com] -logpath = /var/log/mail.log +port = ssh +filter = sshd +logpath = /var/log/dropbear + + +# Generic filter for PAM. Has to be used with action which bans all +# ports such as iptables-allports, shorewall + +[pam-generic] + +enabled = false +# pam-generic filter can be customized to monitor specific subset of 'tty's +filter = pam-generic +banaction = iptables-allports +# port actually must be irrelevant but lets leave it all for some possible uses +port = anyport +logpath = /var/log/auth.log + +[xinetd-fail] + +enabled = false +filter = xinetd-fail +port = all +banaction = iptables-multiport-log +logpath = /var/log/daemon.log +maxretry = 2 + +# .. custom jails # Here we use TCP-Wrappers instead of Netfilter/Iptables. "ignoreregex" is # used to avoid banning the user "myuser". -[ssh-tcpwrapper] +[sshd-tcpwrapper] enabled = false filter = sshd @@ -114,36 +193,138 @@ logpath = /var/log/sshd.log # Here we use blackhole routes for not requiring any additional kernel support # to store large volumes of banned IPs -[ssh-route] +[sshd-route] enabled = false filter = sshd action = route logpath = /var/log/sshd.log -maxretry = 5 # Here we use a combination of Netfilter/Iptables and IPsets # for storing large volumes of banned IPs # # IPset comes in two versions. See ipset -V for which one to use # requires the ipset package and kernel support. -[ssh-iptables-ipset4] +[sshd-iptables-ipset4] enabled = false filter = sshd action = iptables-ipset-proto4[name=SSH, port=ssh, protocol=tcp] logpath = /var/log/sshd.log -maxretry = 5 -[ssh-iptables-ipset6] +[sshd-iptables-ipset6] + enabled = false filter = sshd action = iptables-ipset-proto6[name=SSH, port=ssh, protocol=tcp, bantime=600] logpath = /var/log/sshd.log -maxretry = 5 -# This jail demonstrates the use of wildcards in "logpath". -# Moreover, it is possible to give other files on a new line. +# This jail uses ipfw, the standard firewall on FreeBSD. The "ignoreip" +# option is overridden in this jail. Moreover, the action "mail-whois" defines +# the variable "name" which contains a comma using "". The characters '' are +# valid too. + +[sshd-ipfw] + +enabled = false +filter = sshd +action = ipfw[localhost=192.168.0.1] + sendmail-whois[name="SSH,IPFW", dest=you@example.com] +logpath = /var/log/auth.log +ignoreip = 168.192.0.1 + + +# +# HTTP servers +# + +[apache-auth] + +enabled = false +port = http,https +filter = apache-auth +logpath = /var/log/apache*/*error.log + +# Ban hosts which agent identifies spammer robots crawling the web +# for email addresses. The mail outputs are buffered. + +[apache-badbots] + +enabled = false +port = http,https +filter = apache-badbots +logpath = /var/log/apache*/*access.log + /var/www/*/logs/access_log +bantime = 172800 +maxretry = 1 + +[apache-noscript] + +enabled = false +port = http,https +filter = apache-noscript +logpath = /var/log/apache*/*error.log +maxretry = 6 + +[apache-overflows] + +enabled = false +port = http,https +filter = apache-overflows +logpath = /var/log/apache*/*error.log +maxretry = 2 + +# Ban attackers that try to use PHP's URL-fopen() functionality +# through GET/POST variables. - Experimental, with more than a year +# of usage in production environments. + +[php-url-fopen] + +enabled = false +port = http,https +filter = php-url-fopen +logpath = /var/www/*/logs/access_log + +# A simple PHP-fastcgi jail which works with lighttpd. +# If you run a lighttpd server, then you probably will +# find these kinds of messages in your error_log: +# ALERT – tried to register forbidden variable ‘GLOBALS’ +# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') + +[lighttpd-fastcgi] + +enabled = false +port = http,https +filter = lighttpd-fastcgi +logpath = /var/log/lighttpd/error.log + +# Same as above for mod_auth +# It catches wrong authentifications + +[lighttpd-auth] + +enabled = false +port = http,https +filter = lighttpd-auth +logpath = /var/log/lighttpd/error.log + +[roundcube-auth] + +enabled = false +port = http,https +filter = roundcube-auth +logpath = /var/log/roundcube/userlogins + +[sogo-auth] + +enabled = false +filter = sogo-auth +port = http,https +# without proxy this would be: +# port = 20000 +logpath = /var/log/sogo/sogo.log + +# ... custom jails [apache-tcpwrapper] @@ -151,20 +332,39 @@ enabled = false filter = apache-auth action = hostsdeny logpath = /var/log/apache*/*error.log - /home/www/myhomepage/error.log maxretry = 6 -# The hosts.deny path can be defined with the "file" argument if it is -# not in /etc. -[postfix-tcpwrapper] +# +# FTP servers +# + + +[proftpd] enabled = false -filter = postfix -action = hostsdeny[file=/not/a/standard/path/hosts.deny] - sendmail[name=Postfix, dest=you@example.com] -logpath = /var/log/postfix.log -bantime = 300 +port = ftp,ftp-data,ftps,ftps-data +filter = proftpd +logpath = /var/log/proftpd/proftpd.log + +[pure-ftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = pure-ftpd +logpath = /var/log/auth.log +maxretry = 6 + +[vsftpd] + +enabled = false +port = ftp,ftp-data,ftps,ftps-data +filter = vsftpd +logpath = /var/log/vsftpd.log +# or overwrite it in jails.local to be +# logpath = /var/log/auth.log +# if you want to rely on PAM failed login attempts +# vsftpd's failregex should match both of those formats # Do not ban anybody. Just report information about the remote host. # A notification is sent at most every 600 seconds (bantime). @@ -178,117 +378,78 @@ logpath = /var/log/vsftpd.log maxretry = 5 bantime = 1800 -# Same as above but with banning the IP address. -[vsftpd-iptables] +[wuftpd] enabled = false -filter = vsftpd -action = iptables[name=VSFTPD, port=ftp, protocol=tcp] - sendmail-whois[name=VSFTPD, dest=you@example.com] -logpath = /var/log/vsftpd.log -maxretry = 5 -bantime = 1800 +port = ftp,ftp-data,ftps,ftps-data +filter = wuftpd +logpath = /var/log/syslog +maxretry = 6 -# Ban hosts which agent identifies spammer robots crawling the web -# for email addresses. The mail outputs are buffered. +# +# Mail servers +# -[apache-badbots] +[couriersmtp] enabled = false -filter = apache-badbots -action = iptables-multiport[name=BadBots, port="http,https"] - sendmail-buffered[name=BadBots, lines=5, dest=you@example.com] -logpath = /var/www/*/logs/access_log -bantime = 172800 -maxretry = 1 +port = smtp,ssmtp +filter = couriersmtp +logpath = /var/log/mail.log -# Use shorewall instead of iptables. - -[apache-shorewall] +[postfix] enabled = false -filter = apache-noscript -action = shorewall +port = smtp,ssmtp +filter = postfix +logpath = /var/log/mail.log + +# The hosts.deny path can be defined with the "file" argument if it is +# not in /etc. + +[postfix-tcpwrapper] + +enabled = false +filter = postfix +action = hostsdeny[file=/not/a/standard/path/hosts.deny] sendmail[name=Postfix, dest=you@example.com] -logpath = /var/log/apache2/error_log +logpath = /var/log/postfix.log +bantime = 300 -# Monitor roundcube server +# +# Mail servers authenticators: might be used for smtp,ftp,imap servers, so +# all relevant ports get banned +# -[roundcube-iptables] +[courierauth] enabled = false -filter = roundcube-auth -action = iptables[name=RoundCube, port="http,https"] -logpath = /var/log/roundcube/userlogins +port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s +filter = courierlogin +logpath = /var/log/mail.log -# Monitor SOGo groupware server - -[sogo-iptables] +[sasl] enabled = false -filter = sogo-auth -port = http, https -# without proxy this would be: -# port = 20000 +port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s +filter = sasl +# You might consider monitoring /var/log/mail.warn instead if you are +# running postfix since it would provide the same log lines at the +# "warn" level but overall at the smaller filesize. +logpath = /var/log/mail.log -action = iptables[name=SOGo, port="http,https"] -logpath = /var/log/sogo/sogo.log - -# Ban attackers that try to use PHP's URL-fopen() functionality -# through GET/POST variables. - Experimental, with more than a year -# of usage in production environments. - -[php-url-fopen] +[dovecot] enabled = false -port = http,https -filter = php-url-fopen -logpath = /var/www/*/logs/access_log -maxretry = 1 +port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s +filter = dovecot +logpath = /var/log/mail.log -# A simple PHP-fastcgi jail which works with lighttpd. -# If you run a lighttpd server, then you probably will -# find these kinds of messages in your error_log: -# ALERT – tried to register forbidden variable ‘GLOBALS’ -# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php') -# This jail would block the IP 1.2.3.4. - -[lighttpd-fastcgi] - -enabled = false -port = http,https -filter = lighttpd-fastcgi -# adapt the following two items as needed -logpath = /var/log/lighttpd/error.log -maxretry = 2 - -# Same as above for mod_auth -# It catches wrong authentications - -[lighttpd-auth] - -enabled = false -port = http,https -filter = lighttpd-auth -# adapt the following two items as needed -logpath = /var/log/lighttpd/error.log -maxretry = 2 - -# This jail uses ipfw, the standard firewall on FreeBSD. The "ignoreip" -# option is overridden in this jail. Moreover, the action "mail-whois" defines -# the variable "name" which contains a comma using "". The characters '' are -# valid too. - -[ssh-ipfw] - -enabled = false -filter = sshd -action = ipfw[localhost=192.168.0.1] - sendmail-whois[name="SSH,IPFW", dest=you@example.com] -logpath = /var/log/auth.log -ignoreip = 168.192.0.1 +# +# DNS servers +# # These jails block attacks against named (bind9). By default, logging is off # with bind9 installation. You will need something like this: @@ -319,30 +480,33 @@ ignoreip = 168.192.0.1 # # enabled = false # filter = named-refused -# action = iptables-multiport[name=Named, port="domain,953", protocol=udp] -# sendmail-whois[name=Named, dest=you@example.com] +# port = domain,953 +# protocol = udp # logpath = /var/log/named/security.log # ignoreip = 168.192.0.1 # This jail blocks TCP traffic for DNS requests. -[named-refused-tcp] +[named-refused] enabled = false filter = named-refused -action = iptables-multiport[name=Named, port="domain,953", protocol=tcp] - sendmail-whois[name=Named, dest=you@example.com] +port = domain,953 logpath = /var/log/named/security.log ignoreip = 168.192.0.1 +# +# Miscelaneous +# + # Multiple jails, 1 per protocol, are necessary ATM: # see https://github.com/fail2ban/fail2ban/issues/37 [asterisk-tcp] enabled = false filter = asterisk -action = iptables-multiport[name=asterisk-tcp, port="5060,5061", protocol=tcp] - sendmail-whois[name=Asterisk, dest=you@example.com, sender=fail2ban@example.com] +port = 5060,5061 +protocol = tcp logpath = /var/log/asterisk/messages maxretry = 10 @@ -350,31 +514,27 @@ maxretry = 10 enabled = false filter = asterisk -action = iptables-multiport[name=asterisk-udp, port="5060,5061", protocol=udp] - sendmail-whois[name=Asterisk, dest=you@example.com, sender=fail2ban@example.com] +port = 5060,5061 +protocol = udp logpath = /var/log/asterisk/messages maxretry = 10 # To log wrong MySQL access attempts add to /etc/my.cnf: # log-error=/var/log/mysqld.log # log-warning = 2 -[mysqld-iptables] +[mysqld-auth] enabled = false filter = mysqld-auth -action = iptables[name=mysql, port=3306, protocol=tcp] - sendmail-whois[name=MySQL, dest=root, sender=fail2ban@example.com] +port = 3306 logpath = /var/log/mysqld.log -maxretry = 5 -[guacamole-iptables] +[guacamole] enabled = false filter = guacamole -action = iptables-multiport[name=Guacmole, port="http,https"] - sendmail-whois[name=Guacamole, dest=root, sender=fail2ban@example.com] +port = http,https logpath = /var/log/tomcat*/catalina.out -maxretry = 5 maxlines = 2 From 01499ad0de00b2a0ca2d06959f8643ad0a2f5105 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:07:19 +0100 Subject: [PATCH 073/426] NF: Filters now allow adding of [Init] section similar to actions --- fail2ban/client/actionreader.py | 72 +++++++++----------------- fail2ban/client/configreader.py | 37 +++++++++++++ fail2ban/client/filterreader.py | 45 +++++----------- fail2ban/client/jailreader.py | 47 +++++++++-------- fail2ban/tests/clientreadertestcase.py | 6 +-- 5 files changed, 103 insertions(+), 104 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 787a41c7..b9211a1b 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,66 +28,42 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging -from configreader import ConfigReader +from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class ActionReader(ConfigReader): - - def __init__(self, action, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = action[0] - self.__cInfo = action[1] - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class ActionReader(OptionConfigReader): + + _configOpts = [ + ["string", "actionstart", ""], + ["string", "actionstop", ""], + ["string", "actioncheck", ""], + ["string", "actionban", ""], + ["string", "actionunban", ""], + ] + def read(self): - return ConfigReader.read(self, "action.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "actionstart", ""], - ["string", "actionstop", ""], - ["string", "actioncheck", ""], - ["string", "actionban", ""], - ["string", "actionunban", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) - - if self.has_section("Init"): - for opt in self.options("Init"): - if not self.__cInfo.has_key(opt): - self.__cInfo[opt] = self.get("Init", opt) - + return ConfigReader.read(self, "action.d/" + self._file) + def convert(self): - head = ["set", self.__name] + head = ["set", self._name] stream = list() - stream.append(head + ["addaction", self.__file]) - for opt in self.__opts: + stream.append(head + ["addaction", self._file]) + for opt in self._opts: if opt == "actionstart": - stream.append(head + ["actionstart", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstart", self._file, self._opts[opt]]) elif opt == "actionstop": - stream.append(head + ["actionstop", self.__file, self.__opts[opt]]) + stream.append(head + ["actionstop", self._file, self._opts[opt]]) elif opt == "actioncheck": - stream.append(head + ["actioncheck", self.__file, self.__opts[opt]]) + stream.append(head + ["actioncheck", self._file, self._opts[opt]]) elif opt == "actionban": - stream.append(head + ["actionban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionban", self._file, self._opts[opt]]) elif opt == "actionunban": - stream.append(head + ["actionunban", self.__file, self.__opts[opt]]) + stream.append(head + ["actionunban", self._file, self._opts[opt]]) # cInfo - if self.__cInfo: - for p in self.__cInfo: - stream.append(head + ["setcinfo", self.__file, p, self.__cInfo[p]]) + if self._initOpts: + for p in self._initOpts: + stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]]) return stream - diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6f1e7740..4cb9cd69 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -130,3 +130,40 @@ class ConfigReader(SafeConfigParserWithIncludes): "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values + +class OptionConfigReader(ConfigReader): + + _configOpts = [] + + def __init__(self, file_, jailName, initOpts, **kwargs): + ConfigReader.__init__(self, **kwargs) + self._file = file_ + self._name = jailName + self._initOpts = initOpts + + def setFile(self, fileName): + self._file = fileName + + def getFile(self): + return self.__file + + def setName(self, name): + self._name = name + + def getName(self): + return self._name + + def read(self): + return ConfigReader.read(self, self._file) + + def getOptions(self, pOpts): + self._opts = ConfigReader.getOptions( + self, "Definition", self._configOpts, pOpts) + + if self.has_section("Init"): + for opt in self.options("Init"): + if not self._initOpts.has_key(opt): + self._initOpts[opt] = self.get("Init", opt) + + def convert(self): + raise NotImplementedError diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b00446e..bdfba4d0 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -28,50 +28,33 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging -from configreader import ConfigReader +from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class FilterReader(ConfigReader): - - def __init__(self, fileName, name, **kwargs): - ConfigReader.__init__(self, **kwargs) - self.__file = fileName - self.__name = name - - def setFile(self, fileName): - self.__file = fileName - - def getFile(self): - return self.__file - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - +class FilterReader(OptionConfigReader): + + _configOpts = [ + ["string", "ignoreregex", ""], + ["string", "failregex", ""], + ] + def read(self): - return ConfigReader.read(self, "filter.d/" + self.__file) - - def getOptions(self, pOpts): - opts = [["string", "ignoreregex", ""], - ["string", "failregex", ""]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts, pOpts) + return ConfigReader.read(self, "filter.d/" + self._file) def convert(self): stream = list() - for opt in self.__opts: + for opt in self._opts: if opt == "failregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addfailregex", regex]) + stream.append(["set", self._name, "addfailregex", regex]) elif opt == "ignoreregex": - for regex in self.__opts[opt].split('\n'): + for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self.__name, "addignoreregex", regex]) + stream.append(["set", self._name, "addignoreregex", regex]) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6e35bc0b..f6ac09b3 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -38,7 +38,7 @@ logSys = logging.getLogger(__name__) class JailReader(ConfigReader): - actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) @@ -78,8 +78,10 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name, - basedir=self.getBaseDir()) + filterName, filterOpt = JailReader.splitOption( + self.__opts["filter"]) + self.__filter = FilterReader( + filterName, self.__name, filterOpt, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -92,8 +94,9 @@ class JailReader(ConfigReader): try: if not act: # skip empty actions continue - splitAct = JailReader.splitAction(act) - action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir()) + actName, actOpt = JailReader.splitOption(act) + action = ActionReader( + actName, self.__name, actOpt, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -151,23 +154,23 @@ class JailReader(ConfigReader): return stream #@staticmethod - def splitAction(action): - m = JailReader.actionCRE.match(action) + def splitOption(option): + m = JailReader.optionCRE.match(option) d = dict() mgroups = m.groups() if len(mgroups) == 2: - action_name, action_opts = mgroups + option_name, option_opts = mgroups elif len(mgroups) == 1: - action_name, action_opts = mgroups[0], None + option_name, option_opts = mgroups[0], None else: - raise ValueError("While reading action %s we should have got up to " - "2 groups. Got: %r" % (action, mgroups)) - if not action_opts is None: + raise ValueError("While reading option %s we should have got up to " + "2 groups. Got: %r" % (option, mgroups)) + if not option_opts is None: # Huge bad hack :( This method really sucks. TODO Reimplement it. - actions = "" + options = "" escapeChar = None allowComma = False - for c in action_opts: + for c in option_opts: if c in ('"', "'") and not allowComma: # Start escapeChar = c @@ -178,20 +181,20 @@ class JailReader(ConfigReader): allowComma = False else: if c == ',' and allowComma: - actions += "" + options += "" else: - actions += c + options += c # Split using , - actionsSplit = actions.split(',') + optionsSplit = options.split(',') # Replace the tag with , - actionsSplit = [n.replace("", ',') for n in actionsSplit] + optionsSplit = [n.replace("", ',') for n in optionsSplit] - for param in actionsSplit: + for param in optionsSplit: p = param.split('=') try: d[p[0].strip()] = p[1].strip() except IndexError: - logSys.error("Invalid argument %s in '%s'" % (p, action_opts)) - return [action_name, d] - splitAction = staticmethod(splitAction) + logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) + return [option_name, d] + splitOption = staticmethod(splitOption) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d721ef00..02c65d35 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -112,10 +112,10 @@ class JailReaderTest(unittest.TestCase): self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'ssh-iptables') - def testSplitAction(self): + def testSplitOption(self): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitAction(action) + result = JailReader.splitOption(action) self.assertEquals(expected, result) class FilterReaderTest(unittest.TestCase): @@ -140,7 +140,7 @@ class FilterReaderTest(unittest.TestCase): "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"]] - filterReader = FilterReader("testcase01", "testcase01") + filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) From 9672e44d391d155a17627cbe38736ebb4b141045 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:11:41 +0100 Subject: [PATCH 074/426] ENH: Move jail `maxlines` to filter config --- config/filter.d/guacamole.conf | 4 ++++ config/jail.conf | 4 ---- fail2ban/client/filterreader.py | 3 +++ fail2ban/client/jailreader.py | 1 - man/jail.conf.5 | 5 +++++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 272460e3..49cecc5a 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -16,3 +16,7 @@ failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" fa # Values: TEXT # ignoreregex = + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 2 diff --git a/config/jail.conf b/config/jail.conf index 7ed1bbb6..e8d6db05 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -32,9 +32,6 @@ findtime = 600 # "maxretry" is the number of failures before a host get banned. maxretry = 3 -# "maxlines" is number of log lines to buffer for multi-line regex searches -maxlines = 1 - # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling" and "auto". # This option can be overridden in each jail as well. @@ -375,7 +372,6 @@ action = iptables-multiport[name=Guacmole, port="http,https"] sendmail-whois[name=Guacamole, dest=root, sender=fail2ban@example.com] logpath = /var/log/tomcat*/catalina.out maxretry = 5 -maxlines = 2 # Jail for more extended banning of persistent abusers diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index bdfba4d0..09f0e6a8 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -56,5 +56,8 @@ class FilterReader(OptionConfigReader): # Do not send a command if the rule is empty. if regex != '': stream.append(["set", self._name, "addignoreregex", regex]) + if self._initOpts: + if 'maxlines' in self._initOpts: + stream.append(["set", self._name, "maxlines", self._initOpts["maxlines"]]) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index f6ac09b3..39acc446 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -65,7 +65,6 @@ class JailReader(ConfigReader): ["string", "logencoding", "auto"], ["string", "backend", "auto"], ["int", "maxretry", 3], - ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 552b0ac0..d571dc7b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -140,6 +140,11 @@ Using Python "string interpolation" mechanisms, other definitions are allowed an baduseragents = IE|wget failregex = useragent=%(baduseragents)s +.PP +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +.TP +\fBmaxlines\fR +specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .PP Filters can also have a section called [INCLUDES]. This is used to read other configuration files. From 5b227b6670e4d25de7f741d6441a1ff12be6b5ae Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 18 Apr 2013 22:33:42 +0100 Subject: [PATCH 075/426] TST: Add test for FilterReader [Init] `maxlines` override --- fail2ban/tests/clientreadertestcase.py | 12 +++++++++++- fail2ban/tests/files/filter.d/testcase01.conf | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 02c65d35..3e0b555d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -139,7 +139,8 @@ class FilterReaderTest(unittest.TestCase): "error: PAM: )?User not known to the\\nunderlying authentication." "+$^.+ module for .* from \\s*$"], ['set', 'testcase01', 'addignoreregex', - "^.+ john from host 192.168.1.1\\s*$"]] + "^.+ john from host 192.168.1.1\\s*$"], + ['set', 'testcase01', 'maxlines', "1"]] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) filterReader.read() @@ -150,6 +151,15 @@ class FilterReaderTest(unittest.TestCase): # is unreliable self.assertEquals(sorted(filterReader.convert()), sorted(output)) + filterReader = FilterReader( + "testcase01", "testcase01", {'maxlines': "5"}) + filterReader.setBaseDir(TEST_FILES_DIR) + filterReader.read() + #filterReader.getOptions(["failregex", "ignoreregex"]) + filterReader.getOptions(None) + output[-1][-1] = "5" + self.assertEquals(sorted(filterReader.convert()), sorted(output)) + class JailsReaderTest(unittest.TestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/files/filter.d/testcase01.conf b/fail2ban/tests/files/filter.d/testcase01.conf index 4a3a95e9..c549572d 100644 --- a/fail2ban/tests/files/filter.d/testcase01.conf +++ b/fail2ban/tests/files/filter.d/testcase01.conf @@ -32,3 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro # Values: TEXT # ignoreregex = ^.+ john from host 192.168.1.1\s*$ + +[Init] +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 1 From b47ea7f81334d8bb03f234653b446bbfe256a859 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 19 Apr 2013 18:17:56 +0100 Subject: [PATCH 076/426] ENH: Remove redundant `maxlines` option from jail reader --- fail2ban/client/jailreader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 39acc446..0599a85e 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -126,8 +126,6 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) - elif opt == "maxlines": - stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. From 4cc3a81cc1cc7ce3fd496e854a4476890a4a6cd4 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:07:23 +0100 Subject: [PATCH 077/426] TST: Move test TZ changes to setUp and tearDown methods --- bin/fail2ban-testcases | 19 +------------------ fail2ban/tests/datedetectortestcase.py | 3 +++ fail2ban/tests/filtertestcase.py | 7 +++++++ fail2ban/tests/utils.py | 19 ++++++++++++++++++- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 68a31786..cd9596a0 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -205,24 +205,7 @@ tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) # testRunner = unittest.TextTestRunner(verbosity=verbosity) -try: - # Set the time to a fixed, known value - # Sun Aug 14 12:00:00 CEST 2005 - # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match - old_TZ = os.environ.get('TZ', None) - os.environ['TZ'] = 'Europe/Zurich' - time.tzset() - MyTime.setTime(1124013600) - - tests_results = testRunner.run(tests) - -finally: # pragma: no cover - # Just for the sake of it reset the TZ - # yoh: move all this into setup/teardown methods within tests - os.environ.pop('TZ') - if old_TZ: - os.environ['TZ'] = old_TZ - time.tzset() +tests_results = testRunner.run(tests) if not tests_results.wasSuccessful(): # pragma: no cover sys.exit(1) diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 23f7a174..534abdbb 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -31,16 +31,19 @@ import unittest from fail2ban.server.datedetector import DateDetector from fail2ban.server.datetemplate import DateTemplate +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.__datedetector = DateDetector() self.__datedetector.addDefaultTemplate() def tearDown(self): """Call after every test case.""" + tearDownMyTime() def testGetEpochTime(self): log = "1138049999 [sshd] error: PAM: Authentication failure" diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fa9340ca..053086d4 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -34,6 +34,7 @@ from fail2ban.server.filterpoll import FilterPoll from fail2ban.server.filter import FileFilter, DNSUtils from fail2ban.server.failmanager import FailManager from fail2ban.server.failmanager import FailManagerEmpty +from fail2ban.tests.utils import setUpMyTime, tearDownMyTime TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -213,6 +214,7 @@ class LogFileMonitor(unittest.TestCase): """ def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') @@ -222,6 +224,7 @@ class LogFileMonitor(unittest.TestCase): self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): + tearDownMyTime() _killfile(self.file, self.name) pass @@ -363,6 +366,7 @@ def get_monitor_failures_testcase(Filter_): count = 0 def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = self.name = 'NA' self.name = '%s-%d' % (testclass_name, self.count) MonitorFailures.count += 1 # so we have unique filenames across tests @@ -380,6 +384,7 @@ def get_monitor_failures_testcase(Filter_): def tearDown(self): + tearDownMyTime() #print "D: SLEEPING A BIT" #import time; time.sleep(5) #print "D: TEARING DOWN" @@ -543,6 +548,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + setUpMyTime() self.filter = FileFilter(None) self.filter.setActive(True) # TODO Test this @@ -551,6 +557,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" + tearDownMyTime() diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 6b894193..a5d33983 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,9 +22,11 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback +import logging, os, re, traceback, time from os.path import basename, dirname +from fail2ban.server.mytime import MyTime + # # Following "traceback" functions are adopted from PyMVPA distributed # under MIT/Expat and copyright by PyMVPA developers (i.e. me and @@ -99,3 +101,18 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + +old_TZ = os.environ.get('TZ', None) +def setUpMyTime(): + # Set the time to a fixed, known value + # Sun Aug 14 12:00:00 CEST 2005 + # yoh: we need to adjust TZ to match the one used by Cyril so all the timestamps match + os.environ['TZ'] = 'Europe/Zurich' + time.tzset() + MyTime.setTime(1124013600) + +def tearDownMyTime(): + os.environ.pop('TZ') + if old_TZ: + os.environ['TZ'] = old_TZ + time.tzset() From 9e684abad7b88e93cd66962798723a72427fc819 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:13:21 +0100 Subject: [PATCH 078/426] TST: Move test gathering to function is test utils --- bin/fail2ban-testcases | 89 +--------------------------------------- fail2ban/tests/utils.py | 90 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 88 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index cd9596a0..15eb085a 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -33,16 +33,8 @@ import unittest, logging, sys, time, os if os.path.exists("fail2ban/__init__.py"): sys.path.insert(0, ".") from fail2ban.version import version -from fail2ban.tests import banmanagertestcase -from fail2ban.tests import clientreadertestcase -from fail2ban.tests import failmanagertestcase -from fail2ban.tests import filtertestcase -from fail2ban.tests import servertestcase -from fail2ban.tests import datedetectortestcase -from fail2ban.tests import actiontestcase -from fail2ban.tests import sockettestcase -from fail2ban.tests.utils import FormatterWithTraceBack +from fail2ban.tests.utils import FormatterWithTraceBack, gatherTests from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option @@ -122,84 +114,7 @@ if not opts.log_level or opts.log_level != 'fatal': # pragma: no cover print "Fail2ban %s test suite. Python %s. Please wait..." \ % (version, str(sys.version).replace('\n', '')) - -# -# Gather the tests -# -if not len(regexps): # pragma: no cover - tests = unittest.TestSuite() -else: # pragma: no cover - import re - class FilteredTestSuite(unittest.TestSuite): - _regexps = [re.compile(r) for r in regexps] - def addTest(self, suite): - suite_str = str(suite) - for r in self._regexps: - if r.search(suite_str): - super(FilteredTestSuite, self).addTest(suite) - return - - tests = FilteredTestSuite() - -# Server -#tests.addTest(unittest.makeSuite(servertestcase.StartStop)) -tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) -tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) -# FailManager -tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) -# BanManager -tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) -# ClientReaders -tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) -tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) -# CSocket and AsyncServer -tests.addTest(unittest.makeSuite(sockettestcase.Socket)) - -# Filter -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) -tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) -if not opts.no_network: - tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) - tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) -tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) - -# DateDetector -tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) - -# -# Extensive use-tests of different available filters backends -# - -from fail2ban.server.filterpoll import FilterPoll -filters = [FilterPoll] # always available - -# Additional filters available only if external modules are available -# yoh: Since I do not know better way for parametric tests -# with good old unittest -try: - from fail2ban.server.filtergamin import FilterGamin - filters.append(FilterGamin) -except Exception, e: # pragma: no cover - print "I: Skipping gamin backend testing. Got exception '%s'" % e - -try: - from fail2ban.server.filterpyinotify import FilterPyinotify - filters.append(FilterPyinotify) -except Exception, e: # pragma: no cover - print "I: Skipping pyinotify backend testing. Got exception '%s'" % e - -for Filter_ in filters: - tests.addTest(unittest.makeSuite( - filtertestcase.get_monitor_failures_testcase(Filter_))) - -# Server test for logging elements which break logging used to support -# testcases analysis -tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) - +tests = gatherTests(regexps, opts.no_network) # # Run the tests # diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index a5d33983..57814664 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,11 +22,13 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback, time +import logging, os, re, traceback, time, unittest from os.path import basename, dirname from fail2ban.server.mytime import MyTime +logSys = logging.getLogger(__name__) + # # Following "traceback" functions are adopted from PyMVPA distributed # under MIT/Expat and copyright by PyMVPA developers (i.e. me and @@ -116,3 +118,89 @@ def tearDownMyTime(): if old_TZ: os.environ['TZ'] = old_TZ time.tzset() + +from fail2ban.tests import banmanagertestcase +from fail2ban.tests import clientreadertestcase +from fail2ban.tests import failmanagertestcase +from fail2ban.tests import filtertestcase +from fail2ban.tests import servertestcase +from fail2ban.tests import datedetectortestcase +from fail2ban.tests import actiontestcase +from fail2ban.tests import sockettestcase + +def gatherTests(regexps=None, no_network=False): + if not regexps: # pragma: no cover + tests = unittest.TestSuite() + else: # pragma: no cover + import re + class FilteredTestSuite(unittest.TestSuite): + _regexps = [re.compile(r) for r in regexps] + def addTest(self, suite): + suite_str = str(suite) + for r in self._regexps: + if r.search(suite_str): + super(FilteredTestSuite, self).addTest(suite) + return + + tests = FilteredTestSuite() + + # Server + #tests.addTest(unittest.makeSuite(servertestcase.StartStop)) + tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) + tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction)) + # FailManager + tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) + # BanManager + tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + # ClientReaders + tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + # CSocket and AsyncServer + tests.addTest(unittest.makeSuite(sockettestcase.Socket)) + + # Filter + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFile)) + tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor)) + if not no_network: + tests.addTest(unittest.makeSuite(filtertestcase.GetFailures)) + tests.addTest(unittest.makeSuite(filtertestcase.DNSUtilsTests)) + tests.addTest(unittest.makeSuite(filtertestcase.JailTests)) + + # DateDetector + tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest)) + + # + # Extensive use-tests of different available filters backends + # + + from fail2ban.server.filterpoll import FilterPoll + filters = [FilterPoll] # always available + + # Additional filters available only if external modules are available + # yoh: Since I do not know better way for parametric tests + # with good old unittest + try: + from fail2ban.server.filtergamin import FilterGamin + filters.append(FilterGamin) + except Exception, e: # pragma: no cover + logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e) + + try: + from fail2ban.server.filterpyinotify import FilterPyinotify + filters.append(FilterPyinotify) + except Exception, e: # pragma: no cover + logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e) + + for Filter_ in filters: + tests.addTest(unittest.makeSuite( + filtertestcase.get_monitor_failures_testcase(Filter_))) + + # Server test for logging elements which break logging used to support + # testcases analysis + tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging)) + + return tests From 55810a3c30a745d5b099b40c42d0446b492ea995 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 20 Apr 2013 20:17:36 +0100 Subject: [PATCH 079/426] TST+RF: Add ability to execute test from setup.py with setuptools Note that the fail2ban version can no longer be imported from "fail2ban.version", as this breaks 2to3 conversion for tests --- .travis.yml | 4 +--- setup.py | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3edadda1..8cfeeff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,10 +13,8 @@ install: - pip install pyinotify - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi -before_script: - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi after_success: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi diff --git a/setup.py b/setup.py index 57b75225..4b6e304e 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,13 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from distutils.core import setup +try: + import setuptools + from setuptools import setup +except ImportError: + setuptools = None + from distutils.core import setup + try: # python 3.x from distutils.command.build_py import build_py_2to3 as build_py @@ -36,7 +42,23 @@ from os.path import isfile, join, isdir import sys from glob import glob -from fail2ban.version import version +if setuptools and "test" in sys.argv: + import logging + logSys = logging.getLogger("fail2ban") + hdlr = logging.StreamHandler(sys.stdout) + fmt = logging.Formatter("%(asctime)-15s %(message)s") + hdlr.setFormatter(fmt) + logSys.addHandler(hdlr) + if set(["-q", "--quiet"]) & set(sys.argv): + logSys.setLevel(logging.FATAL) + logging.captureWarnings(True) + elif set(["-v", "--verbose"]) & set(sys.argv): + logSys.setLevel(logging.DEBUG) + else: + logSys.setLevel(logging.INFO) +elif "test" in sys.argv: + print("python distribute required to execute fail2ban tests") + print("") longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or @@ -45,9 +67,17 @@ too many password failures. It updates firewall rules to reject the IP address or executes user defined commands.''' +if setuptools: + setup_extra = { + 'test_suite': "fail2ban.tests.utils.gatherTests", + 'use_2to3': True, + } +else: + setup_extra = {} + setup( name = "fail2ban", - version = version, + version = "0.9.0a", description = "Ban IPs that make too many password failures", long_description = longdesc, author = "Cyril Jaquier", @@ -88,7 +118,8 @@ setup( ('/usr/share/doc/fail2ban', ['README', 'DEVELOP', 'doc/run-rootless.txt'] ) - ] + ], + **setup_extra ) # Do some checks after installation From f07a92f0f72f17092f7f03128dfaae219a316b52 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 00:58:22 +0100 Subject: [PATCH 080/426] RF: setup.py now imports version number again --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b6e304e..6202fde7 100755 --- a/setup.py +++ b/setup.py @@ -75,9 +75,15 @@ if setuptools: else: setup_extra = {} +# Get version number, avoiding importing fail2ban. +# This is due to tests not functioning for python3 as 2to3 takes place later +f = open("fail2ban/version.py") +exec(f.read()) +f.close() + setup( name = "fail2ban", - version = "0.9.0a", + version = version, description = "Ban IPs that make too many password failures", long_description = longdesc, author = "Cyril Jaquier", From 7341031a30f70fc8f6dacbe6828836a3b262f0e6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 20 Apr 2013 23:12:38 -0400 Subject: [PATCH 081/426] ENH: use os.path.join for consistency -- add "Contributors" to authors --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 6202fde7..f99e4f35 100755 --- a/setup.py +++ b/setup.py @@ -18,8 +18,8 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Cyril Jaquier, Steven Hiscocks, Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2013 Fail2Ban Contributors" __license__ = "GPL" try: @@ -77,7 +77,7 @@ else: # Get version number, avoiding importing fail2ban. # This is due to tests not functioning for python3 as 2to3 takes place later -f = open("fail2ban/version.py") +f = open(join("fail2ban", "version.py")) exec(f.read()) f.close() @@ -86,7 +86,7 @@ setup( version = version, description = "Ban IPs that make too many password failures", long_description = longdesc, - author = "Cyril Jaquier", + author = "Cyril Jaquier & Fail2Ban Contributors", author_email = "cyril.jaquier@fail2ban.org", url = "http://www.fail2ban.org", license = "GPL", From 12df12f282c63be40dd6475e9d4e2f15bb702f72 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:21:54 +0100 Subject: [PATCH 082/426] BF: Change logging instance logSys `warn` method to `warning` `warn` is long time depreciated method, which may be dropped in python3.4 http://bugs.python.org/issue13235 --- bin/fail2ban-client | 2 +- fail2ban/client/beautifier.py | 2 +- fail2ban/client/configreader.py | 8 ++++---- fail2ban/client/jailreader.py | 2 +- fail2ban/server/actions.py | 4 ++-- fail2ban/server/asyncserver.py | 2 +- fail2ban/server/filter.py | 6 +++--- fail2ban/server/filterpoll.py | 4 ++-- fail2ban/server/transmitter.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 8068d60f..e0b7efc6 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -102,7 +102,7 @@ class Fail2banClient: def __sigTERMhandler(self, signum, frame): # Print a new line because we probably come from wait print - logSys.warn("Caught signal %d. Exiting" % signum) + logSys.warning("Caught signal %d. Exiting" % signum) sys.exit(-1) def __getCmdLineOptions(self, optList): diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 153ab905..a0ff8ff3 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -133,7 +133,7 @@ class Beautifier: c += 1 msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1] except Exception: - logSys.warn("Beautifier error. Please report the error") + logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify " + `response` + " with " + `self.__inputCmd` + " failed") msg = msg + `response` diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 6f1e7740..b2d3e392 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -70,7 +70,7 @@ class ConfigReader(SafeConfigParserWithIncludes): # files must carry .conf suffix as well config_files += sorted(glob.glob('%s/*.conf' % config_dir)) else: - logSys.warn("%s exists but not a directory or not accessible" + logSys.warning("%s exists but not a directory or not accessible" % config_dir) # check if files are accessible, warn if any is not accessible @@ -80,7 +80,7 @@ class ConfigReader(SafeConfigParserWithIncludes): if os.access(f, os.R_OK): config_files_accessible.append(f) else: - logSys.warn("%s exists but not accessible - skipping" % f) + logSys.warning("%s exists but not accessible - skipping" % f) if len(config_files_accessible): # at least one config exists and accessible @@ -122,11 +122,11 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] except NoOptionError: if not option[2] == None: - logSys.warn("'%s' not defined in '%s'. Using default one: %r" + logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] except ValueError: - logSys.warn("Wrong value for '" + option[1] + "' in '" + sec + + logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6e35bc0b..d8102971 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -105,7 +105,7 @@ class JailReader(ConfigReader): logSys.debug("Caught exception: %s" % (e,)) return False if not len(self.__actions): - logSys.warn("No actions were defined for %s" % self.__name) + logSys.warning("No actions were defined for %s" % self.__name) return True def convert(self): diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c0373239..b7e58975 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -177,7 +177,7 @@ class Actions(JailThread): aInfo["time"] = bTicket.getTime() aInfo["matches"] = "".join(bTicket.getMatches()) if self.__banManager.addBanTicket(bTicket): - logSys.warn("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionBan(aInfo) return True @@ -217,7 +217,7 @@ class Actions(JailThread): aInfo["failures"] = ticket.getAttempt() aInfo["time"] = ticket.getTime() aInfo["matches"] = "".join(ticket.getMatches()) - logSys.warn("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) + logSys.warning("[%s] Unban %s" % (self.jail.getName(), aInfo["ip"])) for action in self.__actions: action.execActionUnban(aInfo) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 8c905010..e6af4b26 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -135,7 +135,7 @@ class AsyncServer(asyncore.dispatcher): if os.path.exists(sock): logSys.error("Fail2ban seems to be already running") if force: - logSys.warn("Forcing execution of the server") + logSys.warning("Forcing execution of the server") os.remove(sock) else: raise AsyncServerException("Server already running") diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a7062f16..d4108dc2 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -632,7 +632,7 @@ class FileContainer: try: line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warn("Error decoding line from '%s' with '%s': %s" % + logSys.warning("Error decoding line from '%s' with '%s': %s" % (self.getFileName(), self.getEncoding(), `line`)) if sys.version_info >= (3,): # In python3, must be decoded line = line.decode(self.getEncoding(), 'ignore') @@ -668,11 +668,11 @@ class DNSUtils: try: return socket.gethostbyname_ex(dns)[2] except socket.gaierror: - logSys.warn("Unable to find a corresponding IP address for %s" + logSys.warning("Unable to find a corresponding IP address for %s" % dns) return list() except socket.error, e: - logSys.warn("Socket error raised trying to resolve hostname %s: %s" + logSys.warning("Socket error raised trying to resolve hostname %s: %s" % (dns, e)) return list() dnsToIp = staticmethod(dnsToIp) diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 3217e958..191b95b3 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -131,10 +131,10 @@ class FilterPoll(FileFilter): % (filename, e)) self.__file404Cnt[filename] += 1 if self.__file404Cnt[filename] > 2: - logSys.warn("Too many errors. Setting the jail idle") + logSys.warning("Too many errors. Setting the jail idle") if self.jail: self.jail.setIdle(True) else: - logSys.warn("No jail is assigned to %s" % self) + logSys.warning("No jail is assigned to %s" % self) self.__file404Cnt[filename] = 0 return False diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2d27ff6e..2a4514da 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -55,7 +55,7 @@ class Transmitter: ret = self.__commandHandler(command) ack = 0, ret except Exception, e: - logSys.warn("Command %r has failed. Received %r" + logSys.warning("Command %r has failed. Received %r" % (command, e)) ack = 1, e return ack From c9b1b88bfcde1ed14ca76360b161f102c184895a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:26:30 +0100 Subject: [PATCH 083/426] TST: Ensure files are closed in tests to remove ResourceWarnings --- fail2ban/tests/clientreadertestcase.py | 4 +++- fail2ban/tests/filtertestcase.py | 29 ++++++++++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d721ef00..6fb125e3 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -53,10 +53,12 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - open("%s/%s" % (self.d, fname), "w").write(""" + f = open("%s/%s" % (self.d, fname), "w") + f.write(""" [section] option = %s """ % value) + f.close() def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 053086d4..e540e41e 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -123,7 +123,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, found, output, count) -def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): """Copy lines from one file to another (which might be already open) Returns open fout @@ -132,8 +132,10 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line # on old Python st_mtime is int, so we should give at least 1 sec so # polling filter could detect the change time.sleep(1) - if isinstance(fin, str): # pragma: no branch - only used with str in test cases - fin = open(fin, 'r') + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + fin = open(in_, 'r') + else: + fin = in_ # Skip for i in xrange(skip): _ = fin.readline() @@ -151,6 +153,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line fout = open(fout, mode) fout.write('\n'.join(lines)) fout.flush() + if isinstance(in_, str): # pragma: no branch - only used with str in test cases + # Opened earlier, therefore must close it + fin.close() # to give other threads possibly some time to crunch time.sleep(0.1) return fout @@ -291,7 +296,7 @@ class LogFileMonitor(unittest.TestCase): # # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -308,6 +313,7 @@ class LogFileMonitor(unittest.TestCase): def testNewChangeViaGetFailures_move(self): # # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') self.filter.getFailures(self.name) @@ -316,7 +322,7 @@ class LogFileMonitor(unittest.TestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -454,7 +460,7 @@ def get_monitor_failures_testcase(Filter_): def test_rewrite_file(self): # if we rewrite the file at once self.file.close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # What if file gets overridden @@ -468,6 +474,7 @@ def get_monitor_failures_testcase(Filter_): def test_move_file(self): # if we move file into a new location while it has been open already + self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') # Poll might need more time @@ -477,25 +484,25 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) def test_new_bogus_file(self): # to make sure that watching whole directory does not effect - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) # create a bogus file in the same directory and see if that doesn't affect - open(self.name + '.bak2', 'w').write('') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) + open(self.name + '.bak2', 'w').close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') From 393679341385d24db09a489d06fde494480d7b3c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:28:34 +0100 Subject: [PATCH 084/426] TST: Change depreciated unittest assertEquals method to assertEqual --- fail2ban/tests/clientreadertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 6fb125e3..754f7998 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -118,7 +118,7 @@ class JailReaderTest(unittest.TestCase): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] result = JailReader.splitAction(action) - self.assertEquals(expected, result) + self.assertEqual(expected, result) class FilterReaderTest(unittest.TestCase): @@ -150,7 +150,7 @@ class FilterReaderTest(unittest.TestCase): # Add sort as configreader uses dictionary and therefore order # is unreliable - self.assertEquals(sorted(filterReader.convert()), sorted(output)) + self.assertEqual(sorted(filterReader.convert()), sorted(output)) class JailsReaderTest(unittest.TestCase): From b182c5b5d455e68c343611f095f5caa05c64052a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:30:13 +0100 Subject: [PATCH 085/426] ENH: For python3.2+ use ConfigPaser which replaces SafeConfigParser Current SafeConfigParser alias to be dropped in future python versions --- fail2ban/client/configparserinc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 0ffa0728..7b054876 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -27,8 +27,12 @@ __date__ = '$Date$' __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' -import logging, os -from ConfigParser import SafeConfigParser +import logging, os, sys +if sys.version_info >= (3,2): # pragma: no cover + # SafeConfigParser deprecitated from python 3.2 (renamed ConfigParser) + from configparser import ConfigParser as SafeConfigParser +else: # pragma: no cover + from ConfigParser import SafeConfigParser # Gets the instance of the logger. logSys = logging.getLogger(__name__) From 9d2d907fc198dffee1366bb812d184c685022916 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 10:59:14 +0100 Subject: [PATCH 086/426] BF: Remove warnings handler which breaks setup.py python2<2.7 and python3<3.2 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f99e4f35..5f65ff37 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ if setuptools and "test" in sys.argv: logSys.addHandler(hdlr) if set(["-q", "--quiet"]) & set(sys.argv): logSys.setLevel(logging.FATAL) - logging.captureWarnings(True) elif set(["-v", "--verbose"]) & set(sys.argv): logSys.setLevel(logging.DEBUG) else: From c95b87c13ceda7e08f0152982590eb93562f6861 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:21:06 +0100 Subject: [PATCH 087/426] ENH: Use os.path.join for filter/action config readers --- fail2ban/client/actionreader.py | 4 ++-- fail2ban/client/filterreader.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index b9211a1b..70c5b3a7 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +import logging, os from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. @@ -44,7 +44,7 @@ class ActionReader(OptionConfigReader): ] def read(self): - return ConfigReader.read(self, "action.d/" + self._file) + return ConfigReader.read(self, os.path.join("action.d", self._file)) def convert(self): head = ["set", self._name] diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 09f0e6a8..19a3f0f3 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +import logging, os from configreader import ConfigReader, OptionConfigReader # Gets the instance of the logger. @@ -41,7 +41,7 @@ class FilterReader(OptionConfigReader): ] def read(self): - return ConfigReader.read(self, "filter.d/" + self._file) + return ConfigReader.read(self, os.path.join("filter.d", self._file)) def convert(self): stream = list() From 1a43a0bce147ecb6bea6902c633ac1582374ab12 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:22:54 +0100 Subject: [PATCH 088/426] ENH: Rename splitAction to extractOptions in jailreader --- fail2ban/client/jailreader.py | 8 ++++---- fail2ban/tests/clientreadertestcase.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 0599a85e..4a33fdc9 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -77,7 +77,7 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - filterName, filterOpt = JailReader.splitOption( + filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( filterName, self.__name, filterOpt, basedir=self.getBaseDir()) @@ -93,7 +93,7 @@ class JailReader(ConfigReader): try: if not act: # skip empty actions continue - actName, actOpt = JailReader.splitOption(act) + actName, actOpt = JailReader.extractOptions(act) action = ActionReader( actName, self.__name, actOpt, basedir=self.getBaseDir()) ret = action.read() @@ -151,7 +151,7 @@ class JailReader(ConfigReader): return stream #@staticmethod - def splitOption(option): + def extractOptions(option): m = JailReader.optionCRE.match(option) d = dict() mgroups = m.groups() @@ -194,4 +194,4 @@ class JailReader(ConfigReader): except IndexError: logSys.error("Invalid argument %s in '%s'" % (p, option_opts)) return [option_name, d] - splitOption = staticmethod(splitOption) + extractOptions = staticmethod(extractOptions) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 3e0b555d..931a15bc 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -115,7 +115,7 @@ class JailReaderTest(unittest.TestCase): def testSplitOption(self): action = "mail-whois[name=SSH]" expected = ['mail-whois', {'name': 'SSH'}] - result = JailReader.splitOption(action) + result = JailReader.extractOptions(action) self.assertEquals(expected, result) class FilterReaderTest(unittest.TestCase): From e57505e0740aa1d3f49ba8b9c9b52a0bb45d74bc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 11:27:32 +0100 Subject: [PATCH 089/426] ENH: Renamed OptionConfigReader to DefinitionInitConfigReader --- fail2ban/client/actionreader.py | 4 ++-- fail2ban/client/configreader.py | 9 ++++++++- fail2ban/client/filterreader.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 70c5b3a7..ca4080d0 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,12 +28,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging, os -from configreader import ConfigReader, OptionConfigReader +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class ActionReader(OptionConfigReader): +class ActionReader(DefinitionInitConfigReader): _configOpts = [ ["string", "actionstart", ""], diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 4cb9cd69..45ea1361 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -131,7 +131,14 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] return values -class OptionConfigReader(ConfigReader): +class DefinitionInitConfigReader(ConfigReader): + """Config reader for files with options grouped in [Definition] and + [Init] sections. + + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 19a3f0f3..62424ddb 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -28,12 +28,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import logging, os -from configreader import ConfigReader, OptionConfigReader +from configreader import ConfigReader, DefinitionInitConfigReader # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class FilterReader(OptionConfigReader): +class FilterReader(DefinitionInitConfigReader): _configOpts = [ ["string", "ignoreregex", ""], From 6f3c66f466a9f9e5abc692d1c0c6131909d94cf3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 13:23:08 +0100 Subject: [PATCH 090/426] ENH: Reimplement warning suppression of setup.py test --quiet --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f65ff37..058c4b21 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ except ImportError: from distutils.command.build_py import build_py from distutils.command.build_scripts import build_scripts from os.path import isfile, join, isdir -import sys +import sys, warnings from glob import glob if setuptools and "test" in sys.argv: @@ -51,6 +51,8 @@ if setuptools and "test" in sys.argv: logSys.addHandler(hdlr) if set(["-q", "--quiet"]) & set(sys.argv): logSys.setLevel(logging.FATAL) + warnings.simplefilter("ignore") + sys.warnoptions.append("ignore") elif set(["-v", "--verbose"]) & set(sys.argv): logSys.setLevel(logging.DEBUG) else: From dadd6aed2f9d20a6d5dc08264433e9b4344ebbe9 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 21 Apr 2013 17:39:56 +0100 Subject: [PATCH 091/426] BF+TST: Correctly reset time in tearDownMyTime --- fail2ban/tests/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 57814664..95646d63 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -118,6 +118,7 @@ def tearDownMyTime(): if old_TZ: os.environ['TZ'] = old_TZ time.tzset() + MyTime.myTime = None from fail2ban.tests import banmanagertestcase from fail2ban.tests import clientreadertestcase From 1fcb5efbd79781da5555235a8512e74e2cfa8ca7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 00:01:30 -0400 Subject: [PATCH 092/426] ENH: make fail2ban-regex aware of possible maxlines in the filter config file --- bin/fail2ban-regex | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 9233a671..e3d75f20 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -70,6 +70,7 @@ class Fail2banRegex: self.__ignoreregex = list() self.__failregex = list() self.__verbose = False + self.__maxlines_set = False # so we allow to override maxlines in cmdline self.encoding = locale.getpreferredencoding() # Setup logging logging.getLogger("fail2ban").handlers = [] @@ -126,6 +127,11 @@ class Fail2banRegex: print "Report bugs to https://github.com/fail2ban/fail2ban/issues" dispUsage = staticmethod(dispUsage) + def setMaxLines(self, v): + if not self.__maxlines_set: + self.__filter.setMaxLines(int(v)) + self.__maxlines_set = True + def getCmdLineOptions(self, optList): """ Gets the command line options """ @@ -142,7 +148,7 @@ class Fail2banRegex: self.encoding = opt[1] elif opt[0] in ["-l", "--maxlines"]: try: - self.__filter.setMaxLines(int(opt[1])) + self.setMaxLines(opt[1]) except ValueError: print "Invlaid value for maxlines: %s" % ( opt[1]) @@ -203,6 +209,20 @@ class Fail2banRegex: print "No section headers in " + value print return False + + # Read out and set possible value of maxlines + try: + maxlines = reader.get("Init", "maxlines") + except NoSectionError, NoOptionError: + # No [Init].maxlines found. + pass + else: + try: + self.setMaxLines(maxlines) + except ValueError: + print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() + return False else: if len(value) > 53: stripReg = value[0:50] + "..." @@ -210,6 +230,8 @@ class Fail2banRegex: stripReg = value print "Use regex line : " + stripReg self.__failregex = [RegexStat(value)] + + print "Use maxlines : %d" % self.__filter.getMaxLines() return True def testIgnoreRegex(self, line): From 54bae189a355d904a7eab498a01b2b829f8ee68e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 10:10:20 -0400 Subject: [PATCH 093/426] Beef up changelog for 0.9 --- ChangeLog | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index baa854a9..e11bbde8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,15 +8,27 @@ Fail2Ban (version 0.9.0a) 20??/??/?? ================================================================================ -ver. 0.9.0 (20??/??/??) - alpha +ver. 0.9.0 (2013/04/??) - alpha ---------- -Will carry all fixes in 0.8.x series and new features and enhancements +Carries all fixes in 0.8.9 and new features and enhancements. Nearly +all development is thanks to Steven Hiscocks (THANKS!) with only +code-review and minor additions from Yaroslav Halchenko. -- Fixes: +- Refactoring: + Steven Hiscocks + * [..5aef036] Core functionality moved into fail2ban/ module. + Closes gh-26 - New features: Steven Hiscocks - * Multiline failregex. Close gh-54 + * [..c7ae460] Multiline failregex. Close gh-54 + * [8af32ed] Guacamole filter and support for Apache Tomcat date + format + * [..4869186] Python3 support +- Enhancements + Steven Hiscocks + * Replacing use of deprecated API (.warning, .assertEqual, etc) + * [..a648cc2] Filters can have options now too ver. 0.8.9 (2013/04/XXX) - wanna-be-stable ---------- From 8a0ac30bd9145f1d8747e9ddce2485bd28cb51b8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 16:59:00 -0400 Subject: [PATCH 094/426] ENH: Throw exception if requested Jail is actually not defined at all --- fail2ban/client/jailreader.py | 8 +++++++- fail2ban/tests/clientreadertestcase.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 13432dc5..9943b697 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -54,7 +54,13 @@ class JailReader(ConfigReader): return self.__name def read(self): - return ConfigReader.read(self, "jail") + out = ConfigReader.read(self, "jail") + # Before returning -- verify that requested section + # exists at all + if not (self.__name in self.sections()): + raise ValueError("Jail %r was not found among available" + % self.__name) + return out def isEnabled(self): return self.__force_enable or self.__opts["enabled"] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 5e79dc04..e55420d8 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -107,6 +107,10 @@ option = %s class JailReaderTest(unittest.TestCase): + def testIncorrectJail(self): + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR) + self.assertRaises(ValueError, jail.read) + def testStockSSHJail(self): jail = JailReader('ssh-iptables', basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jail.read()) From 3ba540eca3a2274c0ef05389367e4e15f1fe706f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 22:23:23 -0400 Subject: [PATCH 095/426] ENH+BF: use %(__name__) by default for filter, defined enabled = false by DEFAULT Now jail.conf is really neat. BF: tests --- config/jail.conf | 108 +++++++------------------ fail2ban/tests/clientreadertestcase.py | 6 +- 2 files changed, 33 insertions(+), 81 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 651ac3b3..7a869b67 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -12,7 +12,7 @@ # [DEFAULT] # bantime = 3600 # -# [ssh-iptables] +# [sshd] # enabled = true # # See jail.conf(5) man page for more information @@ -71,22 +71,30 @@ usedns = warn # auto: will use the system locale setting logencoding = auto +# "enabled" enables the jails. +# By default all jails are disabled, and it should stay this way. +# Enable only relevant to your setup jails in your .local or jail.d/*.conf +# +# Allowed values: true, false +enabled = false + + +# "filter" defines the filter to use by the jail. +# By default jails have names matching their filter name +# +filter = %(__name__)s + # # ACTIONS # -# +# Some options used for actions + # Destination email address used solely for the interpolations in # jail.{conf,local} configuration files. destemail = root@localhost -# Default banning action (e.g. iptables, iptables-new, -# iptables-multiport, shorewall, etc) It is used to define -# action_* variables. Can be overridden globally or per -# section within jail.local file -banaction = iptables-multiport - # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the # mailing. Change mta configuration parameter to mail if you want to # revert to conventional 'mail'. @@ -101,17 +109,23 @@ chain = INPUT # # Action shortcuts. To be used to define action parameter +# Default banning action (e.g. iptables, iptables-new, +# iptables-multiport, shorewall, etc) It is used to define +# action_* variables. Can be overridden globally or per +# section within jail.local file +banaction = iptables-multiport + # The simplest action to take: ban only action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -129,23 +143,18 @@ action = %(action_)s [sshd] -enabled = true -port = ssh -filter = sshd -logpath = /var/log/auth.log - /var/log/sshd.log +port = ssh +logpath = /var/log/auth.log + /var/log/sshd.log [sshd-ddos] -enabled = false -port = ssh -filter = sshd-ddos -logpath = /var/log/auth.log - /var/log/sshd.log +port = ssh +logpath = /var/log/auth.log + /var/log/sshd.log [dropbear] -enabled = false port = ssh filter = sshd logpath = /var/log/dropbear @@ -156,9 +165,7 @@ logpath = /var/log/dropbear [pam-generic] -enabled = false # pam-generic filter can be customized to monitor specific subset of 'tty's -filter = pam-generic banaction = iptables-allports # port actually must be irrelevant but lets leave it all for some possible uses port = anyport @@ -166,8 +173,6 @@ logpath = /var/log/auth.log [xinetd-fail] -enabled = false -filter = xinetd-fail port = all banaction = iptables-multiport-log logpath = /var/log/daemon.log @@ -180,7 +185,6 @@ maxretry = 2 [sshd-tcpwrapper] -enabled = false filter = sshd action = hostsdeny sendmail-whois[name=SSH, dest=you@example.com] @@ -192,7 +196,6 @@ logpath = /var/log/sshd.log [sshd-route] -enabled = false filter = sshd action = route logpath = /var/log/sshd.log @@ -204,14 +207,12 @@ logpath = /var/log/sshd.log # requires the ipset package and kernel support. [sshd-iptables-ipset4] -enabled = false filter = sshd action = iptables-ipset-proto4[name=SSH, port=ssh, protocol=tcp] logpath = /var/log/sshd.log [sshd-iptables-ipset6] -enabled = false filter = sshd action = iptables-ipset-proto6[name=SSH, port=ssh, protocol=tcp, bantime=600] logpath = /var/log/sshd.log @@ -223,7 +224,6 @@ logpath = /var/log/sshd.log [sshd-ipfw] -enabled = false filter = sshd action = ipfw[localhost=192.168.0.1] sendmail-whois[name="SSH,IPFW", dest=you@example.com] @@ -237,9 +237,7 @@ ignoreip = 168.192.0.1 [apache-auth] -enabled = false port = http,https -filter = apache-auth logpath = /var/log/apache*/*error.log # Ban hosts which agent identifies spammer robots crawling the web @@ -247,9 +245,7 @@ logpath = /var/log/apache*/*error.log [apache-badbots] -enabled = false port = http,https -filter = apache-badbots logpath = /var/log/apache*/*access.log /var/www/*/logs/access_log bantime = 172800 @@ -257,17 +253,13 @@ maxretry = 1 [apache-noscript] -enabled = false port = http,https -filter = apache-noscript logpath = /var/log/apache*/*error.log maxretry = 6 [apache-overflows] -enabled = false port = http,https -filter = apache-overflows logpath = /var/log/apache*/*error.log maxretry = 2 @@ -277,9 +269,7 @@ maxretry = 2 [php-url-fopen] -enabled = false port = http,https -filter = php-url-fopen logpath = /var/www/*/logs/access_log # A simple PHP-fastcgi jail which works with lighttpd. @@ -290,9 +280,7 @@ logpath = /var/www/*/logs/access_log [lighttpd-fastcgi] -enabled = false port = http,https -filter = lighttpd-fastcgi logpath = /var/log/lighttpd/error.log # Same as above for mod_auth @@ -300,22 +288,16 @@ logpath = /var/log/lighttpd/error.log [lighttpd-auth] -enabled = false port = http,https -filter = lighttpd-auth logpath = /var/log/lighttpd/error.log [roundcube-auth] -enabled = false port = http,https -filter = roundcube-auth logpath = /var/log/roundcube/userlogins [sogo-auth] -enabled = false -filter = sogo-auth port = http,https # without proxy this would be: # port = 20000 @@ -325,7 +307,6 @@ logpath = /var/log/sogo/sogo.log [apache-tcpwrapper] -enabled = false filter = apache-auth action = hostsdeny logpath = /var/log/apache*/*error.log @@ -339,24 +320,18 @@ maxretry = 6 [proftpd] -enabled = false port = ftp,ftp-data,ftps,ftps-data -filter = proftpd logpath = /var/log/proftpd/proftpd.log [pure-ftpd] -enabled = false port = ftp,ftp-data,ftps,ftps-data -filter = pure-ftpd logpath = /var/log/auth.log maxretry = 6 [vsftpd] -enabled = false port = ftp,ftp-data,ftps,ftps-data -filter = vsftpd logpath = /var/log/vsftpd.log # or overwrite it in jails.local to be # logpath = /var/log/auth.log @@ -368,7 +343,6 @@ logpath = /var/log/vsftpd.log [vsftpd-notification] -enabled = false filter = vsftpd action = sendmail-whois[name=VSFTPD, dest=you@example.com] logpath = /var/log/vsftpd.log @@ -378,9 +352,7 @@ bantime = 1800 [wuftpd] -enabled = false port = ftp,ftp-data,ftps,ftps-data -filter = wuftpd logpath = /var/log/syslog maxretry = 6 @@ -390,16 +362,12 @@ maxretry = 6 [couriersmtp] -enabled = false port = smtp,ssmtp -filter = couriersmtp logpath = /var/log/mail.log [postfix] -enabled = false port = smtp,ssmtp -filter = postfix logpath = /var/log/mail.log # The hosts.deny path can be defined with the "file" argument if it is @@ -407,7 +375,6 @@ logpath = /var/log/mail.log [postfix-tcpwrapper] -enabled = false filter = postfix action = hostsdeny[file=/not/a/standard/path/hosts.deny] sendmail[name=Postfix, dest=you@example.com] @@ -421,7 +388,6 @@ bantime = 300 [courierauth] -enabled = false port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s filter = courierlogin logpath = /var/log/mail.log @@ -429,9 +395,7 @@ logpath = /var/log/mail.log [sasl] -enabled = false port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s -filter = sasl # You might consider monitoring /var/log/mail.warn instead if you are # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. @@ -439,9 +403,7 @@ logpath = /var/log/mail.log [dovecot] -enabled = false port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s -filter = dovecot logpath = /var/log/mail.log # @@ -475,7 +437,6 @@ logpath = /var/log/mail.log # # [named-refused-udp] # -# enabled = false # filter = named-refused # port = domain,953 # protocol = udp @@ -486,7 +447,6 @@ logpath = /var/log/mail.log [named-refused] -enabled = false filter = named-refused port = domain,953 logpath = /var/log/named/security.log @@ -500,7 +460,6 @@ ignoreip = 168.192.0.1 # see https://github.com/fail2ban/fail2ban/issues/37 [asterisk-tcp] -enabled = false filter = asterisk port = 5060,5061 protocol = tcp @@ -509,7 +468,6 @@ maxretry = 10 [asterisk-udp] -enabled = false filter = asterisk port = 5060,5061 protocol = udp @@ -521,15 +479,11 @@ maxretry = 10 # log-warning = 2 [mysqld-auth] -enabled = false -filter = mysqld-auth port = 3306 logpath = /var/log/mysqld.log [guacamole] -enabled = false -filter = guacamole port = http,https logpath = /var/log/tomcat*/catalina.out @@ -540,8 +494,6 @@ logpath = /var/log/tomcat*/catalina.out # an infinite loop constantly feeding itself with non-informative lines [recidive] -enabled = false -filter = recidive logpath = /var/log/fail2ban.log action = iptables-allports[name=recidive] sendmail-whois-lines[name=recidive, logpath=/var/log/fail2ban.log] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e55420d8..f049c208 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -112,11 +112,11 @@ class JailReaderTest(unittest.TestCase): self.assertRaises(ValueError, jail.read) def testStockSSHJail(self): - jail = JailReader('ssh-iptables', basedir=CONFIG_DIR) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) - self.assertEqual(jail.getName(), 'ssh-iptables') + self.assertEqual(jail.getName(), 'sshd') def testSplitOption(self): action = "mail-whois[name=SSH]" @@ -195,7 +195,7 @@ class JailsReaderTest(unittest.TestCase): self.assertTrue(len(comm_commands)) # and we know even some of them by heart - for j in ['ssh-iptables', 'recidive']: + for j in ['sshd', 'recidive']: # by default we have 'auto' backend ATM self.assertTrue(['add', j, 'auto'] in comm_commands) # and warn on useDNS From 47a62b60729e13b6811694dc52a07cb6a71cb539 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 22:26:42 -0400 Subject: [PATCH 096/426] ENH: by default enable a single jail -- sshd --- config/jail.conf | 9 ++++++--- fail2ban/tests/clientreadertestcase.py | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7a869b67..f7b3ee04 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -72,10 +72,12 @@ usedns = warn logencoding = auto # "enabled" enables the jails. -# By default all jails are disabled, and it should stay this way. -# Enable only relevant to your setup jails in your .local or jail.d/*.conf +# By default all (but sshd) jails are disabled. +# It is unlikely that the default value should ever be changed. +# Enable only relevant to your setup jails in your .local or jail.d/*.conf # -# Allowed values: true, false +# true: jail will be enabled and lofiles will get monitored for changes +# false: jail is not enabled enabled = false @@ -143,6 +145,7 @@ action = %(action_)s [sshd] +enabled = true port = ssh logpath = /var/log/auth.log /var/log/sshd.log diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index f049c208..8ff45d76 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -115,7 +115,7 @@ class JailReaderTest(unittest.TestCase): jail = JailReader('sshd', basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) - self.assertFalse(jail.isEnabled()) + self.assertTrue(jail.isEnabled()) self.assertEqual(jail.getName(), 'sshd') def testSplitOption(self): @@ -181,7 +181,12 @@ class JailsReaderTest(unittest.TestCase): comm_commands = jails.convert() # by default None of the jails is enabled and we get no # commands to communicate to the server - self.assertEqual(comm_commands, []) + #self.assertEqual(comm_commands, []) + # by default now we have sshd jail enabled (only) + # so the list of commands should start with + self.assertEqual(comm_commands[0], ['add', 'sshd', 'auto']) + # and end with + self.assertEqual(comm_commands[-1], ['start', 'sshd']) def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises From f4a74d8d8b15dd8bfb860834c9ec37fc79a3ceb6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 22:42:09 -0400 Subject: [PATCH 097/426] RF: rename/unify naming of courier filters/jails --- config/filter.d/{courierlogin.conf => courier-auth.conf} | 0 config/filter.d/{couriersmtp.conf => courier-smtp.conf} | 0 config/jail.conf | 5 ++--- 3 files changed, 2 insertions(+), 3 deletions(-) rename config/filter.d/{courierlogin.conf => courier-auth.conf} (100%) rename config/filter.d/{couriersmtp.conf => courier-smtp.conf} (100%) diff --git a/config/filter.d/courierlogin.conf b/config/filter.d/courier-auth.conf similarity index 100% rename from config/filter.d/courierlogin.conf rename to config/filter.d/courier-auth.conf diff --git a/config/filter.d/couriersmtp.conf b/config/filter.d/courier-smtp.conf similarity index 100% rename from config/filter.d/couriersmtp.conf rename to config/filter.d/courier-smtp.conf diff --git a/config/jail.conf b/config/jail.conf index f7b3ee04..2021f865 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -363,7 +363,7 @@ maxretry = 6 # Mail servers # -[couriersmtp] +[courier-smtp] port = smtp,ssmtp logpath = /var/log/mail.log @@ -389,10 +389,9 @@ bantime = 300 # all relevant ports get banned # -[courierauth] +[courier-auth] port = smtp,ssmtp,imap2,imap3,imaps,pop3,pop3s -filter = courierlogin logpath = /var/log/mail.log From d05e7a57469f17bc908ab40d6b796488c4c2cfa7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2013 23:47:52 -0400 Subject: [PATCH 098/426] TST: elaborate pass through the default jail.conf with basic checking of shipped filters and actions --- fail2ban/tests/clientreadertestcase.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 8ff45d76..7746d291 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -27,6 +27,7 @@ from fail2ban.client.configreader import ConfigReader from fail2ban.client.jailreader import JailReader from fail2ban.client.filterreader import FilterReader from fail2ban.client.jailsreader import JailsReader +from fail2ban.client.actionreader import ActionReader from fail2ban.client.configurator import Configurator TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -188,6 +189,59 @@ class JailsReaderTest(unittest.TestCase): # and end with self.assertEqual(comm_commands[-1], ['start', 'sshd']) + allFilters = set() + + # All jails must have filter and action set + # TODO: evolve into a parametric test + for jail in jails.sections(): + + filterName = jails.get(jail, 'filter') + allFilters.add(filterName) + self.assertTrue(len(filterName)) + # moreover we must have a file for it + # and it must be readable as a Filter + filterReader = FilterReader(filterName, jail, {}) + filterReader.setBaseDir(CONFIG_DIR) + self.assertTrue(filterReader.read()) # opens fine + filterReader.getOptions({}) # reads fine + + # test if filter has failregex set + self.assertTrue(filterReader._opts.get('failregex', '').strip()) + + actions = jails.get(jail, 'action') + self.assertTrue(len(actions.strip())) + + # somewhat duplicating here what is done in JailsReader if + # the jail is enabled + for act in actions.split('\n'): + actName, actOpt = JailReader.extractOptions(act) + self.assertTrue(len(actName)) + self.assertTrue(isinstance(actOpt, dict)) + if actName == 'iptables-multiport': + self.assertTrue('port' in actOpt) + + actionReader = ActionReader( + actName, jail, {}, basedir=CONFIG_DIR) + self.assertTrue(actionReader.read()) + actionReader.getOptions({}) # populate _opts + cmds = actionReader.convert() + self.assertTrue(len(cmds)) + + # all must have some actionban + self.assertTrue(actionReader._opts.get('actionban', '').strip()) + + # Verify that all filters found under config/ have a jail + def get_all_confs(d): + from glob import glob + return set( + os.path.basename(x.replace('.conf', '')) + for x in glob(os.path.join(CONFIG_DIR, d, '*.conf'))) + + # TODO: provide jails for some additional filters + # ['gssftpd', 'qmail', 'apache-nohome', 'exim', 'dropbear', 'webmin-auth', 'cyrus-imap', 'sieve'] + # self.assertEqual(get_all_confs('filter.d').difference(allFilters), + # set(['common'])) + def testReadStockJailConfForceEnabled(self): # more of a smoke test to make sure that no obvious surprises # on users' systems when enabling shipped jails From 203ddb370a936024244a5b9511304e2902bca24f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 23 Apr 2013 10:04:39 -0400 Subject: [PATCH 099/426] PY3(BF): config reader -- handle __name__ interpolation --- fail2ban/client/configparserinc.py | 42 ++++++++++++++++++++++++-- fail2ban/tests/clientreadertestcase.py | 33 +++++++++++++++++--- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7b054876..de38c32b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -28,15 +28,45 @@ __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' import logging, os, sys + if sys.version_info >= (3,2): # pragma: no cover - # SafeConfigParser deprecitated from python 3.2 (renamed ConfigParser) - from configparser import ConfigParser as SafeConfigParser + + # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) + from configparser import ConfigParser as SafeConfigParser, \ + BasicInterpolation + + # And interpolation of __name__ was simply removed, thus we need to + # decorate default interpolator to handle it + class BasicInterpolationWithName(BasicInterpolation): + """Decorator to bring __name__ interpolation back. + + Original handling of __name__ was removed because of + functional deficiencies: http://bugs.python.org/issue10489 + + commit v3.2a4-105-g61f2761 + Author: Lukasz Langa + Date: Sun Nov 21 13:41:35 2010 +0000 + + Issue #10489: removed broken `__name__` support from configparser + + But should be fine to reincarnate for our use case + """ + def _interpolate_some(self, parser, option, accum, rest, section, map, + depth): + if section and not (__name__ in map): + map = map.copy() # just to be safe + map['__name__'] = section + return super(BasicInterpolationWithName, self)._interpolate_some( + parser, option, accum, rest, section, map, depth) + else: # pragma: no cover from ConfigParser import SafeConfigParser # Gets the instance of the logger. logSys = logging.getLogger(__name__) +__all__ = ['SafeConfigParserWithIncludes'] + class SafeConfigParserWithIncludes(SafeConfigParser): """ Class adds functionality to SafeConfigParser to handle included @@ -68,6 +98,14 @@ after = 1.conf SECTION_NAME = "INCLUDES" + if sys.version_info >= (3,2): + # overload constructor only for fancy new Python3's + def __init__(self, *args, **kwargs): + kwargs = kwargs.copy() + kwargs['interpolation'] = BasicInterpolationWithName() + super(SafeConfigParserWithIncludes, self).__init__( + *args, **kwargs) + #@staticmethod def getIncludes(resource, seen = []): """ diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 7746d291..d5fee47d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, shutil, tempfile, unittest +import os, shutil, sys, tempfile, unittest from fail2ban.client.configreader import ConfigReader from fail2ban.client.jailreader import JailReader @@ -47,7 +47,7 @@ class ConfigReaderTest(unittest.TestCase): """Call after every test case.""" shutil.rmtree(self.d) - def _write(self, fname, value): + def _write(self, fname, value=None, content=None): # verify if we don't need to create .d directory if os.path.sep in fname: d = os.path.dirname(fname) @@ -55,10 +55,13 @@ class ConfigReaderTest(unittest.TestCase): if not os.path.exists(d_): os.makedirs(d_) f = open("%s/%s" % (self.d, fname), "w") - f.write(""" + if value is not None: + f.write(""" [section] option = %s -""" % value) + """ % value) + if content is not None: + f.write(content) f.close() def _remove(self, fname): @@ -105,6 +108,28 @@ option = %s self._remove("c.local") self.assertEqual(self._getoption(), 1) + def testInterpolations(self): + self.assertFalse(self.c.read('i')) # nothing is there yet + self._write("i.conf", value=None, content=""" +[DEFAULT] +b = a +zz = the%(__name__)s + +[section] +y = 4%(b)s +e = 5${b} +z = %(__name__)s + +[section2] +z = 3%(__name__)s +""") + self.assertTrue(self.c.read('i')) + self.assertEqual(self.c.sections(), ['section', 'section2']) + self.assertEqual(self.c.get('section', 'y'), '4a') # basic interpolation works + self.assertEqual(self.c.get('section', 'e'), '5${b}') # no extended interpolation + self.assertEqual(self.c.get('section', 'z'), 'section') # __name__ works + self.assertEqual(self.c.get('section', 'zz'), 'thesection') # __name__ works even 'delayed' + self.assertEqual(self.c.get('section2', 'z'), '3section2') # and differs per section ;) class JailReaderTest(unittest.TestCase): From 87bac371396ee14bd37f8f40aa60c891855b3a42 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 23 Apr 2013 13:55:26 -0400 Subject: [PATCH 100/426] ENH: default port to all ports (0:65535) + remove where thus not needed + typos --- config/jail.conf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 2021f865..9750d16f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -23,7 +23,7 @@ [DEFAULT] # -# MISCELANEOUS OPTIONS +# MISCELLANEOUS OPTIONS # # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not @@ -76,7 +76,7 @@ logencoding = auto # It is unlikely that the default value should ever be changed. # Enable only relevant to your setup jails in your .local or jail.d/*.conf # -# true: jail will be enabled and lofiles will get monitored for changes +# true: jail will be enabled and log files will get monitored for changes # false: jail is not enabled enabled = false @@ -108,6 +108,10 @@ protocol = tcp # Specify chain where jumps would need to be added in iptables-* actions chain = INPUT +# Ports to be banned +# Usually should be overridden in a particular jail +port = 0:65535 + # # Action shortcuts. To be used to define action parameter @@ -170,13 +174,10 @@ logpath = /var/log/dropbear # pam-generic filter can be customized to monitor specific subset of 'tty's banaction = iptables-allports -# port actually must be irrelevant but lets leave it all for some possible uses -port = anyport logpath = /var/log/auth.log [xinetd-fail] -port = all banaction = iptables-multiport-log logpath = /var/log/daemon.log maxretry = 2 From 62602a9ed002b9a7f53fe5f458293ac60a67ef7d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 23 Apr 2013 13:58:58 -0400 Subject: [PATCH 101/426] Revert "ENH: by default enable a single jail -- sshd" This reverts commit 47a62b60729e13b6811694dc52a07cb6a71cb539. Enabling any jail by default should be a prerogative of particular distributions (thanks Fabian Wenk for the discussion) Conflicts: config/jail.conf --- config/jail.conf | 6 ++---- fail2ban/tests/clientreadertestcase.py | 9 ++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 55d8202e..c6eea0ef 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -72,9 +72,8 @@ usedns = warn logencoding = auto # "enabled" enables the jails. -# By default all (but sshd) jails are disabled. -# It is unlikely that the default value should ever be changed. -# Enable only relevant to your setup jails in your .local or jail.d/*.conf +# By default all jails are disabled, and it should stay this way. +# Enable only relevant to your setup jails in your .local or jail.d/*.conf # # true: jail will be enabled and log files will get monitored for changes # false: jail is not enabled @@ -149,7 +148,6 @@ action = %(action_)s [sshd] -enabled = true port = ssh logpath = /var/log/auth.log /var/log/sshd.log diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d5fee47d..62d994c8 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -141,7 +141,7 @@ class JailReaderTest(unittest.TestCase): jail = JailReader('sshd', basedir=CONFIG_DIR) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) - self.assertTrue(jail.isEnabled()) + self.assertFalse(jail.isEnabled()) self.assertEqual(jail.getName(), 'sshd') def testSplitOption(self): @@ -207,12 +207,7 @@ class JailsReaderTest(unittest.TestCase): comm_commands = jails.convert() # by default None of the jails is enabled and we get no # commands to communicate to the server - #self.assertEqual(comm_commands, []) - # by default now we have sshd jail enabled (only) - # so the list of commands should start with - self.assertEqual(comm_commands[0], ['add', 'sshd', 'auto']) - # and end with - self.assertEqual(comm_commands[-1], ['start', 'sshd']) + self.assertEqual(comm_commands, []) allFilters = set() From e5261aade6d6ad06cef5eeacf76c7ef14bc99920 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 24 Apr 2013 16:08:54 -0400 Subject: [PATCH 102/426] changelog for new jail.conf --- ChangeLog | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index e11bbde8..3eafa79c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,7 +15,13 @@ Carries all fixes in 0.8.9 and new features and enhancements. Nearly all development is thanks to Steven Hiscocks (THANKS!) with only code-review and minor additions from Yaroslav Halchenko. -- Refactoring: +- Refactoring (IMPORTANT -- Please review your setup and configuration): + Yaroslav Halchenko + * [..bddbf1e] jail.conf was heavily refactored and now is similar + to how it looked on Debian systems: + - default action could be configured once for all jails + - jails definitions only provide customizations (port, logpath) + - no need to specify 'filter' if name matches jail name Steven Hiscocks * [..5aef036] Core functionality moved into fail2ban/ module. Closes gh-26 From 7a29a952c133ab54ae69159636b291b70f2ae846 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 25 Apr 2013 22:29:50 +0100 Subject: [PATCH 103/426] TST: Add test case for jails with multiple of the same action --- fail2ban/tests/clientreadertestcase.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 62d994c8..1f766fc6 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -302,3 +302,26 @@ class JailsReaderTest(unittest.TestCase): configurator._Configurator__jails.setBaseDir('/tmp') self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp') self.assertEqual(configurator.getBaseDir(), CONFIG_DIR) + + def testMultipleSameAction(self): + basedir = tempfile.mkdtemp("fail2ban_conf") + os.mkdir(os.path.join(basedir, "filter.d")) + os.mkdir(os.path.join(basedir, "action.d")) + open(os.path.join(basedir, "action.d", "testaction1.conf"), 'w').close() + open(os.path.join(basedir, "filter.d", "testfilter1.conf"), 'w').close() + jailfd = open(os.path.join(basedir, "jail.conf"), 'w') + jailfd.write(""" +[testjail1] +action = testaction1[name=test1] + testaction1[name=test2] +filter = testfilter1 +""") + jailfd.close() + jails = JailsReader(basedir=basedir) + self.assertTrue(jails.read()) + self.assertTrue(jails.getOptions()) + comm_commands = jails.convert() + + action_names = [comm[-1] for comm in comm_commands if comm[:3] == ['set', 'testjail1', 'addaction']] + + self.assertNotEqual(len(set(action_names)), 1) From 45c9c45b41e6b1c6de3a2ae9764666ed515e7694 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 25 Apr 2013 22:36:08 +0100 Subject: [PATCH 104/426] BF+RF: Allow multiple of same action in a single jail --- fail2ban/client/actionreader.py | 27 +++++++++++++++++++-------- fail2ban/client/configreader.py | 10 +++++----- fail2ban/client/filterreader.py | 6 +++--- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index ca4080d0..d2d71a74 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -43,27 +43,38 @@ class ActionReader(DefinitionInitConfigReader): ["string", "actionunban", ""], ] + def __init__(self, file_, jailName, initOpts, **kwargs): + self._name = initOpts.pop("name", file_) + DefinitionInitConfigReader.__init__( + self, file_, jailName, initOpts, **kwargs) + + def setName(self, name): + self._name = name + + def getName(self): + return self._name + def read(self): return ConfigReader.read(self, os.path.join("action.d", self._file)) def convert(self): - head = ["set", self._name] + head = ["set", self._jailName] stream = list() - stream.append(head + ["addaction", self._file]) + stream.append(head + ["addaction", self._name]) for opt in self._opts: if opt == "actionstart": - stream.append(head + ["actionstart", self._file, self._opts[opt]]) + stream.append(head + ["actionstart", self._name, self._opts[opt]]) elif opt == "actionstop": - stream.append(head + ["actionstop", self._file, self._opts[opt]]) + stream.append(head + ["actionstop", self._name, self._opts[opt]]) elif opt == "actioncheck": - stream.append(head + ["actioncheck", self._file, self._opts[opt]]) + stream.append(head + ["actioncheck", self._name, self._opts[opt]]) elif opt == "actionban": - stream.append(head + ["actionban", self._file, self._opts[opt]]) + stream.append(head + ["actionban", self._name, self._opts[opt]]) elif opt == "actionunban": - stream.append(head + ["actionunban", self._file, self._opts[opt]]) + stream.append(head + ["actionunban", self._name, self._opts[opt]]) # cInfo if self._initOpts: for p in self._initOpts: - stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]]) + stream.append(head + ["setcinfo", self._name, p, self._initOpts[p]]) return stream diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 9a1fedaf..ac6af8f7 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -145,7 +145,7 @@ class DefinitionInitConfigReader(ConfigReader): def __init__(self, file_, jailName, initOpts, **kwargs): ConfigReader.__init__(self, **kwargs) self._file = file_ - self._name = jailName + self._jailName = jailName self._initOpts = initOpts def setFile(self, fileName): @@ -154,11 +154,11 @@ class DefinitionInitConfigReader(ConfigReader): def getFile(self): return self.__file - def setName(self, name): - self._name = name + def setJailName(self, jailName): + self._jailName = jailName - def getName(self): - return self._name + def getJailName(self): + return self._jailName def read(self): return ConfigReader.read(self, self._file) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 62424ddb..5369bc5f 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -50,14 +50,14 @@ class FilterReader(DefinitionInitConfigReader): for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self._name, "addfailregex", regex]) + stream.append(["set", self._jailName, "addfailregex", regex]) elif opt == "ignoreregex": for regex in self._opts[opt].split('\n'): # Do not send a command if the rule is empty. if regex != '': - stream.append(["set", self._name, "addignoreregex", regex]) + stream.append(["set", self._jailName, "addignoreregex", regex]) if self._initOpts: if 'maxlines' in self._initOpts: - stream.append(["set", self._name, "maxlines", self._initOpts["maxlines"]]) + stream.append(["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) return stream From bec70cbe4be49dafa5cf91b009186f961db4db6c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Apr 2013 19:11:58 +0100 Subject: [PATCH 105/426] TST: Clean up after jails test for MultipleSameAction --- fail2ban/tests/clientreadertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 1f766fc6..012fc2bf 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -325,3 +325,5 @@ filter = testfilter1 action_names = [comm[-1] for comm in comm_commands if comm[:3] == ['set', 'testjail1', 'addaction']] self.assertNotEqual(len(set(action_names)), 1) + + shutil.rmtree(basedir) From d07df6637071b7ffa8be988a834a6bb71d9bf932 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Apr 2013 20:06:09 +0100 Subject: [PATCH 106/426] NF: Allow setting of timeout for execution of action commands This uses subprocess.Popen, polling until `timeout` seconds has passed or the command has exit. If the command has not exited, fail2ban then sends SIGTERM, and if this is unsuccessful, SIGKILL. The timeout can be changed for an entire action via action [Init] options, or via jail.conf override, or fail2ban-client. The default timeout period is 60 seconds. --- fail2ban/client/actionreader.py | 4 +- fail2ban/protocol.py | 2 + fail2ban/server/action.py | 82 ++++++++++++++++++++++++-------- fail2ban/server/server.py | 6 +++ fail2ban/server/transmitter.py | 8 ++++ fail2ban/tests/actiontestcase.py | 9 +++- fail2ban/tests/servertestcase.py | 8 ++++ 7 files changed, 97 insertions(+), 22 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index ca4080d0..2928a37f 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -61,8 +61,10 @@ class ActionReader(DefinitionInitConfigReader): stream.append(head + ["actionban", self._file, self._opts[opt]]) elif opt == "actionunban": stream.append(head + ["actionunban", self._file, self._opts[opt]]) - # cInfo if self._initOpts: + if "timeout" in self._initOpts: + stream.append(head + ["timeout", self._file, self._opts["timeout"]]) + # cInfo for p in self._initOpts: stream.append(head + ["setcinfo", self._file, p, self._initOpts[p]]) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index cacde4c2..607cabed 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -73,6 +73,7 @@ protocol = [ ["set delaction ", "removes the action from "], ["set setcinfo ", "sets for of the action for "], ["set delcinfo ", "removes for the action for "], +["set timeout ", "sets as the command timeout in seconds for the action for "], ["set actionstart ", "sets the start command of the action for "], ["set actionstop ", "sets the stop command of the action for "], ["set actioncheck ", "sets the check command of the action for "], @@ -96,6 +97,7 @@ protocol = [ ["get actionban ", "gets the ban command for the action for "], ["get actionunban ", "gets the unban command for the action for "], ["get cinfo ", "gets the value for for the action for "], +["get timeout ", "gets the command timeout in seconds for the action for "], ] ## diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index b8356799..6c36bbca 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os +import logging, os, subprocess, time, signal import threading #from subprocess import call @@ -39,7 +39,7 @@ _cmd_lock = threading.Lock() # Some hints on common abnormal exit codes _RETCODE_HINTS = { - 0x7f00: '"Command not found". Make sure that all commands in %(realCmd)r ' + 127: '"Command not found". Make sure that all commands in %(realCmd)r ' 'are in the PATH of fail2ban-server process ' '(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). ' 'You may want to start ' @@ -48,6 +48,10 @@ _RETCODE_HINTS = { 'additional informative error messages appear in the terminals.' } +# Dictionary to lookup signal name from number +signame = dict((num, name) + for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) + ## # Execute commands. # @@ -59,6 +63,7 @@ class Action: def __init__(self, name): self.__name = name + self.__timeout = 60 self.__cInfo = dict() ## Command executed in order to initialize the system. self.__actionStart = '' @@ -88,6 +93,23 @@ class Action: def getName(self): return self.__name + ## + # Sets the timeout period for commands. + # + # @param timeout timeout period in seconds + + def setTimeout(self, timeout): + self.__timeout = int(timeout) + logSys.debug("Set action %s timeout = %i" % (self.__name, timeout)) + + ## + # Returns the action timeout period for commands. + # + # @return the timeout period in seconds + + def getTimeout(self): + return self.__timeout + ## # Sets a "CInfo". # @@ -144,7 +166,7 @@ class Action: def execActionStart(self): startCmd = Action.replaceTag(self.__actionStart, self.__cInfo) - return Action.executeCmd(startCmd) + return Action.executeCmd(startCmd, self.__timeout) ## # Set the "ban" command. @@ -240,7 +262,7 @@ class Action: def execActionStop(self): stopCmd = Action.replaceTag(self.__actionStop, self.__cInfo) - return Action.executeCmd(stopCmd) + return Action.executeCmd(stopCmd, self.__timeout) def escapeTag(tag): for c in '\\#&;`|*?~<>^()[]{}$\n\'"': @@ -294,12 +316,12 @@ class Action: return True checkCmd = Action.replaceTag(self.__actionCheck, self.__cInfo) - if not Action.executeCmd(checkCmd): + if not Action.executeCmd(checkCmd, self.__timeout): logSys.error("Invariant check failed. Trying to restore a sane" + " environment") self.execActionStop() self.execActionStart() - if not Action.executeCmd(checkCmd): + if not Action.executeCmd(checkCmd, self.__timeout): logSys.fatal("Unable to restore environment") return False @@ -312,7 +334,7 @@ class Action: # Replace static fields realCmd = Action.replaceTag(realCmd, self.__cInfo) - return Action.executeCmd(realCmd) + return Action.executeCmd(realCmd, self.__timeout) ## # Executes a command. @@ -327,27 +349,47 @@ class Action: # @return True if the command succeeded #@staticmethod - def executeCmd(realCmd): + def executeCmd(realCmd, timeout=60): logSys.debug(realCmd) _cmd_lock.acquire() try: # Try wrapped within another try needed for python version < 2.5 try: - # The following line gives deadlock with multiple jails - #retcode = call(realCmd, shell=True) - retcode = os.system(realCmd) - if retcode == 0: - logSys.debug("%s returned successfully" % realCmd) - return True - else: - msg = _RETCODE_HINTS.get(retcode, None) - logSys.error("%s returned %x" % (realCmd, retcode)) - if msg: - logSys.info("HINT on %x: %s" - % (retcode, msg % locals())) + popen = subprocess.Popen(realCmd, shell=True) + stime = time.time() + retcode = popen.poll() + while time.time() - stime <= timeout and retcode is None: + time.sleep(0.1) + retcode = popen.poll() + if retcode is None: + logSys.error("%s timed out after %i seconds." % + (realCmd, timeout)) + os.kill(popen.pid, signal.SIGTERM) # Terminate the process + time.sleep(0.1) + retcode = popen.poll() + if retcode is None: # Still going... + os.kill(popen.pid, signal.SIGKILL) # Kill the process + time.sleep(0.1) + retcode = popen.poll() except OSError, e: logSys.error("%s failed with %s" % (realCmd, e)) + return False finally: _cmd_lock.release() + + if retcode == 0: + logSys.debug("%s returned successfully" % realCmd) + return True + elif retcode is None: + logSys.error("Unable to kill PID %i: %s" % (popen.pid, realCmd)) + elif retcode < 0: + logSys.error("%s killed with %s" % + (realCmd, signame.get(-retcode, "signal %i" % -retcode))) + else: + msg = _RETCODE_HINTS.get(retcode, None) + logSys.error("%s returned %i" % (realCmd, retcode)) + if msg: + logSys.info("HINT on %i: %s" + % (retcode, msg % locals())) return False executeCmd = staticmethod(executeCmd) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 632d0546..23533fde 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -289,6 +289,12 @@ class Server: def getActionUnban(self, name, action): return self.__jails.getAction(name).getAction(action).getActionUnban() + + def setActionTimeout(self, name, action, value): + self.__jails.getAction(name).getAction(action).setTimeout(value) + + def getActionTimeout(self, name, action): + return self.__jails.getAction(name).getAction(action).getTimeout() # Status def status(self): diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2a4514da..d30809b8 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -234,6 +234,11 @@ class Transmitter: value = " ".join(command[3:]) self.__server.setActionUnban(name, act, value) return self.__server.getActionUnban(name, act) + elif command[1] == "timeout": + act = command[2] + value = int(command[3]) + self.__server.setActionTimeout(name, act, value) + return self.__server.getActionTimeout(name, act) raise Exception("Invalid command (no set action or not yet implemented)") def __commandGet(self, command): @@ -286,6 +291,9 @@ class Transmitter: act = command[2] key = command[3] return self.__server.getCInfo(name, act, key) + elif command[1] == "timeout": + act = command[2] + return self.__server.getActionTimeout(name, act) raise Exception("Invalid command (no get action or not yet implemented)") def status(self, command): diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 54c37ef0..6cac0c14 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -95,4 +95,11 @@ class ExecuteAction(unittest.TestCase): def testExecuteIncorrectCmd(self): Action.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null') - self.assertTrue(self._is_logged('HINT on 7f00: "Command not found"')) + self.assertTrue(self._is_logged('HINT on 127: "Command not found"')) + + def testExecuteTimeout(self): + stime = time.time() + Action.executeCmd('sleep 60', timeout=2) # Should take a minute + self.assertAlmostEqual(time.time() - stime, 2.1, places=1) + self.assertTrue(self._is_logged('sleep 60 timed out after 2 seconds')) + self.assertTrue(self._is_logged('sleep 60 killed with SIGTERM')) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 385b83d4..17c386ab 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -467,6 +467,14 @@ class Transmitter(TransmitterBase): self.transm.proceed( ["set", self.jailName, "delcinfo", action, "KEY"]), (0, None)) + self.assertEqual( + self.transm.proceed( + ["set", self.jailName, "timeout", action, "10"]), + (0, 10)) + self.assertEqual( + self.transm.proceed( + ["get", self.jailName, "timeout", action]), + (0, 10)) self.assertEqual( self.transm.proceed(["set", self.jailName, "delaction", action]), (0, None)) From a3e216b0b24789f4be6368ccff883f08484b958d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 27 Apr 2013 20:56:31 +0100 Subject: [PATCH 107/426] BF: Change name->actname for multi action jails to avoid clash Primary examples is `name` is used in iptables actions for the chain. Also changed pop->get so actname can be used as keyword --- fail2ban/client/actionreader.py | 2 +- fail2ban/tests/clientreadertestcase.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index d2d71a74..7ae3c45b 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -44,7 +44,7 @@ class ActionReader(DefinitionInitConfigReader): ] def __init__(self, file_, jailName, initOpts, **kwargs): - self._name = initOpts.pop("name", file_) + self._name = initOpts.get("actname", file_) DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 012fc2bf..61101aa5 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -312,8 +312,8 @@ class JailsReaderTest(unittest.TestCase): jailfd = open(os.path.join(basedir, "jail.conf"), 'w') jailfd.write(""" [testjail1] -action = testaction1[name=test1] - testaction1[name=test2] +action = testaction1[actname=test1] + testaction1[actname=test2] filter = testfilter1 """) jailfd.close() From 6d2ff47e712e2f9c621a65de0f88eb3c5590e5da Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 28 Apr 2013 13:23:06 +0100 Subject: [PATCH 108/426] NF: Allow return of list of actions from jail via fail2ban-client --- fail2ban/client/beautifier.py | 6 ++++++ fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 8 ++++++++ fail2ban/server/server.py | 3 +++ fail2ban/server/transmitter.py | 2 ++ fail2ban/tests/servertestcase.py | 6 +++++- 6 files changed, 25 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index a0ff8ff3..a92e9520 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -132,6 +132,12 @@ class Beautifier: msg = msg + "|- [" + str(c) + "]: " + ip + "\n" c += 1 msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1] + elif inC[2] == "actions": + if len(response) == 0: + msg = "No actions for jail %s" % inC[1] + else: + msg = "The jail %s has the following actions:\n" % inC[1] + msg += ", ".join(action.getName() for action in response) except Exception: logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify " + `response` + " with " + `self.__inputCmd` + diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index cacde4c2..334e7908 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -90,6 +90,7 @@ protocol = [ ["get maxretry", "gets the number of failures allowed for "], ["get maxlines", "gets the number of lines to buffer for "], ["get addaction", "gets the last action which has been added for "], +["get actions", "gets a list of actions for "], ["get actionstart ", "gets the start command for the action for "], ["get actionstop ", "gets the stop command for the action for "], ["get actioncheck ", "gets the check command for the action for "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index b7e58975..c0e6899d 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -104,6 +104,14 @@ class Actions(JailThread): self.__actions.append(action) return action + ## + # Returns the list of actions + # + # @return list of actions + + def getActions(self): + return self.__actions + ## # Set the ban time. # diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 632d0546..092867bf 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -236,6 +236,9 @@ class Server: def getLastAction(self, name): return self.__jails.getAction(name).getLastAction() + def getActions(self, name): + return self.__jails.getAction(name).getActions() + def delAction(self, name, value): self.__jails.getAction(name).delAction(value) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2a4514da..b293217e 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -265,6 +265,8 @@ class Transmitter: # Action elif command[1] == "bantime": return self.__server.getBanTime(name) + elif command[1] == "actions": + return self.__server.getActions(name) elif command[1] == "addaction": return self.__server.getLastAction(name).getName() elif command[1] == "actionstart": diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 385b83d4..67c7e6ae 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -440,8 +440,12 @@ class Transmitter(TransmitterBase): self.transm.proceed(["set", self.jailName, "addaction", action]), (0, action)) self.assertEqual( - self.transm.proceed(["get", self.jailName, "addaction", action]), + self.transm.proceed(["get", self.jailName, "addaction"]), (0, action)) + self.assertEqual( + self.transm.proceed( + ["get", self.jailName, "actions"])[1][0].getName(), + action) for cmd, value in zip(cmdList, cmdValueList): self.assertEqual( self.transm.proceed( From 219860ed8e57c022d0d911e589ef40c2c0f3b0f8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 28 Apr 2013 13:23:57 +0100 Subject: [PATCH 109/426] BF: Raise ValueError for adding of duplicate named action --- fail2ban/server/actions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index c0e6899d..0f7e6b72 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -65,6 +65,9 @@ class Actions(JailThread): # @param name The action name def addAction(self, name): + # Check is action name already exists + if name in [action.getName() for action in self.__actions]: + raise ValueError("Action %s already exists" % name) action = Action(name) self.__actions.append(action) From f196709be1d2390461fe85c6af1dee6ac6704ebf Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 29 Apr 2013 23:40:18 +0100 Subject: [PATCH 110/426] ENH: Update asterisk example jail.conf entry for multiaction --- config/jail.conf | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c6eea0ef..1e663a22 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -457,23 +457,15 @@ ignoreip = 168.192.0.1 # Miscelaneous # -# Multiple jails, 1 per protocol, are necessary ATM: -# see https://github.com/fail2ban/fail2ban/issues/37 -[asterisk-tcp] +[asterisk] -filter = asterisk port = 5060,5061 -protocol = tcp -logpath = /var/log/asterisk/messages -maxretry = 10 - -[asterisk-udp] - -filter = asterisk -port = 5060,5061 -protocol = udp logpath = /var/log/asterisk/messages maxretry = 10 +# Astrix requires both tcp and udp +action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] + %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] + %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] # To log wrong MySQL access attempts add to /etc/my.cnf: # log-error=/var/log/mysqld.log From aab9df9f909ef37d0623d39b8d5d038dc5bbde4d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 29 Apr 2013 23:41:10 +0100 Subject: [PATCH 111/426] DOC: Document use of multiple actions with `actname` in jail.conf man --- man/jail.conf.5 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index d571dc7b..3f9c0884 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -59,7 +59,15 @@ The following options are applicable to all jails. Their meaning is described in \fBbackend\fR .TP \fBusedns\fR - +.PP +Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename. In the case where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplicatione.g.: +.PP +.nf +[ssh-iptables-ipset] +enabled = true +action = sendmail[name=ssh, dest=john@example.com, actname=mail-john] + sendmail[name=ssh, dest=paul@example.com, actname=mail-paul] +.fi .SH "ACTION FILES" Action files specify which commands are executed to ban and unban an IP address. They are located under \fI/etc/fail2ban/action.d\fR. From c0e8ce5854f0a36e2d9615ed3b2f8eaf2c2d0418 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 1 May 2013 21:07:13 +0100 Subject: [PATCH 112/426] NF: For action execution, log stdout and stderr if debug or cmd error --- fail2ban/server/action.py | 16 ++++++++++++++-- fail2ban/tests/actiontestcase.py | 8 ++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 6c36bbca..dcc4dad6 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os, subprocess, time, signal +import logging, os, subprocess, time, signal, tempfile import threading #from subprocess import call @@ -353,8 +353,11 @@ class Action: logSys.debug(realCmd) _cmd_lock.acquire() try: # Try wrapped within another try needed for python version < 2.5 + stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") + stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") try: - popen = subprocess.Popen(realCmd, shell=True) + popen = subprocess.Popen( + realCmd, stdout=stdout, stderr=stderr, shell=True) stime = time.time() retcode = popen.poll() while time.time() - stime <= timeout and retcode is None: @@ -376,6 +379,15 @@ class Action: finally: _cmd_lock.release() + std_level = retcode == 0 and logging.DEBUG or logging.ERROR + if std_level >= logSys.getEffectiveLevel(): + stdout.seek(0) + logSys.log(std_level, "%s stdout: %r" % (realCmd, stdout.read())) + stderr.seek(0) + logSys.log(std_level, "%s stderr: %r" % (realCmd, stderr.read())) + stdout.close() + stderr.close() + if retcode == 0: logSys.debug("%s returned successfully" % realCmd) return True diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 6cac0c14..abeab7d6 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -103,3 +103,11 @@ class ExecuteAction(unittest.TestCase): self.assertAlmostEqual(time.time() - stime, 2.1, places=1) self.assertTrue(self._is_logged('sleep 60 timed out after 2 seconds')) self.assertTrue(self._is_logged('sleep 60 killed with SIGTERM')) + + def testCaptureStdOutErr(self): + Action.executeCmd('echo "How now brown cow"') + self.assertTrue(self._is_logged("'How now brown cow\\n'")) + Action.executeCmd( + 'echo "The rain in Spain stays mainly in the plain" 1>&2') + self.assertTrue(self._is_logged( + "'The rain in Spain stays mainly in the plain\\n'")) From 097fe63dcb32cf2e734f13e12b2aca45624cd739 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 2 May 2013 22:34:58 -0400 Subject: [PATCH 113/426] Changelog for preceeding actname merge --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index e11bbde8..18bda27c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,8 @@ code-review and minor additions from Yaroslav Halchenko. Steven Hiscocks * Replacing use of deprecated API (.warning, .assertEqual, etc) * [..a648cc2] Filters can have options now too + * [..e019ab7] Multiple instances of the same action are allowed in the + same jail -- use actname option to disambiguate. ver. 0.8.9 (2013/04/XXX) - wanna-be-stable ---------- From 93ad298aa89d9982cee988a6bd085f8b67973ae0 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 3 May 2013 00:25:42 -0400 Subject: [PATCH 114/426] Changelog entry for timeout option + fixed tabs in previous entry --- ChangeLog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 58e712e7..a696a24f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,7 +21,7 @@ code-review and minor additions from Yaroslav Halchenko. to how it looked on Debian systems: - default action could be configured once for all jails - jails definitions only provide customizations (port, logpath) - - no need to specify 'filter' if name matches jail name + - no need to specify 'filter' if name matches jail name Steven Hiscocks * [..5aef036] Core functionality moved into fail2ban/ module. Closes gh-26 @@ -31,6 +31,9 @@ code-review and minor additions from Yaroslav Halchenko. * [8af32ed] Guacamole filter and support for Apache Tomcat date format * [..4869186] Python3 support + * [..b6059f4] 'timeout' option for actions Close gh-60 and Debian bug + #410077. Also it would now capture and include stdout and stderr + into logging messages in case of error or at DEBUG loglevel. - Enhancements Steven Hiscocks * Replacing use of deprecated API (.warning, .assertEqual, etc) From 20049dd3a07778731695ab5718160ff1c156e886 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 4 May 2013 16:59:01 +0100 Subject: [PATCH 115/426] NF: Generate datetemplates from strptime format strings Can also set custom time format from fail2ban-client, replacing the default detectors --- fail2ban/client/beautifier.py | 6 ++ fail2ban/protocol.py | 2 + fail2ban/server/datedetector.py | 117 ++++++------------------- fail2ban/server/datetemplate.py | 47 +++++++++- fail2ban/server/filter.py | 35 ++++++++ fail2ban/server/server.py | 6 ++ fail2ban/server/transmitter.py | 6 ++ fail2ban/tests/datedetectortestcase.py | 2 +- 8 files changed, 130 insertions(+), 91 deletions(-) diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index a92e9520..e58f53ba 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -113,6 +113,12 @@ class Beautifier: elif inC[2] == "logencoding": msg = "Current log encoding is set to:\n" msg = msg + response + elif inC[2] == "datepattern": + msg = "Current date pattern set to: " + if response is None: + msg = msg + "Default Detectors" + else: + msg = msg + "%s (%s)" % response elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"): if len(response) == 0: msg = "No IP address/network is ignored" diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 334e7908..fb6c433d 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -64,6 +64,7 @@ protocol = [ ["set delignoreregex ", "removes the regular expression at for ignoreregex"], ["set findtime