From aec709f4c19d4c873345e184c71c5181d15b126c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 22 Jan 2013 20:54:14 +0000 Subject: [PATCH 01/17] Initial changes and test for multi-line filtering --- client/jailreader.py | 3 +++ common/protocol.py | 2 ++ man/fail2ban-client.1 | 9 ++++++++ server/failregex.py | 2 +- server/filter.py | 30 +++++++++++++++++++++++++- server/server.py | 6 ++++++ server/transmitter.py | 6 ++++++ testcases/files/testcase-multiline.log | 12 +++++++++++ testcases/filtertestcase.py | 15 +++++++++++++ 9 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 testcases/files/testcase-multiline.log diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..ca9cc5a8 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -63,6 +63,7 @@ class JailReader(ConfigReader): ["string", "logpath", "/var/log/messages"], ["string", "backend", "auto"], ["int", "maxretry", 3], + ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], @@ -114,6 +115,8 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) + elif opt == "maxlines": + stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. diff --git a/common/protocol.py b/common/protocol.py index 2f8ffa6c..de7c84a8 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -66,6 +66,7 @@ protocol = [ ["set banip ", "manually Ban for "], ["set unbanip ", "manually Unban in "], ["set maxretry ", "sets the number of failures before banning the host for "], +["set maxlines ", "sets the number of to buffer for regex search for "], ["set addaction ", "adds a new action named for "], ["set delaction ", "removes the action from "], ["set setcinfo ", "sets for of the action for "], @@ -84,6 +85,7 @@ protocol = [ ["get bantime", "gets the time a host is banned for "], ["get usedns", "gets the usedns setting for "], ["get maxretry", "gets the number of failures allowed for "], +["get maxlines", "gets the number of lines to buffer for "], ["get addaction", "gets the last action which has been added for "], ["get actionstart ", "gets the start command for the action for "], ["get actionstop ", "gets the stop command for the action for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 1bbddf09..d60c9481 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -145,6 +145,11 @@ sets the number of failures before banning the host for .TP +\fBset maxlines \fR +sets the number of to +buffer for regex search for + +.TP \fBset addaction \fR adds a new action named for @@ -222,6 +227,10 @@ gets the time a host is banned for gets the number of failures allowed for .TP +\fBget maxlines\fR +gets the number lines to +buffer for +.TP \fBget addaction\fR gets the last action which has been added for diff --git a/server/failregex.py b/server/failregex.py index 8ce9597a..c595b4de 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -51,7 +51,7 @@ class Regex: if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: - self._regexObj = re.compile(regex) + self._regexObj = re.compile(regex, re.MULTILINE) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % diff --git a/server/filter.py b/server/filter.py index b37e37e6..3ab21b17 100644 --- a/server/filter.py +++ b/server/filter.py @@ -36,6 +36,7 @@ from mytime import MyTime from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time +from collections import deque # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -71,6 +72,10 @@ class Filter(JailThread): self.__findTime = 6000 ## The ignore IP list. self.__ignoreIpList = [] + ## Size of line buffer + self.__line_buffer_size = 1 + ## Line buffer + self.__line_buffer = deque() self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -204,6 +209,25 @@ class Filter(JailThread): def getMaxRetry(self): return self.failManager.getMaxRetry() + ## + # Set the maximum line buffer size. + # + # @param value the line buffer size + + def setMaxLines(self, value): + if value < 1: + value = 1 + self.__line_buffer_size = value + logSys.info("Set maxLines = %s" % value) + + ## + # Get the maximum line buffer size. + # + # @return the line buffer size + + def getMaxLines(self): + return self.__line_buffer_size + ## # Main loop. # @@ -305,7 +329,10 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine) + self.__line_buffer.append(logLine) + while len(self.__line_buffer) > self.__line_buffer_size: + self.__line_buffer.popleft() + return self.findFailure(timeLine, "".join(self.__line_buffer)) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -365,6 +392,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: + self.__line_buffer.clear() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/server/server.py b/server/server.py index d9532be2..32d72770 100644 --- a/server/server.py +++ b/server/server.py @@ -216,6 +216,12 @@ class Server: def getMaxRetry(self, name): return self.__jails.getFilter(name).getMaxRetry() + def setMaxLines(self, name, value): + self.__jails.getFilter(name).setMaxLines(value) + + def getMaxLines(self, name): + return self.__jails.getFilter(name).getMaxLines() + # Action def addAction(self, name, value): self.__jails.getAction(name).addAction(value) diff --git a/server/transmitter.py b/server/transmitter.py index 23b609a1..23fb3eba 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -167,6 +167,10 @@ class Transmitter: value = command[2] self.__server.setMaxRetry(name, int(value)) return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + value = command[2] + self.__server.setMaxLines(name, int(value)) + return self.__server.getMaxLines(name) # command elif command[1] == "bantime": value = command[2] @@ -245,6 +249,8 @@ class Transmitter: return self.__server.getFindTime(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + return self.__server.getMaxLines(name) # Action elif command[1] == "bantime": return self.__server.getBanTime(name) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log new file mode 100644 index 00000000..b91f2756 --- /dev/null +++ b/testcases/files/testcase-multiline.log @@ -0,0 +1,12 @@ +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index c10fa78d..31d89239 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -499,6 +499,7 @@ class GetFailures(unittest.TestCase): FILENAME_03 = "testcases/files/testcase03.log" FILENAME_04 = "testcases/files/testcase04.log" FILENAME_USEDNS = "testcases/files/testcase-usedns.log" + FILENAME_MULTILINE = "testcases/files/testcase-multiline.log" # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, @@ -604,6 +605,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLine(self): + output = ("212.41.96.185", 3, 1124013598.0) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("Invalid user .+\n.+ from $") + self.filter.addIgnoreRegex("user fuck") + + self.filter.setMaxLines(2) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From 5c7e3841e095b7d0581091c8425c92ad626f8e89 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 18:26:49 +0000 Subject: [PATCH 02/17] Simplify and change some filter line buffer Include change variable names to `fail2ban` style --- server/filter.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/server/filter.py b/server/filter.py index 3ab21b17..a66a3c06 100644 --- a/server/filter.py +++ b/server/filter.py @@ -36,7 +36,6 @@ from mytime import MyTime from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time -from collections import deque # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -73,9 +72,9 @@ class Filter(JailThread): ## The ignore IP list. self.__ignoreIpList = [] ## Size of line buffer - self.__line_buffer_size = 1 + self.__lineBufferSize = 1 ## Line buffer - self.__line_buffer = deque() + self.__lineBuffer = [] self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -215,10 +214,8 @@ class Filter(JailThread): # @param value the line buffer size def setMaxLines(self, value): - if value < 1: - value = 1 - self.__line_buffer_size = value - logSys.info("Set maxLines = %s" % value) + self.__lineBufferSize = max(1, value) + logSys.info("Set maxLines = %i" % self.__lineBufferSize) ## # Get the maximum line buffer size. @@ -226,7 +223,7 @@ class Filter(JailThread): # @return the line buffer size def getMaxLines(self): - return self.__line_buffer_size + return self.__lineBufferSize ## # Main loop. @@ -329,10 +326,9 @@ class Filter(JailThread): else: timeLine = l logLine = l - self.__line_buffer.append(logLine) - while len(self.__line_buffer) > self.__line_buffer_size: - self.__line_buffer.popleft() - return self.findFailure(timeLine, "".join(self.__line_buffer)) + self.__lineBuffer = ((self.__lineBuffer + + [logLine])[-self.__lineBufferSize:]) + return self.findFailure(timeLine, "".join(self.__lineBuffer)) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -392,7 +388,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: - self.__line_buffer.clear() + self.__lineBuffer = [] try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) From 055aeeb227e129fdf56bacb3e1ee006264cd2350 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 18:42:25 +0000 Subject: [PATCH 03/17] Filter for multi-line now stores last time match This is useful for log files which dont contain a date/time on every line --- server/filter.py | 5 ++++- testcases/files/testcase-multiline.log | 20 ++++++++++---------- testcases/filtertestcase.py | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/server/filter.py b/server/filter.py index a66a3c06..4d99ae33 100644 --- a/server/filter.py +++ b/server/filter.py @@ -75,6 +75,8 @@ class Filter(JailThread): self.__lineBufferSize = 1 ## Line buffer self.__lineBuffer = [] + ## Store last time stamp, applicable for multi-line + self.__lastTimeLine = "" self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -319,12 +321,13 @@ class Filter(JailThread): if timeMatch: # Lets split into time part and log part of the line timeLine = timeMatch.group() + self.__lastTimeLine = timeLine # Lets leave the beginning in as well, so if there is no # anchore at the beginning of the time regexp, we don't # at least allow injection. Should be harmless otherwise logLine = l[:timeMatch.start()] + l[timeMatch.end():] else: - timeLine = l + timeLine = self.__lastTimeLine or l logLine = l self.__lineBuffer = ((self.__lineBuffer + [logLine])[-self.__lineBufferSize:]) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index b91f2756..12132920 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,12 +1,12 @@ -Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user duck + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto + from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user duck... Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] Invalid user duck... Aug 14 11:59:58 [sshd] from 212.41.96.185 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 31d89239..4083d231 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -608,8 +608,8 @@ class GetFailures(unittest.TestCase): def testGetFailuresMultiLine(self): output = ("212.41.96.185", 3, 1124013598.0) self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("Invalid user .+\n.+ from $") - self.filter.addIgnoreRegex("user fuck") + self.filter.addFailRegex("Invalid user .+\n.* from $") + self.filter.addIgnoreRegex("user duck") self.filter.setMaxLines(2) From 00ab42549228cca2034dd0f47ff5ece34b885c5e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 19:10:27 +0000 Subject: [PATCH 04/17] Changed multi-line test to provided example --- testcases/files/testcase-multiline.log | 36 +++++++++++++++++--------- testcases/filtertestcase.py | 9 +++---- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 12132920..70bd99ad 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,12 +1,24 @@ -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user toto - from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 -Aug 14 11:59:58 [sshd] Invalid user duck... -Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10) +Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10) +Aug 14 11:59:58 yyyy rsyncd[23864]: building file list +Aug 14 11:59:58 yyyy rsyncd[28101]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[28101]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[28101]: building file list +Aug 14 11:59:58 yyyy rsyncd[28101]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes received 65115 bytes total size 29221543998 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115 +Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[1762]: building file list +Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 115 to 116 +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error +Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error +Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[7788]: building file list +Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 +Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 4083d231..7cd4c5ec 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,12 +606,11 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = ("212.41.96.185", 3, 1124013598.0) + output = ("192.0.43.10", 1, 1124013598.0) self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("Invalid user .+\n.* from $") - self.filter.addIgnoreRegex("user duck") - - self.filter.setMaxLines(2) + self.filter.addFailRegex("rsyncd\[(?P\d+)\]: connect from .+ \(\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) From 5952819a580c9e9824a19cd7a123b8b696edfe9a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 23 Jan 2013 19:32:55 +0000 Subject: [PATCH 05/17] Sanitise testcase log 04 --- testcases/files/testcase04.log | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testcases/files/testcase04.log b/testcases/files/testcase04.log index c0304d06..987abec6 100644 --- a/testcases/files/testcase04.log +++ b/testcases/files/testcase04.log @@ -1,15 +1,15 @@ Sep 21 22:03:07 [sshd] Invalid user toto from 212.41.96.185 -1124012400 [sshd] Invalid user fuck from 212.41.96.185 +1124012400 [sshd] Invalid user duck from 212.41.96.185 Sep 21 21:03:38 [sshd] Invalid user toto from 212.41.96.185 -1124012500 [sshd] Invalid user fuck from 212.41.96.185 +1124012500 [sshd] Invalid user duck from 212.41.96.185 Sep 21 21:03:46 [sshd] Invalid user toto from 212.41.96.185 -Aug 14 11:58:48 [sshd] Invalid user fuck from 212.41.96.185 +Aug 14 11:58:48 [sshd] Invalid user duck from 212.41.96.185 Aug 14 11:59:58 [sshd] Invalid user toto from 212.41.96.185 -Sep 21 21:04:03 [sshd] Invalid user fuck from 212.41.96.185 +Sep 21 21:04:03 [sshd] Invalid user duck from 212.41.96.185 - Last output repeated twice - 2005/08/14 11:57:00 [sshd] Invalid user toto from 212.41.96.186 -2005/08/14 11:58:00 [sshd] Invalid user fuck from 212.41.96.186 +2005/08/14 11:58:00 [sshd] Invalid user duck from 212.41.96.186 2005/08/14 11:59:00 [sshd] Invalid user toto from 212.41.96.186 -2005/08/14 12:00:00 [sshd] Invalid user fuck from 212.41.96.186 +2005/08/14 12:00:00 [sshd] Invalid user duck from 212.41.96.186 - Last output repeated twice - Sep 21 21:09:01 [sshd] Invalid user toto from 212.41.96.185 From 9b4806bfd31e286e8b3271e7765c53f1c3c6445c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 24 Jan 2013 18:16:17 +0000 Subject: [PATCH 06/17] Added regex applicable for multi-line This allows lines captured by regex to remain in the line buffer in Filter --- server/failregex.py | 23 +++++++++++++++++++++++ server/filter.py | 2 +- testcases/files/testcase-multiline.log | 6 +++++- testcases/filtertestcase.py | 8 +++++--- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index c595b4de..84b237c7 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -48,6 +48,11 @@ class Regex: # Perform shortcuts expansions. # Replace "" with default regular expression for host. regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]+)") + # Replace "" with regular expression for multiple lines. + regexSplit = regex.split("") + regex = regexSplit[0] + for n, regexLine in enumerate(regexSplit[1:]): + regex += "\n(?P(?:(.*\n)*?))" % n + regexLine if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: @@ -131,3 +136,21 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host + + ## + # Returns unmatched lines. + # + # This returns unmatched lines inlcuding captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + unmatchedLines = self._matchCache.string[:self._matchCache.start()] + n = 0 + while True: + try: + unmatchedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + unmatchedLines += self._matchCache.string[self._matchCache.end():] + return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 4d99ae33..0b0e0535 100644 --- a/server/filter.py +++ b/server/filter.py @@ -391,7 +391,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: - self.__lineBuffer = [] + self.__lineBuffer = failRegex.getUnmatchedLines() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 70bd99ad..c0151d34 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -19,6 +19,10 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test co Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: building file list +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 7cd4c5ec..262074e6 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,15 +606,17 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = ("192.0.43.10", 1, 1124013598.0) + output = [("192.0.43.10", 1, 1124013598.0), + ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("rsyncd\[(?P\d+)\]: connect from .+ \(\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error") + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output) + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) From 28f68a693f8f042497cd6a18de49bd9c4ff85a54 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 24 Jan 2013 21:12:45 +0000 Subject: [PATCH 07/17] Minor typo in server/failregex.py --- server/failregex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/failregex.py b/server/failregex.py index 84b237c7..b7084ce2 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -140,7 +140,7 @@ class FailRegex(Regex): ## # Returns unmatched lines. # - # This returns unmatched lines inlcuding captured by the tag. + # This returns unmatched lines including captured by the tag. # @return list of unmatched lines def getUnmatchedLines(self): From ea466d59f478c18e4a3695032499b3c390d93222 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:11:40 +0000 Subject: [PATCH 08/17] ignoreregex now functions correctly with multiline Ignore regexs are now only compared to lines that match the failregex. Supporting test also added for multiline regex and overlapping multiline regex matches. --- server/failregex.py | 66 +++++++++++++++++++------- server/filter.py | 9 ++-- testcases/files/testcase-multiline.log | 7 ++- testcases/filtertestcase.py | 16 ++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index b7084ce2..d98ffa27 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -93,6 +93,54 @@ class Regex: else: return False + ## + # Returns skipped lines. + # + # This returns skipped lines captured by the tag. + # @return list of skipped lines + + def getSkippedLines(self): + if not self._matchCache: + return [] + skippedLines = "" + n = 0 + while True: + try: + skippedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + return skippedLines.splitlines(True) + + ## + # Returns unmatched lines. + # + # This returns unmatched lines including captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + if not self._matchCache: + return [] + unmatchedLines = ( + self._matchCache.string[:self._matchCache.start()].splitlines(True) + + self.getSkippedLines() + + self._matchCache.string[self._matchCache.end():].splitlines(True)) + return unmatchedLines + + ## + # Returns matched lines. + # + # This returns matched lines by excluding those captured + # by the tag. + # @return list of matched lines + + def getMatchedLines(self): + if not self._matchCache: + return [] + matchedLines = self._matchCache.string[ + self._matchCache.start():self._matchCache.end()].splitlines(True) + return [line for line in matchedLines + if line not in self.getSkippedLines()] ## # Exception dedicated to the class Regex. @@ -136,21 +184,3 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host - - ## - # Returns unmatched lines. - # - # This returns unmatched lines including captured by the tag. - # @return list of unmatched lines - - def getUnmatchedLines(self): - unmatchedLines = self._matchCache.string[:self._matchCache.start()] - n = 0 - while True: - try: - unmatchedLines += self._matchCache.group("skiplines%i" % n) - n += 1 - except IndexError: - break - unmatchedLines += self._matchCache.string[self._matchCache.end():] - return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 0b0e0535..5f2f0f2b 100644 --- a/server/filter.py +++ b/server/filter.py @@ -374,14 +374,15 @@ class Filter(JailThread): def findFailure(self, timeLine, logLine): failList = list() - # Checks if we must ignore this line. - if self.ignoreLine(logLine): - # The ignoreregex matched. Return. - return failList # Iterates over all the regular expressions. for failRegex in self.__failRegex: failRegex.search(logLine) if failRegex.hasMatched(): + # Checks if we must ignore this match. + if self.ignoreLine("".join(failRegex.getMatchedLines())): + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer = failRegex.getUnmatchedLines() + continue # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) if date == None: diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index c0151d34..69dac361 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,3 +1,4 @@ +Aug 14 11:58:58 yyyy rsyncd[9874]: connect from example.com (192.0.43.10) Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: building file list @@ -9,6 +10,7 @@ Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes received 65115 bytes Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115 Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) + Aug 14 11:59:58 yyyy rsyncd[1762]: building file list Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. @@ -16,6 +18,8 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error + + Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list @@ -24,5 +28,6 @@ Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (c Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: Received SIGINT Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:59 yyyy rsyncd[9874]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 262074e6..4e60c242 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,7 +606,7 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 1, 1124013598.0), + output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -620,6 +620,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLineIgnoreRegex(self): + output = [("192.0.43.10", 2, 1124013599.0)] + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addIgnoreRegex("rsync error: Received SIGINT") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output.pop()) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From 7234c2a3aa07faa8dea755b973e835b13e8a37d6 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:16:55 +0000 Subject: [PATCH 09/17] Added multiregex test for multi-line filter --- testcases/files/testcase-multiline.log | 2 ++ testcases/filtertestcase.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 69dac361..a8d977ab 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -13,6 +13,7 @@ Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[1762]: building file list Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:58 yyyy sendmail[30222]: r0NNNlC0030222: from=, size=6420, class=0, nrcpts=1, msgid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=[192.0.43.15] (may be forged) Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 115 to 116 Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. @@ -25,6 +26,7 @@ Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy spamd[19119]: spamd: result: Y 11 - AWL,BAYES_50,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE,RCVD_IN_BRBL_LASTEXT,RCVD_IN_PSBL,RCVD_IN_RP_RNBL,RDNS_NONE,URIBL_BLACK,URIBL_DBL_SPAM scantime=1.2,size=6910,user=sa-milt,uid=499,required_score=5.0,rhost=localhost,raddr=127.0.0.1,rport=57429,mid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>,bayes=0.536244,autolearn=no Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: building file list diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 4e60c242..f61abacb 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -634,6 +634,24 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLineMultiRegex(self): + output = [("192.0.43.10", 2, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0), + ("192.0.43.15", 1, 1124013598.0)] + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self): From d05f42075811f4299954f1d74ee913b42fed465e Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:28:48 +0000 Subject: [PATCH 10/17] Added FilterReader test --- fail2ban-testcases | 1 + testcases/clientreadertestcase.py | 37 +++++++++++++++ testcases/files/filter.d/testcase-common.conf | 47 +++++++++++++++++++ testcases/files/filter.d/testcase01.conf | 34 ++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 testcases/files/filter.d/testcase-common.conf create mode 100644 testcases/files/filter.d/testcase01.conf diff --git a/fail2ban-testcases b/fail2ban-testcases index aaf78525..3ea3f413 100755 --- a/fail2ban-testcases +++ b/fail2ban-testcases @@ -115,6 +115,7 @@ tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) # ClientReader tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) +tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) # Filter tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP)) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 83121345..058ed5dd 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -29,6 +29,8 @@ __license__ = "GPL" import unittest from client.jailreader import JailReader +from client.configreader import ConfigReader +from client.filterreader import FilterReader class JailReaderTest(unittest.TestCase): @@ -44,3 +46,38 @@ class JailReaderTest(unittest.TestCase): result = JailReader.splitAction(action) self.assertEquals(expected, result) +class FilterReaderTest(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + ConfigReader.setBaseDir("testcases/files/") + + def tearDown(self): + """Call after every test case.""" + + def testConvert(self): + output = [['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?Authentication failure for .* from \\s*$"], + ['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?User not known to the underlying authentication mo" + "dule for .* from \\s*$"], + ['set', 'testcase01', 'addfailregex', + "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" + "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" + "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" + "error: PAM: )?User not known to the\\nunderlying authentication." + "+$^.+ module for .* from \\s*$"], + ['set', 'testcase01', 'addignoreregex', + "^.+ john from host 192.168.1.1\\s*$"]] + filterReader = FilterReader("testcase01", "testcase01") + filterReader.read() + #filterReader.getOptions(["failregex", "ignoreregex"]) + filterReader.getOptions(None) + + self.assertEquals(filterReader.convert(), output) diff --git a/testcases/files/filter.d/testcase-common.conf b/testcases/files/filter.d/testcase-common.conf new file mode 100644 index 00000000..18bf41c5 --- /dev/null +++ b/testcases/files/filter.d/testcase-common.conf @@ -0,0 +1,47 @@ +# Generic configuration items (to be used as interpolations) in other +# filters or actions configurations +# +# Author: Yaroslav Halchenko +# +# $Revision$ +# + +[INCLUDES] + +# Load customizations if any available +after = common.local + + +[DEFAULT] + +# Daemon definition is to be specialized (if needed) in .conf file +_daemon = \S* + +# +# Shortcuts for easier comprehension of the failregex +# +# PID. +# EXAMPLES: [123] +__pid_re = (?:\[\d+\]) + +# Daemon name (with optional source_file:line or whatever) +# EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix) +__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? + +# Combinations of daemon name and PID +# EXAMPLES: sshd[31607], pop(pam_unix)[4920] +__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:) + +# Some messages have a kernel prefix with a timestamp +# EXAMPLES: kernel: [769570.846956] +__kernel_prefix = kernel: \[\d+\.\d+\] + +__hostname = \S+ + +# +# Common line prefixes (beginnings) which could be used in filters +# +# [hostname] [vserver tag] daemon_id spaces +# this can be optional (for instance if we match named native log files) +__prefix_line = \s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s* + diff --git a/testcases/files/filter.d/testcase01.conf b/testcases/files/filter.d/testcase01.conf new file mode 100644 index 00000000..4a3a95e9 --- /dev/null +++ b/testcases/files/filter.d/testcase01.conf @@ -0,0 +1,34 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# +# $Revision$ +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + + +[Definition] + +_daemon = sshd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from \s*$ + ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ + ^%(__prefix_line)s(?:error: PAM: )?User not known to the\nunderlying authentication.+$^.+ module for .* from \s*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = ^.+ john from host 192.168.1.1\s*$ From 99914ac0f3d7af67f88b4ffb1c4413bb4c72871d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Jan 2013 09:17:48 +0000 Subject: [PATCH 11/17] Regex get(Un)MatchedLines now returns whole lines only Fix issue where for regexs not anchored at start/end of line, that getMatchedLines and getUnmatchedLines returned partial lines --- server/failregex.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index d98ffa27..608d5652 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -81,6 +81,19 @@ class Regex: def search(self, value): self._matchCache = self._regexObj.search(value) + if self.hasMatched(): + # Find start of the first line where the match was found + try: + self._matchLineStart = self._matchCache.string.rindex( + "\n", 0, self._matchCache.start() +1 ) + 1 + except ValueError: + self._matchLineStart = 0 + # Find end of the last line where the match was found + try: + self._matchLineEnd = self._matchCache.string.index( + "\n", self._matchCache.end() - 1) + 1 + except ValueError: + self._matchLineEnd = len(self._matchCache.string) ## # Checks if the previous call to search() matched. @@ -119,12 +132,12 @@ class Regex: # @return list of unmatched lines def getUnmatchedLines(self): - if not self._matchCache: + if not self.hasMatched(): return [] unmatchedLines = ( - self._matchCache.string[:self._matchCache.start()].splitlines(True) + self._matchCache.string[:self._matchLineStart].splitlines(True) + self.getSkippedLines() - + self._matchCache.string[self._matchCache.end():].splitlines(True)) + + self._matchCache.string[self._matchLineEnd:].splitlines(True)) return unmatchedLines ## @@ -135,10 +148,10 @@ class Regex: # @return list of matched lines def getMatchedLines(self): - if not self._matchCache: + if not self.hasMatched(): return [] matchedLines = self._matchCache.string[ - self._matchCache.start():self._matchCache.end()].splitlines(True) + self._matchLineStart:self._matchLineEnd].splitlines(True) return [line for line in matchedLines if line not in self.getSkippedLines()] From b48c17b8c49aed67f0979b462d7c5e0dead211ef Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Jan 2013 10:41:58 +0000 Subject: [PATCH 12/17] Added 'maxlines' option to fail2ban-regex This allows multi-line regex to be tested --- fail2ban-regex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 3900c909..4e5698ab 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -105,6 +105,7 @@ class Fail2banRegex: print " -h, --help display this help message" print " -V, --version print the version" print " -v, --verbose verbose output" + print " -l INT, --maxlines=INT set maxlines for multi-line regex default: 1" print print "Log:" print " string a string representing a log line" @@ -133,6 +134,14 @@ class Fail2banRegex: sys.exit(0) elif opt[0] in ["-v", "--verbose"]: self.__verbose = True + elif opt[0] in ["-l", "--maxlines"]: + try: + self.__filter.setMaxLines(int(opt[1])) + except ValueError: + print "Invlaid value for maxlines: %s" % ( + opt[1]) + fail2banRegex.dispUsage() + sys.exit(-1) #@staticmethod def logIsFile(value): @@ -310,8 +319,8 @@ if __name__ == "__main__": fail2banRegex = Fail2banRegex() # Reads the command line options. try: - cmdOpts = 'hVcv' - cmdLongOpts = ['help', 'version', 'verbose'] + cmdOpts = 'hVcvl:' + cmdLongOpts = ['help', 'version', 'verbose', 'maxlines='] optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: fail2banRegex.dispUsage() From 02218294bc0d012e22cec7d4c33c810df187a2be Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 28 Jan 2013 18:41:12 +0000 Subject: [PATCH 13/17] Removed "common.local" include for FilterReader test --- testcases/files/filter.d/testcase-common.conf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/testcases/files/filter.d/testcase-common.conf b/testcases/files/filter.d/testcase-common.conf index 18bf41c5..af7df1f7 100644 --- a/testcases/files/filter.d/testcase-common.conf +++ b/testcases/files/filter.d/testcase-common.conf @@ -6,12 +6,6 @@ # $Revision$ # -[INCLUDES] - -# Load customizations if any available -after = common.local - - [DEFAULT] # Daemon definition is to be specialized (if needed) in .conf file From efea62e03f9da6e277d9a60a5af9e3df87f692ec Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 28 Jan 2013 20:47:32 +0000 Subject: [PATCH 14/17] Revert changes to man/fail2ban-client.1 --- man/fail2ban-client.1 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index d60c9481..1bbddf09 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -145,11 +145,6 @@ sets the number of failures before banning the host for .TP -\fBset maxlines \fR -sets the number of to -buffer for regex search for - -.TP \fBset addaction \fR adds a new action named for @@ -227,10 +222,6 @@ gets the time a host is banned for gets the number of failures allowed for .TP -\fBget maxlines\fR -gets the number lines to -buffer for -.TP \fBget addaction\fR gets the last action which has been added for From 9dc662af2794f11ab3d1dc1e68c14b87038f486e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Feb 2013 16:00:05 -0500 Subject: [PATCH 15/17] Introducing 0.9.x series with 0.9.0a0 0.9.0a0 is chosen so that StrictVersion works within python 2.x --- ChangeLog | 12 +++++++++++- README | 2 +- common/version.py | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index eabd6e81..59533c64 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,19 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.8) 2012/12/06 +Fail2Ban (version 0.9.0a) 20??/??/?? ================================================================================ + +ver. 0.9.0 (20??/??/??) - alpha +---------- + +Will carry all fixes in 0.8.x series and new features and enhancements + +- Fixes: +- New features: +- Enhancements: + ver. 0.8.8 (2012/12/06) - stable ---------- - Fixes: diff --git a/README b/README index db97aa8b..0a3d5117 100644 --- a/README +++ b/README @@ -4,7 +4,7 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.8.8) 2012/07/31 +Fail2Ban (version 0.9.0a0) 20??/??/?? ================================================================================ Fail2Ban scans log files like /var/log/pwdfail and bans IP that makes too many diff --git a/common/version.py b/common/version.py index 2a1c0d0b..a4499b21 100644 --- a/common/version.py +++ b/common/version.py @@ -22,7 +22,7 @@ # $Revision$ __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -version = "0.8.8" +version = "0.9.0a0" From 4d4c2d7e0209bd1d7c93ac936df2f19441f228d9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Feb 2013 16:04:44 -0500 Subject: [PATCH 16/17] Brief changelog entry for multiline failregex. With this Close gh-54 --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 59533c64..e5979331 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ Will carry all fixes in 0.8.x series and new features and enhancements - Fixes: - New features: + Steven Hiscocks + * Multiline failregex. Close gh-54 - Enhancements: ver. 0.8.8 (2012/12/06) - stable From e43fcc80dbfb1c9b70ccbb200b691894b22328ca Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 30 Mar 2013 18:30:23 -0400 Subject: [PATCH 17/17] BF: setBaseDir is not static method now -- so set it for the filterReader in question --- testcases/clientreadertestcase.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/testcases/clientreadertestcase.py b/testcases/clientreadertestcase.py index 21dd4745..8eccb2b9 100644 --- a/testcases/clientreadertestcase.py +++ b/testcases/clientreadertestcase.py @@ -113,13 +113,6 @@ class JailReaderTest(unittest.TestCase): class FilterReaderTest(unittest.TestCase): - def setUp(self): - """Call before every test case.""" - ConfigReader.setBaseDir("testcases/files/") - - def tearDown(self): - """Call after every test case.""" - def testConvert(self): output = [['set', 'testcase01', 'addfailregex', "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" @@ -141,6 +134,7 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addignoreregex', "^.+ john from host 192.168.1.1\\s*$"]] filterReader = FilterReader("testcase01", "testcase01") + filterReader.setBaseDir("testcases/files/") filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None)