From aec709f4c19d4c873345e184c71c5181d15b126c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 22 Jan 2013 20:54:14 +0000 Subject: [PATCH] Initial changes and test for multi-line filtering --- client/jailreader.py | 3 +++ common/protocol.py | 2 ++ man/fail2ban-client.1 | 9 ++++++++ server/failregex.py | 2 +- server/filter.py | 30 +++++++++++++++++++++++++- server/server.py | 6 ++++++ server/transmitter.py | 6 ++++++ testcases/files/testcase-multiline.log | 12 +++++++++++ testcases/filtertestcase.py | 15 +++++++++++++ 9 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 testcases/files/testcase-multiline.log diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..ca9cc5a8 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -63,6 +63,7 @@ class JailReader(ConfigReader): ["string", "logpath", "/var/log/messages"], ["string", "backend", "auto"], ["int", "maxretry", 3], + ["int", "maxlines", 1], ["int", "findtime", 600], ["int", "bantime", 600], ["string", "usedns", "warn"], @@ -114,6 +115,8 @@ class JailReader(ConfigReader): backend = self.__opts[opt] elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) + elif opt == "maxlines": + stream.append(["set", self.__name, "maxlines", self.__opts[opt]]) elif opt == "ignoreip": for ip in self.__opts[opt].split(): # Do not send a command if the rule is empty. diff --git a/common/protocol.py b/common/protocol.py index 2f8ffa6c..de7c84a8 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -66,6 +66,7 @@ protocol = [ ["set banip ", "manually Ban for "], ["set unbanip ", "manually Unban in "], ["set maxretry ", "sets the number of failures before banning the host for "], +["set maxlines ", "sets the number of to buffer for regex search for "], ["set addaction ", "adds a new action named for "], ["set delaction ", "removes the action from "], ["set setcinfo ", "sets for of the action for "], @@ -84,6 +85,7 @@ protocol = [ ["get bantime", "gets the time a host is banned for "], ["get usedns", "gets the usedns setting for "], ["get maxretry", "gets the number of failures allowed for "], +["get maxlines", "gets the number of lines to buffer for "], ["get addaction", "gets the last action which has been added for "], ["get actionstart ", "gets the start command for the action for "], ["get actionstop ", "gets the stop command for the action for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 1bbddf09..d60c9481 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -145,6 +145,11 @@ sets the number of failures before banning the host for .TP +\fBset maxlines \fR +sets the number of to +buffer for regex search for + +.TP \fBset addaction \fR adds a new action named for @@ -222,6 +227,10 @@ gets the time a host is banned for gets the number of failures allowed for .TP +\fBget maxlines\fR +gets the number lines to +buffer for +.TP \fBget addaction\fR gets the last action which has been added for diff --git a/server/failregex.py b/server/failregex.py index 8ce9597a..c595b4de 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -51,7 +51,7 @@ class Regex: if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: - self._regexObj = re.compile(regex) + self._regexObj = re.compile(regex, re.MULTILINE) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % diff --git a/server/filter.py b/server/filter.py index b37e37e6..3ab21b17 100644 --- a/server/filter.py +++ b/server/filter.py @@ -36,6 +36,7 @@ from mytime import MyTime from failregex import FailRegex, Regex, RegexException import logging, re, os, fcntl, time +from collections import deque # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -71,6 +72,10 @@ class Filter(JailThread): self.__findTime = 6000 ## The ignore IP list. self.__ignoreIpList = [] + ## Size of line buffer + self.__line_buffer_size = 1 + ## Line buffer + self.__line_buffer = deque() self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -204,6 +209,25 @@ class Filter(JailThread): def getMaxRetry(self): return self.failManager.getMaxRetry() + ## + # Set the maximum line buffer size. + # + # @param value the line buffer size + + def setMaxLines(self, value): + if value < 1: + value = 1 + self.__line_buffer_size = value + logSys.info("Set maxLines = %s" % value) + + ## + # Get the maximum line buffer size. + # + # @return the line buffer size + + def getMaxLines(self): + return self.__line_buffer_size + ## # Main loop. # @@ -305,7 +329,10 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine) + self.__line_buffer.append(logLine) + while len(self.__line_buffer) > self.__line_buffer_size: + self.__line_buffer.popleft() + return self.findFailure(timeLine, "".join(self.__line_buffer)) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -365,6 +392,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: + self.__line_buffer.clear() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/server/server.py b/server/server.py index d9532be2..32d72770 100644 --- a/server/server.py +++ b/server/server.py @@ -216,6 +216,12 @@ class Server: def getMaxRetry(self, name): return self.__jails.getFilter(name).getMaxRetry() + def setMaxLines(self, name, value): + self.__jails.getFilter(name).setMaxLines(value) + + def getMaxLines(self, name): + return self.__jails.getFilter(name).getMaxLines() + # Action def addAction(self, name, value): self.__jails.getAction(name).addAction(value) diff --git a/server/transmitter.py b/server/transmitter.py index 23b609a1..23fb3eba 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -167,6 +167,10 @@ class Transmitter: value = command[2] self.__server.setMaxRetry(name, int(value)) return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + value = command[2] + self.__server.setMaxLines(name, int(value)) + return self.__server.getMaxLines(name) # command elif command[1] == "bantime": value = command[2] @@ -245,6 +249,8 @@ class Transmitter: return self.__server.getFindTime(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) + elif command[1] == "maxlines": + return self.__server.getMaxLines(name) # Action elif command[1] == "bantime": return self.__server.getBanTime(name) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log new file mode 100644 index 00000000..b91f2756 --- /dev/null +++ b/testcases/files/testcase-multiline.log @@ -0,0 +1,12 @@ +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user toto... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 +Aug 14 11:59:58 [sshd] Invalid user fuck... +Aug 14 11:59:58 [sshd] from 212.41.96.185 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index c10fa78d..31d89239 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -499,6 +499,7 @@ class GetFailures(unittest.TestCase): FILENAME_03 = "testcases/files/testcase03.log" FILENAME_04 = "testcases/files/testcase04.log" FILENAME_USEDNS = "testcases/files/testcase-usedns.log" + FILENAME_MULTILINE = "testcases/files/testcase-multiline.log" # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, @@ -604,6 +605,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLine(self): + output = ("212.41.96.185", 3, 1124013598.0) + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("Invalid user .+\n.+ from $") + self.filter.addIgnoreRegex("user fuck") + + self.filter.setMaxLines(2) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self):