Initial changes and test for multi-line filtering

client/jailreader.py

@@ -63,6 +63,7 @@ class JailReader(ConfigReader):
 				["string", "logpath", "/var/log/messages"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
+				["int", "maxlines", 1],
 				["int", "findtime", 600],
 				["int", "bantime", 600],
 				["string", "usedns", "warn"],
@@ -114,6 +115,8 @@ class JailReader(ConfigReader):
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
 				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
+			elif opt == "maxlines":
+				stream.append(["set", self.__name, "maxlines", self.__opts[opt]])
 			elif opt == "ignoreip":
 				for ip in self.__opts[opt].split():
 					# Do not send a command if the rule is empty.

common/protocol.py

@@ -66,6 +66,7 @@ protocol = [
 ["set <JAIL> banip <IP>", "manually Ban <IP> for <JAIL>"], 
 ["set <JAIL> unbanip <IP>", "manually Unban <IP> in <JAIL>"], 
 ["set <JAIL> maxretry <RETRY>", "sets the number of failures <RETRY> before banning the host for <JAIL>"], 
+["set <JAIL> maxlines <LINES>", "sets the number of <LINES> to buffer for regex search for <JAIL>"], 
 ["set <JAIL> addaction <ACT>", "adds a new action named <NAME> for <JAIL>"], 
 ["set <JAIL> delaction <ACT>", "removes the action <NAME> from <JAIL>"], 
 ["set <JAIL> setcinfo <ACT> <KEY> <VALUE>", "sets <VALUE> for <KEY> of the action <NAME> for <JAIL>"], 
@@ -84,6 +85,7 @@ protocol = [
 ["get <JAIL> bantime", "gets the time a host is banned for <JAIL>"],
 ["get <JAIL> usedns", "gets the usedns setting for <JAIL>"],
 ["get <JAIL> maxretry", "gets the number of failures allowed for <JAIL>"],
+["get <JAIL> maxlines", "gets the number of lines to buffer for <JAIL>"],
 ["get <JAIL> addaction", "gets the last action which has been added for <JAIL>"],
 ["get <JAIL> actionstart <ACT>", "gets the start command for the action <ACT> for <JAIL>"],
 ["get <JAIL> actionstop <ACT>", "gets the stop command for the action <ACT> for <JAIL>"],

man/fail2ban-client.1

@@ -145,6 +145,11 @@ sets the number of failures
 <RETRY> before banning the host
 for <JAIL>
 .TP
+\fBset <JAIL> maxlines <LINES>\fR
+sets the number of <LINES> to
+buffer for regex search for
+<JAIL>
+.TP
 \fBset <JAIL> addaction <ACT>\fR
 adds a new action named <NAME> for
 <JAIL>
@@ -222,6 +227,10 @@ gets the time a host is banned for
 gets the number of failures
 allowed for <JAIL>
 .TP
+\fBget <JAIL> maxlines\fR
+gets the number lines to
+buffer for <JAIL>
+.TP
 \fBget <JAIL> addaction\fR
 gets the last action which has
 been added for <JAIL>

server/failregex.py

@@ -51,7 +51,7 @@ class Regex:
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
 		try:
-			self._regexObj = re.compile(regex)
+			self._regexObj = re.compile(regex, re.MULTILINE)
 			self._regex = regex
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %

server/filter.py

@@ -36,6 +36,7 @@ from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
 
 import logging, re, os, fcntl, time
+from collections import deque
 
 # Gets the instance of the logger.
 logSys = logging.getLogger("fail2ban.filter")
@@ -71,6 +72,10 @@ class Filter(JailThread):
 		self.__findTime = 6000
 		## The ignore IP list.
 		self.__ignoreIpList = []
+		## Size of line buffer
+		self.__line_buffer_size = 1
+		## Line buffer
+		self.__line_buffer = deque()
 
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
@@ -204,6 +209,25 @@ class Filter(JailThread):
 	def getMaxRetry(self):
 		return self.failManager.getMaxRetry()
 
+	##
+	# Set the maximum line buffer size.
+	#
+	# @param value the line buffer size
+
+	def setMaxLines(self, value):
+		if value < 1:
+			value = 1
+		self.__line_buffer_size = value
+		logSys.info("Set maxLines = %s" % value)
+
+	##
+	# Get the maximum line buffer size.
+	#
+	# @return the line buffer size
+
+	def getMaxLines(self):
+		return self.__line_buffer_size
+
 	##
 	# Main loop.
 	#
@@ -305,7 +329,10 @@ class Filter(JailThread):
 		else:
 			timeLine = l
 			logLine = l
-		return self.findFailure(timeLine, logLine)
+		self.__line_buffer.append(logLine)
+		while len(self.__line_buffer) > self.__line_buffer_size:
+			self.__line_buffer.popleft()
+		return self.findFailure(timeLine, "".join(self.__line_buffer))
 
 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
@@ -365,6 +392,7 @@ class Filter(JailThread):
 								 "in order to get support for this format."
 								 % (logLine, timeLine))
 				else:
+					self.__line_buffer.clear()
 					try:
 						host = failRegex.getHost()
 						ipMatch = DNSUtils.textToIp(host, self.__useDns)

server/server.py

@@ -216,6 +216,12 @@ class Server:
 	def getMaxRetry(self, name):
 		return self.__jails.getFilter(name).getMaxRetry()
 	
+	def setMaxLines(self, name, value):
+		self.__jails.getFilter(name).setMaxLines(value)
+	
+	def getMaxLines(self, name):
+		return self.__jails.getFilter(name).getMaxLines()
+	
 	# Action
 	def addAction(self, name, value):
 		self.__jails.getAction(name).addAction(value)

server/transmitter.py

@@ -167,6 +167,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setMaxRetry(name, int(value))
 			return self.__server.getMaxRetry(name)
+		elif command[1] == "maxlines":
+			value = command[2]
+			self.__server.setMaxLines(name, int(value))
+			return self.__server.getMaxLines(name)
 		# command
 		elif command[1] == "bantime":
 			value = command[2]
@@ -245,6 +249,8 @@ class Transmitter:
 			return self.__server.getFindTime(name)
 		elif command[1] == "maxretry":
 			return self.__server.getMaxRetry(name)
+		elif command[1] == "maxlines":
+			return self.__server.getMaxLines(name)
 		# Action
 		elif command[1] == "bantime":
 			return self.__server.getBanTime(name)

testcases/files/testcase-multiline.log

@@ -0,0 +1,12 @@
+Aug 14 11:59:58 [sshd] Invalid user toto...
+Aug 14 11:59:58 [sshd] from 212.41.96.185
+Aug 14 11:59:58 [sshd] Invalid user toto...
+Aug 14 11:59:58 [sshd] from 212.41.96.185
+Aug 14 11:59:58 [sshd] Invalid user fuck...
+Aug 14 11:59:58 [sshd] from 212.41.96.185
+Aug 14 11:59:58 [sshd] Invalid user toto...
+Aug 14 11:59:58 [sshd] from 212.41.96.185
+Aug 14 11:59:58 [sshd] Invalid user fuck...
+Aug 14 11:59:58 [sshd] from 212.41.96.185
+Aug 14 11:59:58 [sshd] Invalid user fuck...
+Aug 14 11:59:58 [sshd] from 212.41.96.185

testcases/filtertestcase.py

@@ -499,6 +499,7 @@ class GetFailures(unittest.TestCase):
 	FILENAME_03 = "testcases/files/testcase03.log"
 	FILENAME_04 = "testcases/files/testcase04.log"
 	FILENAME_USEDNS = "testcases/files/testcase-usedns.log"
+	FILENAME_MULTILINE = "testcases/files/testcase-multiline.log"
 
 	# so that they could be reused by other tests
 	FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
@@ -604,6 +605,20 @@ class GetFailures(unittest.TestCase):
 
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 
+	def testGetFailuresMultiLine(self):
+		output = ("212.41.96.185", 3, 1124013598.0)
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addFailRegex("Invalid user .+\n.+ from <HOST>$")
+		self.filter.addIgnoreRegex("user fuck")
+
+		self.filter.setMaxLines(2)
+
+		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
+
+		_assert_correct_last_attempt(self, self.filter, output)
+
+		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+
 class DNSUtilsTests(unittest.TestCase):
 
 	def testUseDns(self):