Merge remote-tracking branch 'github_kwirk_fail2ban/multi-line' into 0.9

* github_kwirk_fail2ban/multi-line:
  Revert changes to man/fail2ban-client.1
  Removed "common.local" include for FilterReader test
  Added 'maxlines' option to fail2ban-regex
  Regex get(Un)MatchedLines now returns whole lines only
  Added FilterReader test
  Added multiregex test for multi-line filter
  ignoreregex now functions correctly with multiline
  Minor typo in server/failregex.py
  Added <SKIPLINES> regex applicable for multi-line
  Sanitise testcase log 04
  Changed multi-line test to provided example
  Filter for multi-line now stores last time match
  Simplify and change some filter line buffer
  Initial changes and test for multi-line filtering

client/jailreader.py

@@ -63,6 +63,7 @@ class JailReader(ConfigReader):
 				["string", "logpath", "/var/log/messages"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
+				["int", "maxlines", 1],
 				["int", "findtime", 600],
 				["int", "bantime", 600],
 				["string", "usedns", "warn"],
@@ -114,6 +115,8 @@ class JailReader(ConfigReader):
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
 				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
+			elif opt == "maxlines":
+				stream.append(["set", self.__name, "maxlines", self.__opts[opt]])
 			elif opt == "ignoreip":
 				for ip in self.__opts[opt].split():
 					# Do not send a command if the rule is empty.

common/protocol.py

@@ -66,6 +66,7 @@ protocol = [
 ["set <JAIL> banip <IP>", "manually Ban <IP> for <JAIL>"], 
 ["set <JAIL> unbanip <IP>", "manually Unban <IP> in <JAIL>"], 
 ["set <JAIL> maxretry <RETRY>", "sets the number of failures <RETRY> before banning the host for <JAIL>"], 
+["set <JAIL> maxlines <LINES>", "sets the number of <LINES> to buffer for regex search for <JAIL>"], 
 ["set <JAIL> addaction <ACT>", "adds a new action named <NAME> for <JAIL>"], 
 ["set <JAIL> delaction <ACT>", "removes the action <NAME> from <JAIL>"], 
 ["set <JAIL> setcinfo <ACT> <KEY> <VALUE>", "sets <VALUE> for <KEY> of the action <NAME> for <JAIL>"], 
@@ -84,6 +85,7 @@ protocol = [
 ["get <JAIL> bantime", "gets the time a host is banned for <JAIL>"],
 ["get <JAIL> usedns", "gets the usedns setting for <JAIL>"],
 ["get <JAIL> maxretry", "gets the number of failures allowed for <JAIL>"],
+["get <JAIL> maxlines", "gets the number of lines to buffer for <JAIL>"],
 ["get <JAIL> addaction", "gets the last action which has been added for <JAIL>"],
 ["get <JAIL> actionstart <ACT>", "gets the start command for the action <ACT> for <JAIL>"],
 ["get <JAIL> actionstop <ACT>", "gets the stop command for the action <ACT> for <JAIL>"],

fail2ban-regex

@@ -113,6 +113,7 @@ class Fail2banRegex:
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print "    -v, --verbose           verbose output"
+		print "    -l INT, --maxlines=INT  set maxlines for multi-line regex default: 1"
 		print
 		print "Log:"
 		print "    string                  a string representing a log line"
@@ -141,6 +142,14 @@ class Fail2banRegex:
 				sys.exit(0)
 			elif opt[0] in ["-v", "--verbose"]:
 				self.__verbose = True
+			elif opt[0] in ["-l", "--maxlines"]:
+				try:
+					self.__filter.setMaxLines(int(opt[1]))
+				except ValueError:
+					print "Invlaid value for maxlines: %s" % (
+                        opt[1])
+					fail2banRegex.dispUsage()
+					sys.exit(-1)
 
 	#@staticmethod
 	def logIsFile(value):
@@ -318,8 +327,8 @@ if __name__ == "__main__":
 	fail2banRegex = Fail2banRegex()
 	# Reads the command line options.
 	try:
-		cmdOpts = 'hVcv'
+		cmdOpts = 'hVcvl:'
-		cmdLongOpts = ['help', 'version', 'verbose']
+		cmdLongOpts = ['help', 'version', 'verbose', 'maxlines=']
 		optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts)
 	except getopt.GetoptError:
 		fail2banRegex.dispUsage()

fail2ban-testcases

@@ -132,6 +132,7 @@ tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
 tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure))
 # ClientReader
 tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))
+tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest))
 
 # Filter
 if not opts.no_network:

server/failregex.py

@@ -48,10 +48,15 @@ class Regex:
 		# Perform shortcuts expansions.
 		# Replace "<HOST>" with default regular expression for host.
 		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]+)")
+		# Replace "<SKIPLINES>" with regular expression for multiple lines.
+		regexSplit = regex.split("<SKIPLINES>")
+		regex = regexSplit[0]
+		for n, regexLine in enumerate(regexSplit[1:]):
+			regex += "\n(?P<skiplines%i>(?:(.*\n)*?))" % n + regexLine
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
 		try:
-			self._regexObj = re.compile(regex)
+			self._regexObj = re.compile(regex, re.MULTILINE)
 			self._regex = regex
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
@@ -76,6 +81,19 @@ class Regex:
 	
 	def search(self, value):
 		self._matchCache = self._regexObj.search(value)
+		if self.hasMatched():
+			# Find start of the first line where the match was found
+			try:
+				self._matchLineStart = self._matchCache.string.rindex(
+					"\n", 0, self._matchCache.start() +1 ) + 1
+			except ValueError:
+				self._matchLineStart = 0
+			# Find end of the last line where the match was found
+			try:
+				self._matchLineEnd = self._matchCache.string.index(
+					"\n", self._matchCache.end() - 1) + 1
+			except ValueError:
+				self._matchLineEnd = len(self._matchCache.string)
 	
 	##
 	# Checks if the previous call to search() matched.
@@ -88,6 +106,54 @@ class Regex:
 		else:
 			return False
 
+	##
+	# Returns skipped lines.
+	#
+	# This returns skipped lines captured by the <SKIPLINES> tag.
+	# @return list of skipped lines
+	
+	def getSkippedLines(self):
+		if not self._matchCache:
+			return []
+		skippedLines = ""
+		n = 0
+		while True:
+			try:
+				skippedLines += self._matchCache.group("skiplines%i" % n)
+				n += 1
+			except IndexError:
+				break
+		return skippedLines.splitlines(True)
+
+	##
+	# Returns unmatched lines.
+	#
+	# This returns unmatched lines including captured by the <SKIPLINES> tag.
+	# @return list of unmatched lines
+	
+	def getUnmatchedLines(self):
+		if not self.hasMatched():
+			return []
+		unmatchedLines = (
+			self._matchCache.string[:self._matchLineStart].splitlines(True)
+			+ self.getSkippedLines()
+			+ self._matchCache.string[self._matchLineEnd:].splitlines(True))
+		return unmatchedLines
+
+	##
+	# Returns matched lines.
+	#
+	# This returns matched lines by excluding those captured
+	# by the <SKIPLINES> tag.
+	# @return list of matched lines
+	
+	def getMatchedLines(self):
+		if not self.hasMatched():
+			return []
+		matchedLines = self._matchCache.string[
+			self._matchLineStart:self._matchLineEnd].splitlines(True)
+		return [line for line in matchedLines
+			if line not in self.getSkippedLines()]
 
 ##
 # Exception dedicated to the class Regex.

server/filter.py

@@ -71,6 +71,12 @@ class Filter(JailThread):
 		self.__findTime = 6000
 		## The ignore IP list.
 		self.__ignoreIpList = []
+		## Size of line buffer
+		self.__lineBufferSize = 1
+		## Line buffer
+		self.__lineBuffer = []
+		## Store last time stamp, applicable for multi-line
+		self.__lastTimeLine = ""
 
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
@@ -204,6 +210,23 @@ class Filter(JailThread):
 	def getMaxRetry(self):
 		return self.failManager.getMaxRetry()
 
+	##
+	# Set the maximum line buffer size.
+	#
+	# @param value the line buffer size
+
+	def setMaxLines(self, value):
+		self.__lineBufferSize = max(1, value)
+		logSys.info("Set maxLines = %i" % self.__lineBufferSize)
+
+	##
+	# Get the maximum line buffer size.
+	#
+	# @return the line buffer size
+
+	def getMaxLines(self):
+		return self.__lineBufferSize
+
 	##
 	# Main loop.
 	#
@@ -298,14 +321,17 @@ class Filter(JailThread):
 		if timeMatch:
 			# Lets split into time part and log part of the line
 			timeLine = timeMatch.group()
+			self.__lastTimeLine = timeLine
 			# Lets leave the beginning in as well, so if there is no
 			# anchore at the beginning of the time regexp, we don't
 			# at least allow injection. Should be harmless otherwise
 			logLine  = l[:timeMatch.start()] + l[timeMatch.end():]
 		else:
-			timeLine = l
+			timeLine = self.__lastTimeLine or l
 			logLine = l
-		return self.findFailure(timeLine, logLine)
+		self.__lineBuffer = ((self.__lineBuffer +
+				[logLine])[-self.__lineBufferSize:])
+		return self.findFailure(timeLine, "".join(self.__lineBuffer))
 
 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
@@ -348,14 +374,15 @@ class Filter(JailThread):
 
 	def findFailure(self, timeLine, logLine):
 		failList = list()
-		# Checks if we must ignore this line.
-		if self.ignoreLine(logLine):
-			# The ignoreregex matched. Return.
-			return failList
 		# Iterates over all the regular expressions.
 		for failRegex in self.__failRegex:
 			failRegex.search(logLine)
 			if failRegex.hasMatched():
+				# Checks if we must ignore this match.
+				if self.ignoreLine("".join(failRegex.getMatchedLines())):
+					# The ignoreregex matched. Remove ignored match.
+					self.__lineBuffer = failRegex.getUnmatchedLines()
+					continue
 				# The failregex matched.
 				date = self.dateDetector.getUnixTime(timeLine)
 				if date == None:
@@ -365,6 +392,7 @@ class Filter(JailThread):
 								 "in order to get support for this format."
 								 % (logLine, timeLine))
 				else:
+					self.__lineBuffer = failRegex.getUnmatchedLines()
 					try:
 						host = failRegex.getHost()
 						ipMatch = DNSUtils.textToIp(host, self.__useDns)

server/server.py

@@ -216,6 +216,12 @@ class Server:
 	def getMaxRetry(self, name):
 		return self.__jails.getFilter(name).getMaxRetry()
 	
+	def setMaxLines(self, name, value):
+		self.__jails.getFilter(name).setMaxLines(value)
+	
+	def getMaxLines(self, name):
+		return self.__jails.getFilter(name).getMaxLines()
+	
 	# Action
 	def addAction(self, name, value):
 		self.__jails.getAction(name).addAction(value)

server/transmitter.py

@@ -167,6 +167,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setMaxRetry(name, int(value))
 			return self.__server.getMaxRetry(name)
+		elif command[1] == "maxlines":
+			value = command[2]
+			self.__server.setMaxLines(name, int(value))
+			return self.__server.getMaxLines(name)
 		# command
 		elif command[1] == "bantime":
 			value = command[2]
@@ -245,6 +249,8 @@ class Transmitter:
 			return self.__server.getFindTime(name)
 		elif command[1] == "maxretry":
 			return self.__server.getMaxRetry(name)
+		elif command[1] == "maxlines":
+			return self.__server.getMaxLines(name)
 		# Action
 		elif command[1] == "bantime":
 			return self.__server.getBanTime(name)

testcases/clientreadertestcase.py

@@ -29,6 +29,8 @@ __license__ = "GPL"
 
 import unittest
 from client.jailreader import JailReader
+from client.configreader import ConfigReader
+from client.filterreader import FilterReader
 
 class JailReaderTest(unittest.TestCase):
 
@@ -44,3 +46,38 @@ class JailReaderTest(unittest.TestCase):
 		result = JailReader.splitAction(action)
 		self.assertEquals(expected, result)
 		
+class FilterReaderTest(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		ConfigReader.setBaseDir("testcases/files/")
+
+	def tearDown(self):
+		"""Call after every test case."""
+
+	def testConvert(self):
+		output = [['set', 'testcase01', 'addfailregex',
+			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
+			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
+			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
+			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$"],
+			['set', 'testcase01', 'addfailregex',
+			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
+			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
+			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
+			"error: PAM: )?User not known to the underlying authentication mo"
+			"dule for .* from <HOST>\\s*$"],
+			['set', 'testcase01', 'addfailregex',
+			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
+			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
+			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
+			"error: PAM: )?User not known to the\\nunderlying authentication."
+			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"],
+			['set', 'testcase01', 'addignoreregex', 
+			"^.+ john from host 192.168.1.1\\s*$"]]
+		filterReader = FilterReader("testcase01", "testcase01")
+		filterReader.read()
+		#filterReader.getOptions(["failregex", "ignoreregex"])
+		filterReader.getOptions(None)
+
+		self.assertEquals(filterReader.convert(), output)

testcases/files/filter.d/testcase-common.conf

@@ -0,0 +1,41 @@
+# Generic configuration items (to be used as interpolations) in other
+# filters  or actions configurations
+#
+# Author: Yaroslav Halchenko
+#
+# $Revision$
+#
+
+[DEFAULT]
+
+# Daemon definition is to be specialized (if needed) in .conf file
+_daemon = \S*
+
+#
+# Shortcuts for easier comprehension of the failregex
+#
+# PID.
+# EXAMPLES: [123]
+__pid_re = (?:\[\d+\])
+
+# Daemon name (with optional source_file:line or whatever)
+# EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix)
+__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
+
+# Combinations of daemon name and PID
+# EXAMPLES: sshd[31607], pop(pam_unix)[4920]
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:)
+
+# Some messages have a kernel prefix with a timestamp
+# EXAMPLES: kernel: [769570.846956]
+__kernel_prefix = kernel: \[\d+\.\d+\]
+
+__hostname = \S+
+
+#
+# Common line prefixes (beginnings) which could be used in filters
+#
+#       [hostname] [vserver tag] daemon_id spaces
+# this can be optional (for instance if we match named native log files)
+__prefix_line = \s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s*
+

testcases/files/filter.d/testcase01.conf

@@ -0,0 +1,34 @@
+# Fail2Ban configuration file
+#
+# Author: Cyril Jaquier
+#
+# $Revision$
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = testcase-common.conf
+
+
+[Definition]
+
+_daemon = sshd
+
+# Option:  failregex
+# Notes.:  regex to match the password failures messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+# Values:  TEXT
+#
+failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from <HOST>\s*$
+            ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from <HOST>\s*$
+            ^%(__prefix_line)s(?:error: PAM: )?User not known to the\nunderlying authentication.+$<SKIPLINES>^.+ module for .* from <HOST>\s*$
+
+# Option:  ignoreregex
+# Notes.:  regex to ignore. If this regex matches, the line is ignored.
+# Values:  TEXT
+#
+ignoreregex = ^.+ john from host 192.168.1.1\s*$

testcases/files/testcase-multiline.log

@@ -0,0 +1,35 @@
+Aug 14 11:58:58 yyyy rsyncd[9874]: connect from example.com (192.0.43.10)
+Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10)
+Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10)
+Aug 14 11:59:58 yyyy rsyncd[23864]: building file list
+Aug 14 11:59:58 yyyy rsyncd[28101]: connect from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[28101]: rsync on xxx/ from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[28101]: building file list
+Aug 14 11:59:58 yyyy rsyncd[28101]: sent 294382 bytes  received 781 bytes  total size 29221543998
+Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes  received 65115 bytes  total size 29221543998
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115
+Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11)
+
+Aug 14 11:59:58 yyyy rsyncd[1762]: building file list
+Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes  received 781 bytes  total size 29221543998
+Aug 14 11:59:58 yyyy sendmail[30222]: r0NNNlC0030222: from=<bounce-25497-9881290652-user=example.com@example.com>, size=6420, class=0, nrcpts=1, msgid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=[192.0.43.15] (may be forged)
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test.
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 115 to 116
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test.
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error
+Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error
+
+
+Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[7788]: building file list
+Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes  received 6768 bytes  total size 29221543998
+Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9]
+Aug 14 11:59:58 yyyy spamd[19119]: spamd: result: Y 11 - AWL,BAYES_50,DKIM_SIGNED,DKIM_VALID,DKIM_VALID_AU,HTML_MESSAGE,RCVD_IN_BRBL_LASTEXT,RCVD_IN_PSBL,RCVD_IN_RP_RNBL,RDNS_NONE,URIBL_BLACK,URIBL_DBL_SPAM scantime=1.2,size=6910,user=sa-milt,uid=499,required_score=5.0,rhost=localhost,raddr=127.0.0.1,rport=57429,mid=<0.0.9881290652.3772024cf8879cycvau18081.0@example.com>,bayes=0.536244,autolearn=no
+Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[5534]: building file list
+Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: Received SIGINT
+Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes  received 781 bytes  total size 29221543998
+Aug 14 11:59:59 yyyy rsyncd[9874]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9]

testcases/files/testcase04.log

@@ -1,15 +1,15 @@
 Sep 21 22:03:07 [sshd] Invalid user toto from 212.41.96.185
-1124012400 [sshd] Invalid user fuck from 212.41.96.185
+1124012400 [sshd] Invalid user duck from 212.41.96.185
 Sep 21 21:03:38 [sshd] Invalid user toto from 212.41.96.185
-1124012500 [sshd] Invalid user fuck from 212.41.96.185
+1124012500 [sshd] Invalid user duck from 212.41.96.185
 Sep 21 21:03:46 [sshd] Invalid user toto from 212.41.96.185
-Aug 14 11:58:48 [sshd] Invalid user fuck from 212.41.96.185
+Aug 14 11:58:48 [sshd] Invalid user duck from 212.41.96.185
 Aug 14 11:59:58 [sshd] Invalid user toto from 212.41.96.185
-Sep 21 21:04:03 [sshd] Invalid user fuck from 212.41.96.185
+Sep 21 21:04:03 [sshd] Invalid user duck from 212.41.96.185
                 - Last output repeated twice -
 2005/08/14 11:57:00 [sshd] Invalid user toto from 212.41.96.186
-2005/08/14 11:58:00 [sshd] Invalid user fuck from 212.41.96.186
+2005/08/14 11:58:00 [sshd] Invalid user duck from 212.41.96.186
 2005/08/14 11:59:00 [sshd] Invalid user toto from 212.41.96.186
-2005/08/14 12:00:00 [sshd] Invalid user fuck from 212.41.96.186
+2005/08/14 12:00:00 [sshd] Invalid user duck from 212.41.96.186
                 - Last output repeated twice -
 Sep 21 21:09:01 [sshd] Invalid user toto from 212.41.96.185

testcases/filtertestcase.py

@@ -499,6 +499,7 @@ class GetFailures(unittest.TestCase):
 	FILENAME_03 = "testcases/files/testcase03.log"
 	FILENAME_04 = "testcases/files/testcase04.log"
 	FILENAME_USEDNS = "testcases/files/testcase-usedns.log"
+	FILENAME_MULTILINE = "testcases/files/testcase-multiline.log"
 
 	# so that they could be reused by other tests
 	FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
@@ -604,6 +605,53 @@ class GetFailures(unittest.TestCase):
 
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 
+	def testGetFailuresMultiLine(self):
+		output = [("192.0.43.10", 2, 1124013599.0),
+			("192.0.43.11", 1, 1124013598.0)]
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
+		self.filter.setMaxLines(100)
+		self.filter.setMaxRetry(1)
+
+		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
+
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+
+		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+
+	def testGetFailuresMultiLineIgnoreRegex(self):
+		output = [("192.0.43.10", 2, 1124013599.0)]
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
+		self.filter.addIgnoreRegex("rsync error: Received SIGINT")
+		self.filter.setMaxLines(100)
+		self.filter.setMaxRetry(1)
+
+		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
+
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+
+		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+
+	def testGetFailuresMultiLineMultiRegex(self):
+		output = [("192.0.43.10", 2, 1124013599.0),
+			("192.0.43.11", 1, 1124013598.0),
+			("192.0.43.15", 1, 1124013598.0)]
+		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
+		self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")
+		self.filter.setMaxLines(100)
+		self.filter.setMaxRetry(1)
+
+		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
+
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+
+		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+
 class DNSUtilsTests(unittest.TestCase):
 
 	def testUseDns(self):