Added <SKIPLINES> regex applicable for multi-line

This allows lines captured by <SKIPLINES> regex to remain in the line
buffer in Filter

server/failregex.py

@@ -48,6 +48,11 @@ class Regex:
 		# Perform shortcuts expansions.
 		# Replace "<HOST>" with default regular expression for host.
 		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]+)")
+		# Replace "<SKIPLINES>" with regular expression for multiple lines.
+		regexSplit = regex.split("<SKIPLINES>")
+		regex = regexSplit[0]
+		for n, regexLine in enumerate(regexSplit[1:]):
+			regex += "\n(?P<skiplines%i>(?:(.*\n)*?))" % n + regexLine
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
 		try:
@@ -131,3 +136,21 @@ class FailRegex(Regex):
 			r = self._matchCache.re
 			raise RegexException("No 'host' found in '%s' using '%s'" % (s, r))
 		return host
+
+	##
+	# Returns unmatched lines.
+	#
+	# This returns unmatched lines inlcuding captured by the <SKIPLINES> tag.
+	# @return list of unmatched lines
+	
+	def getUnmatchedLines(self):
+		unmatchedLines = self._matchCache.string[:self._matchCache.start()]
+		n = 0
+		while True:
+			try:
+				unmatchedLines += self._matchCache.group("skiplines%i" % n)
+				n += 1
+			except IndexError:
+				break
+		unmatchedLines += self._matchCache.string[self._matchCache.end():]
+		return unmatchedLines.splitlines(True)

server/filter.py

@@ -391,7 +391,7 @@ class Filter(JailThread):
 								 "in order to get support for this format."
 								 % (logLine, timeLine))
 				else:
-					self.__lineBuffer = []
+					self.__lineBuffer = failRegex.getUnmatchedLines()
 					try:
 						host = failRegex.getHost()
 						ipMatch = DNSUtils.textToIp(host, self.__useDns)

testcases/files/testcase-multiline.log

@@ -19,6 +19,10 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test co
 Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11)
 Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11)
 Aug 14 11:59:58 yyyy rsyncd[7788]: building file list
-Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes  received 781 bytes  total size 29221543998
 Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes  received 6768 bytes  total size 29221543998
 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9]
+Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11)
+Aug 14 11:59:58 yyyy rsyncd[5534]: building file list
+Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9]
+Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes  received 781 bytes  total size 29221543998

testcases/filtertestcase.py

@@ -606,15 +606,17 @@ class GetFailures(unittest.TestCase):
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 
 	def testGetFailuresMultiLine(self):
-		output = ("192.0.43.10", 1, 1124013598.0)
+		output = [("192.0.43.10", 1, 1124013598.0),
+			("192.0.43.11", 1, 1124013598.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
-		self.filter.addFailRegex("rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error")
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
 
-		_assert_correct_last_attempt(self, self.filter, output)
+		_assert_correct_last_attempt(self, self.filter, output.pop())
+		_assert_correct_last_attempt(self, self.filter, output.pop())
 
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)