Merge pull request #1733 from sebres/0.10-repl-skiplines

Normalizes replacement of `<SKIPLINES>` + no multiline failregex per default

fail2ban/client/fail2banregex.py

@@ -55,11 +55,14 @@ from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack,
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
-def debuggexURL(sample, regex, useDns="yes"):
+def debuggexURL(sample, regex, multiline=False, useDns="yes"):
-	q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns),
+	args = {
-							'str': sample,
+		're': Regex._resolveHostTag(regex, useDns=useDns),
-							'flavor': 'python' })
+		'str': sample,
-	return 'https://www.debuggex.com/?' + q
+		'flavor': 'python'
+	}
+	if multiline: args['flags'] = 'm'
+	return 'https://www.debuggex.com/?' + urllib.urlencode(args)
 
 def output(args): # pragma: no cover (overriden in test-cases)
 	print(args)
@@ -400,6 +403,7 @@ class Fail2banRegex(object):
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
 			ret = self._filter.processLine(line, date)
+			lines = []
 			line = self._filter.processedLine()
 			for match in ret:
 				# Append True/False flag depending if line was matched by
@@ -422,9 +426,17 @@ class Fail2banRegex(object):
 								"".join(bufLine[::2])))
 				except ValueError:
 					pass
-				else:
+				# if buffering - add also another lines from match:
-					self._line_stats.matched += 1
+				if self._print_all_matched:
-					self._line_stats.missed -= 1
+					if not self._debuggex:
+						self._line_stats.matched_lines.append("".join(bufLine))
+					else:
+						lines.append(bufLine[0] + bufLine[2])
+				self._line_stats.matched += 1
+				self._line_stats.missed -= 1
+		if lines: # pre-lines parsed in multiline mode (buffering)
+			lines.append(line)
+			line = "\n".join(lines)
 		return line, ret
 
 	def process(self, test_lines):
@@ -472,6 +484,7 @@ class Fail2banRegex(object):
 		assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
 		lines = lstats[ltype]
 		l = lstats[ltype + '_lines']
+		multiline = self._filter.getMaxLines() > 1
 		if lines:
 			header = "%s line(s):" % (ltype.capitalize(),)
 			if self._debuggex:
@@ -485,7 +498,8 @@ class Fail2banRegex(object):
 					for arg in [l, regexlist]:
 						ans = [ x + [y] for x in ans for y in arg ]
 					b = map(lambda a: a[0] +  ' | ' + a[1].getFailRegex() + ' |  ' + 
-						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans)
+						debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), 
+							multiline, self._opts.usedns), ans)
 					pprint_list([x.rstrip() for x in b], header)
 				else:
 					output( "%s too many to print.  Use --print-all-%s " \
@@ -599,8 +613,19 @@ class Fail2banRegex(object):
 			output( "Use    journal match : %s" % " ".join(journalmatch) )
 			test_lines = journal_lines_gen(flt, myjournal)
 		else:
-			output( "Use      single line : %s" % shortstr(cmd_log) )
+			# if single line parsing (without buffering)
-			test_lines = [ cmd_log ]
+			if self._filter.getMaxLines() <= 1:
+				output( "Use      single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
+				test_lines = [ cmd_log ]
+			else: # multi line parsing (with buffering)
+				test_lines = cmd_log.split("\n")
+				output( "Use      multi line : %s line(s)" % len(test_lines) )
+				for i, l in enumerate(test_lines):
+					if i >= 5:
+						output( "| ..." ); break
+					output( "| %2.2s: %s" % (i+1, shortstr(l)) )
+				output( "`-" )
+			
 		output( "" )
 
 		self.process(test_lines)

fail2ban/server/failregex.py

@@ -103,20 +103,16 @@ class Regex:
 	# avoid construction of invalid object.
 	# @param value the regular expression
 	
-	def __init__(self, regex, **kwargs):
+	def __init__(self, regex, multiline=False, **kwargs):
 		self._matchCache = None
 		# Perform shortcuts expansions.
-		# Resolve "<HOST>" tag using default regular expression for host:
+		# Replace standard f2b-tags (like "<HOST>", etc) using default regular expressions:
 		regex = Regex._resolveHostTag(regex, **kwargs)
-		# Replace "<SKIPLINES>" with regular expression for multiple lines.
+		#
-		regexSplit = regex.split("<SKIPLINES>")
-		regex = regexSplit[0]
-		for n, regexLine in enumerate(regexSplit[1:]):
-			regex += "\n(?P<skiplines%i>(?:(.*\n)*?))" % n + regexLine
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
 		try:
-			self._regexObj = re.compile(regex, re.MULTILINE)
+			self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
 			self._regex = regex
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
@@ -135,6 +131,9 @@ class Regex:
 	def _resolveHostTag(regex, useDns="yes"):
 
 		openTags = dict()
+		props = {
+			'nl': 0, # new lines counter by <SKIPLINES> tag;
+		}
 		# tag interpolation callable:
 		def substTag(m):
 			tag = m.group()
@@ -142,6 +141,11 @@ class Regex:
 			# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
 			if tn == "HOST":
 				return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR]
+			# replace "<SKIPLINES>" with regular expression for multiple lines (by buffering with maxlines)
+			if tn == "SKIPLINES":
+				nl = props['nl']
+				props['nl'] = nl + 1
+				return r"\n(?P<skiplines%i>(?:(?:.*\n)*?))" % (nl,)
 			# static replacement from RH4TAG:
 			try:
 				return RH4TAG[tn]

fail2ban/server/filter.py

@@ -161,13 +161,11 @@ class Filter(JailThread):
 	# @param value the regular expression
 
 	def addFailRegex(self, value):
+		multiLine = self.getMaxLines() > 1
 		try:
-			regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns)
+			regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine,
+				useDns=self.__useDns)
 			self.__failRegex.append(regex)
-			if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
-				logSys.warning(
-					"Mutliline regex set for jail %r "
-					"but maxlines not greater than 1", self.jailName)
 		except RegexException as e:
 			logSys.error(e)
 			raise e

fail2ban/tests/fail2banregextestcase.py

@@ -252,6 +252,44 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		)
 		self.assertTrue(fail2banRegex.start(args))
 
+	def testDirectMultilineBuf(self):
+		# test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed):
+		for preLines in (0, 20):
+			self.pruneLog("[test-phase %s]" % preLines)
+			(opts, args, fail2banRegex) = _Fail2banRegex(
+				"--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5", 
+				("1490349000 TEST-NL\n"*preLines) + 
+				"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
+				r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
+			)
+			self.assertTrue(fail2banRegex.start(args))
+			self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2))
+			# both matched lines were printed:
+			self.assertLogged("|  1490349000 FAIL", "|  1490349001 HOST 192.0.2.34", all=True)
+
+
+	def testDirectMultilineBufDebuggex(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5",
+			"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
+			r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
+		# the sequence in args-dict is currently undefined (so can be 1st argument)
+		self.assertLogged("&flags=m", "?flags=m")
+
+	def testSinglelineWithNLinContent(self):
+		# 
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--usedns", "no", "-d", "^Epoch", "--print-all-matched",
+			"1490349000 FAIL: failure\nhost: 192.0.2.35",
+			r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$"
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
+
+
 	def testWrongFilterFile(self):
 		# use test log as filter file to cover eror cases...
 		(opts, args, fail2banRegex) = _Fail2banRegex(

fail2ban/tests/filtertestcase.py

@@ -1479,8 +1479,8 @@ class GetFailures(LogCaptureTestCase):
 		output = [("192.0.43.10", 2, 1124013599.0),
 			("192.0.43.11", 1, 1124013598.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
-		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxLines(100)
+		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
@@ -1497,9 +1497,9 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
+		self.filter.setMaxLines(100)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addIgnoreRegex("rsync error: Received SIGINT")
-		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
@@ -1513,9 +1513,9 @@ class GetFailures(LogCaptureTestCase):
 			("192.0.43.11", 1, 1124013598.0),
 			("192.0.43.15", 1, 1124013598.0)]
 		self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
+		self.filter.setMaxLines(100)
 		self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
 		self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")
-		self.filter.setMaxLines(100)
 		self.filter.setMaxRetry(1)
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)

fail2ban/tests/samplestestcase.py

@@ -40,8 +40,8 @@ TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 # regexp to test greedy catch-all should be not-greedy:
-RE_HOST = Regex('<HOST>').getRegex()
+RE_HOST = Regex._resolveHostTag('<HOST>')
-RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?).*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
+RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
 
 
 class FilterSamplesRegex(unittest.TestCase):