diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 782513a7..45dbfe95 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -55,11 +55,14 @@ from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, # Gets the instance of the logger. logSys = getLogger("fail2ban") -def debuggexURL(sample, regex, useDns="yes"): - q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns), - 'str': sample, - 'flavor': 'python' }) - return 'https://www.debuggex.com/?' + q +def debuggexURL(sample, regex, multiline=False, useDns="yes"): + args = { + 're': Regex._resolveHostTag(regex, useDns=useDns), + 'str': sample, + 'flavor': 'python' + } + if multiline: args['flags'] = 'm' + return 'https://www.debuggex.com/?' + urllib.urlencode(args) def output(args): # pragma: no cover (overriden in test-cases) print(args) @@ -400,6 +403,7 @@ class Fail2banRegex(object): fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() try: ret = self._filter.processLine(line, date) + lines = [] line = self._filter.processedLine() for match in ret: # Append True/False flag depending if line was matched by @@ -422,9 +426,17 @@ class Fail2banRegex(object): "".join(bufLine[::2]))) except ValueError: pass - else: - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 + if lines: # pre-lines parsed in multiline mode (buffering) + lines.append(line) + line = "\n".join(lines) return line, ret def process(self, test_lines): @@ -472,6 +484,7 @@ class Fail2banRegex(object): assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] + multiline = self._filter.getMaxLines() > 1 if lines: header = "%s line(s):" % (ltype.capitalize(),) if self._debuggex: @@ -485,7 +498,8 @@ class Fail2banRegex(object): for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + - debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans) + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), + multiline, self._opts.usedns), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ @@ -599,8 +613,19 @@ class Fail2banRegex(object): output( "Use journal match : %s" % " ".join(journalmatch) ) test_lines = journal_lines_gen(flt, myjournal) else: - output( "Use single line : %s" % shortstr(cmd_log) ) - test_lines = [ cmd_log ] + # if single line parsing (without buffering) + if self._filter.getMaxLines() <= 1: + output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) + test_lines = [ cmd_log ] + else: # multi line parsing (with buffering) + test_lines = cmd_log.split("\n") + output( "Use multi line : %s line(s)" % len(test_lines) ) + for i, l in enumerate(test_lines): + if i >= 5: + output( "| ..." ); break + output( "| %2.2s: %s" % (i+1, shortstr(l)) ) + output( "`-" ) + output( "" ) self.process(test_lines) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 59f59978..d5c9345f 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -103,20 +103,16 @@ class Regex: # avoid construction of invalid object. # @param value the regular expression - def __init__(self, regex, **kwargs): + def __init__(self, regex, multiline=False, **kwargs): self._matchCache = None # Perform shortcuts expansions. - # Resolve "" tag using default regular expression for host: + # Replace standard f2b-tags (like "", etc) using default regular expressions: regex = Regex._resolveHostTag(regex, **kwargs) - # Replace "" with regular expression for multiple lines. - regexSplit = regex.split("") - regex = regexSplit[0] - for n, regexLine in enumerate(regexSplit[1:]): - regex += "\n(?P(?:(.*\n)*?))" % n + regexLine + # if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: - self._regexObj = re.compile(regex, re.MULTILINE) + self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % @@ -135,6 +131,9 @@ class Regex: def _resolveHostTag(regex, useDns="yes"): openTags = dict() + props = { + 'nl': 0, # new lines counter by tag; + } # tag interpolation callable: def substTag(m): tag = m.group() @@ -142,6 +141,11 @@ class Regex: # 3 groups instead of - separated ipv4, ipv6 and host (dns) if tn == "HOST": return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR] + # replace "" with regular expression for multiple lines (by buffering with maxlines) + if tn == "SKIPLINES": + nl = props['nl'] + props['nl'] = nl + 1 + return r"\n(?P(?:(?:.*\n)*?))" % (nl,) # static replacement from RH4TAG: try: return RH4TAG[tn] diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b425060a..75536d57 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -161,13 +161,11 @@ class Filter(JailThread): # @param value the regular expression def addFailRegex(self, value): + multiLine = self.getMaxLines() > 1 try: - regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns) + regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, + useDns=self.__useDns) self.__failRegex.append(regex) - if "\n" in regex.getRegex() and not self.getMaxLines() > 1: - logSys.warning( - "Mutliline regex set for jail %r " - "but maxlines not greater than 1", self.jailName) except RegexException as e: logSys.error(e) raise e diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 0cd0e303..8bfedad1 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -252,6 +252,44 @@ class Fail2banRegexTest(LogCaptureTestCase): ) self.assertTrue(fail2banRegex.start(args)) + def testDirectMultilineBuf(self): + # test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed): + for preLines in (0, 20): + self.pruneLog("[test-phase %s]" % preLines) + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5", + ("1490349000 TEST-NL\n"*preLines) + + "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", + r"^\s*FAIL\s*$^\s*HOST \s*$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2)) + # both matched lines were printed: + self.assertLogged("| 1490349000 FAIL", "| 1490349001 HOST 192.0.2.34", all=True) + + + def testDirectMultilineBufDebuggex(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5", + "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", + r"^\s*FAIL\s*$^\s*HOST \s*$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + # the sequence in args-dict is currently undefined (so can be 1st argument) + self.assertLogged("&flags=m", "?flags=m") + + def testSinglelineWithNLinContent(self): + # + (opts, args, fail2banRegex) = _Fail2banRegex( + "--usedns", "no", "-d", "^Epoch", "--print-all-matched", + "1490349000 FAIL: failure\nhost: 192.0.2.35", + r"^\s*FAIL:\s*.*\nhost:\s+$" + ) + self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + + def testWrongFilterFile(self): # use test log as filter file to cover eror cases... (opts, args, fail2banRegex) = _Fail2banRegex( diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 10310a5d..ce665e72 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1479,8 +1479,8 @@ class GetFailures(LogCaptureTestCase): output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) - self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) @@ -1497,9 +1497,9 @@ class GetFailures(LogCaptureTestCase): def testGetFailuresMultiLineIgnoreRegex(self): output = [("192.0.43.10", 2, 1124013599.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) + self.filter.setMaxLines(100) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") - self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) @@ -1513,9 +1513,9 @@ class GetFailures(LogCaptureTestCase): ("192.0.43.11", 1, 1124013598.0), ("192.0.43.15", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) + self.filter.setMaxLines(100) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") - self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 0ba11c2e..121c1c5c 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -40,8 +40,8 @@ TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config") TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") # regexp to test greedy catch-all should be not-greedy: -RE_HOST = Regex('').getRegex() -RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?).*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$') +RE_HOST = Regex._resolveHostTag('') +RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$') class FilterSamplesRegex(unittest.TestCase):