From 9b4806bfd31e286e8b3271e7765c53f1c3c6445c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 24 Jan 2013 18:16:17 +0000 Subject: [PATCH] Added regex applicable for multi-line This allows lines captured by regex to remain in the line buffer in Filter --- server/failregex.py | 23 +++++++++++++++++++++++ server/filter.py | 2 +- testcases/files/testcase-multiline.log | 6 +++++- testcases/filtertestcase.py | 8 +++++--- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index c595b4de..84b237c7 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -48,6 +48,11 @@ class Regex: # Perform shortcuts expansions. # Replace "" with default regular expression for host. regex = regex.replace("", "(?:::f{4,6}:)?(?P[\w\-.^_]+)") + # Replace "" with regular expression for multiple lines. + regexSplit = regex.split("") + regex = regexSplit[0] + for n, regexLine in enumerate(regexSplit[1:]): + regex += "\n(?P(?:(.*\n)*?))" % n + regexLine if regex.lstrip() == '': raise RegexException("Cannot add empty regex") try: @@ -131,3 +136,21 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host + + ## + # Returns unmatched lines. + # + # This returns unmatched lines inlcuding captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + unmatchedLines = self._matchCache.string[:self._matchCache.start()] + n = 0 + while True: + try: + unmatchedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + unmatchedLines += self._matchCache.string[self._matchCache.end():] + return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 4d99ae33..0b0e0535 100644 --- a/server/filter.py +++ b/server/filter.py @@ -391,7 +391,7 @@ class Filter(JailThread): "in order to get support for this format." % (logLine, timeLine)) else: - self.__lineBuffer = [] + self.__lineBuffer = failRegex.getUnmatchedLines() try: host = failRegex.getHost() ipMatch = DNSUtils.textToIp(host, self.__useDns) diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index 70bd99ad..c0151d34 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -19,6 +19,10 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test co Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[21919]: sent 2836906453 bytes received 6768 bytes total size 29221543998 Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) +Aug 14 11:59:58 yyyy rsyncd[5534]: building file list +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 7cd4c5ec..262074e6 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,15 +606,17 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = ("192.0.43.10", 1, 1124013598.0) + output = [("192.0.43.10", 1, 1124013598.0), + ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) - self.filter.addFailRegex("rsyncd\[(?P\d+)\]: connect from .+ \(\)\n(?:.*\n)*?.+ rsyncd\[(?P=pid)\]: rsync error") + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output) + _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output.pop()) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)