From ea466d59f478c18e4a3695032499b3c390d93222 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 25 Jan 2013 18:11:40 +0000 Subject: [PATCH] ignoreregex now functions correctly with multiline Ignore regexs are now only compared to lines that match the failregex. Supporting test also added for multiline regex and overlapping multiline regex matches. --- server/failregex.py | 66 +++++++++++++++++++------- server/filter.py | 9 ++-- testcases/files/testcase-multiline.log | 7 ++- testcases/filtertestcase.py | 16 ++++++- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/server/failregex.py b/server/failregex.py index b7084ce2..d98ffa27 100644 --- a/server/failregex.py +++ b/server/failregex.py @@ -93,6 +93,54 @@ class Regex: else: return False + ## + # Returns skipped lines. + # + # This returns skipped lines captured by the tag. + # @return list of skipped lines + + def getSkippedLines(self): + if not self._matchCache: + return [] + skippedLines = "" + n = 0 + while True: + try: + skippedLines += self._matchCache.group("skiplines%i" % n) + n += 1 + except IndexError: + break + return skippedLines.splitlines(True) + + ## + # Returns unmatched lines. + # + # This returns unmatched lines including captured by the tag. + # @return list of unmatched lines + + def getUnmatchedLines(self): + if not self._matchCache: + return [] + unmatchedLines = ( + self._matchCache.string[:self._matchCache.start()].splitlines(True) + + self.getSkippedLines() + + self._matchCache.string[self._matchCache.end():].splitlines(True)) + return unmatchedLines + + ## + # Returns matched lines. + # + # This returns matched lines by excluding those captured + # by the tag. + # @return list of matched lines + + def getMatchedLines(self): + if not self._matchCache: + return [] + matchedLines = self._matchCache.string[ + self._matchCache.start():self._matchCache.end()].splitlines(True) + return [line for line in matchedLines + if line not in self.getSkippedLines()] ## # Exception dedicated to the class Regex. @@ -136,21 +184,3 @@ class FailRegex(Regex): r = self._matchCache.re raise RegexException("No 'host' found in '%s' using '%s'" % (s, r)) return host - - ## - # Returns unmatched lines. - # - # This returns unmatched lines including captured by the tag. - # @return list of unmatched lines - - def getUnmatchedLines(self): - unmatchedLines = self._matchCache.string[:self._matchCache.start()] - n = 0 - while True: - try: - unmatchedLines += self._matchCache.group("skiplines%i" % n) - n += 1 - except IndexError: - break - unmatchedLines += self._matchCache.string[self._matchCache.end():] - return unmatchedLines.splitlines(True) diff --git a/server/filter.py b/server/filter.py index 0b0e0535..5f2f0f2b 100644 --- a/server/filter.py +++ b/server/filter.py @@ -374,14 +374,15 @@ class Filter(JailThread): def findFailure(self, timeLine, logLine): failList = list() - # Checks if we must ignore this line. - if self.ignoreLine(logLine): - # The ignoreregex matched. Return. - return failList # Iterates over all the regular expressions. for failRegex in self.__failRegex: failRegex.search(logLine) if failRegex.hasMatched(): + # Checks if we must ignore this match. + if self.ignoreLine("".join(failRegex.getMatchedLines())): + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer = failRegex.getUnmatchedLines() + continue # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) if date == None: diff --git a/testcases/files/testcase-multiline.log b/testcases/files/testcase-multiline.log index c0151d34..69dac361 100644 --- a/testcases/files/testcase-multiline.log +++ b/testcases/files/testcase-multiline.log @@ -1,3 +1,4 @@ +Aug 14 11:58:58 yyyy rsyncd[9874]: connect from example.com (192.0.43.10) Aug 14 11:58:58 yyyy rsyncd[23864]: connect from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: rsync on xxx/ from example.com (192.0.43.10) Aug 14 11:59:58 yyyy rsyncd[23864]: building file list @@ -9,6 +10,7 @@ Aug 14 11:59:58 yyyy rsyncd[18067]: sent 2833586339 bytes received 65115 bytes Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute: 194 Temperature_Celsius changed from 116 to 115 Aug 14 11:59:58 yyyy rsyncd[1762]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[1762]: rsync on xxx/ from irrelevant (192.0.43.11) + Aug 14 11:59:58 yyyy rsyncd[1762]: building file list Aug 14 11:59:58 yyyy rsyncd[1762]: sent 294382 bytes received 781 bytes total size 29221543998 Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], starting scheduled Short Self-Test. @@ -16,6 +18,8 @@ Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], SMART Usage Attribute Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], starting scheduled Short Self-Test. Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sda [SAT], previous self-test completed without error Aug 14 11:59:58 yyyy smartd[2635]: Device: /dev/sdb [SAT], previous self-test completed without error + + Aug 14 11:59:58 yyyy rsyncd[7788]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[7788]: building file list @@ -24,5 +28,6 @@ Aug 14 11:59:58 yyyy rsyncd[23864]: rsync error: timeout in data send/receive (c Aug 14 11:59:58 yyyy rsyncd[5534]: connect from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: rsync on xxx/ from irrelevant (192.0.43.11) Aug 14 11:59:58 yyyy rsyncd[5534]: building file list -Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] +Aug 14 11:59:58 yyyy rsyncd[7788]: rsync error: Received SIGINT Aug 14 11:59:58 yyyy rsyncd[5534]: sent 294382 bytes received 781 bytes total size 29221543998 +Aug 14 11:59:59 yyyy rsyncd[9874]: rsync error: timeout in data send/receive (code 30) at io.c(137) [sender=3.0.9] diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 262074e6..4e60c242 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -606,7 +606,7 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 1, 1124013598.0), + output = [("192.0.43.10", 2, 1124013599.0), ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -620,6 +620,20 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + def testGetFailuresMultiLineIgnoreRegex(self): + output = [("192.0.43.10", 2, 1124013599.0)] + self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) + self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addIgnoreRegex("rsync error: Received SIGINT") + self.filter.setMaxLines(100) + self.filter.setMaxRetry(1) + + self.filter.getFailures(GetFailures.FILENAME_MULTILINE) + + _assert_correct_last_attempt(self, self.filter, output.pop()) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + class DNSUtilsTests(unittest.TestCase): def testUseDns(self):