From c80297045ecb5340e71f637328325f7abd692de4 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 28 Dec 2013 18:02:16 +0000 Subject: [PATCH] ENH: Pass date time straight from systemd backend Removes need to reparse the date time back from the ISO format --- bin/fail2ban-regex | 41 +++++++++++------- fail2ban/client/beautifier.py | 2 +- fail2ban/server/filter.py | 73 ++++++++++++++++++-------------- fail2ban/server/filtersystemd.py | 24 ++++++----- fail2ban/tests/filtertestcase.py | 3 +- 5 files changed, 83 insertions(+), 60 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 49f6e5e0..fc39c958 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -239,7 +239,9 @@ class Fail2banRegex(object): if not self._datepattern_set: self._filter.setDatePattern(pattern) self._datepattern_set = True - print "Use datepattern : %s" % self._filter.getDatePattern()[1] + if pattern is not None: + print "Use datepattern : %s" % ( + self._filter.getDatePattern()[1], ) def setMaxLines(self, v): if not self._maxlines_set: @@ -318,11 +320,11 @@ class Fail2banRegex(object): return False return found - def testRegex(self, line): + def testRegex(self, line, date=None): orgLineBuffer = self._filter._Filter__lineBuffer fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() try: - line, ret = self._filter.processLine(line, checkAllRegex=True) + line, ret = self._filter.processLine(line, date, checkAllRegex=True) for match in ret: # Append True/False flag depending if line was matched by # more than one regex @@ -353,11 +355,16 @@ class Fail2banRegex(object): def process(self, test_lines): for line_no, line in enumerate(test_lines): - line = line.strip('\r\n') - if line.startswith('#') or not line: - # skip comment and empty lines - continue - line_datetimestripped, ret = fail2banRegex.testRegex(line) + if isinstance(line, tuple): + line_datetimestripped, ret = fail2banRegex.testRegex( + line[0], line[1]) + line = "".join(line[0]) + else: + line = line.rstrip('\r\n') + if line.startswith('#') or not line: + # skip comment and empty lines + continue + line_datetimestripped, ret = fail2banRegex.testRegex(line) is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped) if is_ignored: @@ -373,7 +380,7 @@ class Fail2banRegex(object): self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 - if line_no % 10 == 0: + if line_no % 10 == 0 and self._filter.dateDetector is not None: self._filter.dateDetector.sortTemplate() @@ -439,12 +446,14 @@ class Fail2banRegex(object): _ = print_failregexes("Ignoreregex", self._ignoreregex) - print "\nDate template hits:" - out = [] - for template in self._filter.dateDetector.getTemplates(): - if self._verbose or template.getHits(): - out.append("[%d] %s" % (template.getHits(), template.getName())) - pprint_list(out, "[# of hits] date format") + if self._filter.dateDetector is not None: + print "\nDate template hits:" + out = [] + for template in self._filter.dateDetector.getTemplates(): + if self._verbose or template.getHits(): + out.append("[%d] %s" % ( + template.getHits(), template.getName())) + pprint_list(out, "[# of hits] date format") print "\nLines: %s" % self._line_stats @@ -523,7 +532,7 @@ if __name__ == "__main__": sys.exit(-1) myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) journalmatch = fail2banRegex._journalmatch - fail2banRegex.setDatePattern("ISO8601") + fail2banRegex.setDatePattern(None) if journalmatch: try: for element in journalmatch: diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index ab795a08..4bfa9345 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -136,7 +136,7 @@ class Beautifier: elif inC[2] == "datepattern": msg = "Current date pattern set to: " if response is None: - msg = msg + "Default Detectors" + msg = msg + "Not set/required" elif response[0] is None: msg = msg + "%s" % response[1] else: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index f713ee67..4d6b8041 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -199,8 +199,10 @@ class Filter(JailThread): # @param pattern the date template pattern def setDatePattern(self, pattern): - dateDetector = DateDetector() - if pattern.upper() == "ISO8601": + if pattern is None: + self.dateDetector = None + return + elif pattern.upper() == "ISO8601": template = DateISO8601() template.setName("ISO8601") elif pattern.upper() == "EPOCH": @@ -215,8 +217,8 @@ class Filter(JailThread): template.setPattern(pattern[1:], anchor=True) else: template.setPattern(pattern, anchor=False) - dateDetector.appendTemplate(template) - self.dateDetector = dateDetector + self.dateDetector = DateDetector() + self.dateDetector.appendTemplate(template) logSys.info("Date pattern set to `%r`: `%s`" % (pattern, template.getName())) logSys.debug("Date pattern regex for %r: %s" % @@ -228,17 +230,18 @@ class Filter(JailThread): # @return pattern of the date template pattern def getDatePattern(self): - templates = self.dateDetector.getTemplates() - if len(templates) > 1: - return None # Default Detectors in use - elif len(templates) == 1: - if hasattr(templates[0], "getPattern"): - pattern = templates[0].getPattern() - if templates[0].getRegex()[0] == "^": - pattern = "^" + pattern - else: - pattern = None - return pattern, templates[0].getName() + if self.dateDetector is not None: + templates = self.dateDetector.getTemplates() + if len(templates) > 1: + return None, "Default Detectors" + elif len(templates) == 1: + if hasattr(templates[0], "getPattern"): + pattern = templates[0].getPattern() + if templates[0].getRegex()[0] == "^": + pattern = "^" + pattern + else: + pattern = None + return pattern, templates[0].getName() ## # Set the maximum retry value. @@ -361,28 +364,32 @@ class Filter(JailThread): return False - def processLine(self, line, returnRawHost=False, checkAllRegex=False): + def processLine(self, line, date=None, returnRawHost=False, + checkAllRegex=False): """Split the time portion from log msg and return findFailures on them """ - l = line.rstrip('\r\n') - logSys.log(7, "Working on line %r", line) - - timeMatch = self.dateDetector.matchTime(l) - if timeMatch: - tupleLine = ( - l[:timeMatch.start()], - l[timeMatch.start():timeMatch.end()], - l[timeMatch.end():]) + if date: + tupleLine = line else: - tupleLine = (l, "", "") + l = line.rstrip('\r\n') + logSys.log(7, "Working on line %r", line) + + timeMatch = self.dateDetector.matchTime(l) + if timeMatch: + tupleLine = ( + l[:timeMatch.start()], + l[timeMatch.start():timeMatch.end()], + l[timeMatch.end():]) + else: + tupleLine = (l, "", "") return "".join(tupleLine[::2]), self.findFailure( - tupleLine, returnRawHost, checkAllRegex) + tupleLine, date, returnRawHost, checkAllRegex) - def processLineAndAdd(self, line): + def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager """ - for element in self.processLine(line)[1]: + for element in self.processLine(line, date)[1]: failregex = element[0] ip = element[1] unixTime = element[2] @@ -421,7 +428,8 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, returnRawHost=False, checkAllRegex=False): + def findFailure(self, tupleLine, date=None, returnRawHost=False, + checkAllRegex=False): failList = list() # Checks if we must ignore this line. @@ -432,7 +440,10 @@ class Filter(JailThread): return failList timeText = tupleLine[1] - if timeText: + if date: + self.__lastTimeText = timeText + self.__lastDate = date + elif timeText: dateTimeMatch = self.dateDetector.getTime(timeText) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 87daa8e6..180cb84b 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -22,7 +22,7 @@ __author__ = "Steven Hiscocks" __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import logging, datetime +import logging, datetime, time from distutils.version import LooseVersion from systemd import journal @@ -57,7 +57,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # Initialise systemd-journal connection self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x}) self.__matches = [] - self.setDatePattern("ISO8601") + self.setDatePattern(None) logSys.debug("Created FilterSystemd") @@ -162,8 +162,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover @staticmethod def formatJournalEntry(logentry): - logelements = [logentry.get('_SOURCE_REALTIME_TIMESTAMP', - logentry.get('__REALTIME_TIMESTAMP')).isoformat()] + logelements = [""] if logentry.get('_HOSTNAME'): logelements.append(logentry['_HOSTNAME']) if logentry.get('SYSLOG_IDENTIFIER'): @@ -188,18 +187,22 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements.append(logentry.get('MESSAGE', '')) try: - logline = u" ".join(logelements) + u"\n" + logline = u" ".join(logelements) except UnicodeDecodeError: # Python 2, so treat as string - logline = " ".join([str(logline) for logline in logelements]) + "\n" + logline = " ".join([str(logline) for logline in logelements]) except TypeError: # Python 3, one or more elements bytes logSys.warning("Error decoding log elements from journal: %s" % repr(logelements)) - logline = self._joinStrAndBytes(logelements) + "\n" + logline = self._joinStrAndBytes(logelements) - logSys.debug("Read systemd journal entry: %s" % repr(logline)) - return logline + date = logentry.get('_SOURCE_REALTIME_TIMESTAMP', + logentry.get('__REALTIME_TIMESTAMP')) + logSys.debug("Read systemd journal entry: %r" % + "".join([date.isoformat(), logline])) + return (('', date.isoformat(), logline), + time.mktime(date.timetuple()) + date.microsecond/1.0E6) ## # Main loop. @@ -232,7 +235,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover continue if logentry: self.processLineAndAdd( - self.formatJournalEntry(logentry)) + *self.formatJournalEntry(logentry)) self.__modified = True else: break @@ -243,7 +246,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) - self.dateDetector.sortTemplate() self.__modified = False self.__journal.wait(self.getSleepTime()) logSys.debug((self.jail is not None and self.jail.getName() diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b17bf363..ca273cb2 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -203,7 +203,8 @@ class BasicFilter(unittest.TestCase): self.assertEqual(self.filter.getUseDns(), 'no') def testGetSetDatePattern(self): - self.assertEqual(self.filter.getDatePattern(), None) + self.assertEqual(self.filter.getDatePattern(), + (None, "Default Detectors")) self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z") self.assertEqual(self.filter.getDatePattern(), ("^%Y-%m-%d-%H%M%S.%f %z",