From 218905c924b4cde3408ebd7b399712f08ec33b19 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 22 Mar 2018 10:16:40 +0100 Subject: [PATCH] performance optimization: findFailure, search regex etc, handling with buffer/tuple-lines optimized (especially multi-regex resp. multi-lines filters) --- fail2ban/server/failregex.py | 13 +++++- fail2ban/server/filter.py | 88 +++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index d5c9345f..40d106db 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -185,6 +185,13 @@ class Regex: def getRegex(self): return self._regex + ## + # Returns string buffer using join of the tupleLines. + # + @staticmethod + def _tupleLinesBuf(tupleLines): + return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n" + ## # Searches the regular expression. # @@ -194,8 +201,10 @@ class Regex: # @param a list of tupples. The tupples are ( prematch, datematch, postdatematch ) def search(self, tupleLines, orgLines=None): - self._matchCache = self._regexObj.search( - "\n".join("".join(value[::2]) for value in tupleLines) + "\n") + buf = tupleLines + if not isinstance(tupleLines, basestring): + buf = Regex._tupleLinesBuf(tupleLines) + self._matchCache = self._regexObj.search(buf) if self._matchCache: if orgLines is None: orgLines = tupleLines # if single-line: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 5b9125ed..0ba54be0 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -582,8 +582,9 @@ class Filter(JailThread): # @return: a boolean def ignoreLine(self, tupleLines): + buf = Regex._tupleLinesBuf(tupleLines) for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): - ignoreRegex.search(tupleLines) + ignoreRegex.search(buf, tupleLines) if ignoreRegex.hasMatched(): return ignoreRegexIndex return None @@ -626,6 +627,7 @@ class Filter(JailThread): def findFailure(self, tupleLine, date=None): failList = list() + ll = logSys.getEffectiveLevel() returnRawHost = self.returnRawHost cidr = IPAddr.CIDR_UNSPEC if self.__useDns == "raw": @@ -635,7 +637,7 @@ class Filter(JailThread): # Checks if we mut ignore this line. if self.ignoreLine([tupleLine[::2]]) is not None: # The ignoreregex matched. Return. - logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", + if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", "".join(tupleLine[::2])) return failList @@ -662,7 +664,7 @@ class Filter(JailThread): date = self.__lastDate if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): - logSys.log(5, "Ignore line since time %s < %s - %s", + if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", date, MyTime.time(), self.getFindTime()) return failList @@ -671,58 +673,60 @@ class Filter(JailThread): self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] else: orgBuffer = self.__lineBuffer = [tupleLine[:3]] - logSys.log(5, "Looking for match of %r", self.__lineBuffer) + if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer) + buf = Regex._tupleLinesBuf(self.__lineBuffer) # Pre-filter fail regex (if available): preGroups = {} if self.__prefRegex: - if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover - logSys.log(5, " Looking for prefregex %r", self.__prefRegex.getRegex()) - self.__prefRegex.search(self.__lineBuffer) + if ll <= 5: logSys.log(5, " Looking for prefregex %r", self.__prefRegex.getRegex()) + self.__prefRegex.search(buf, self.__lineBuffer) if not self.__prefRegex.hasMatched(): - logSys.log(5, " Prefregex not matched") + if ll <= 5: logSys.log(5, " Prefregex not matched") return failList preGroups = self.__prefRegex.getGroups() - logSys.log(7, " Pre-filter matched %s", preGroups) + if ll <= 7: logSys.log(7, " Pre-filter matched %s", preGroups) repl = preGroups.get('content') # Content replacement: if repl: del preGroups['content'] - self.__lineBuffer = [('', '', repl)] + self.__lineBuffer, buf = [('', '', repl)], None # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): - if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover - logSys.log(5, " Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex()) - failRegex.search(self.__lineBuffer, orgBuffer) - if not failRegex.hasMatched(): - continue - # The failregex matched. - logSys.log(7, " Matched failregex %d: %s", failRegexIndex, failRegex.getGroups()) - # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: - # The ignoreregex matched. Remove ignored match. - self.__lineBuffer = failRegex.getUnmatchedTupleLines() - logSys.log(7, " Matched ignoreregex and was ignored") - if not self.checkAllRegex: - break - else: - continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), timeText) - continue - self.__lineBuffer = failRegex.getUnmatchedTupleLines() - # retrieve failure-id, host, etc from failure match: try: + # buffer from tuples if changed: + if buf is None: + buf = Regex._tupleLinesBuf(self.__lineBuffer) + if ll <= 5: logSys.log(5, " Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex()) + failRegex.search(buf, orgBuffer) + if not failRegex.hasMatched(): + continue + # The failregex matched. + if ll <= 7: logSys.log(7, " Matched failregex %d: %s", failRegexIndex, failRegex.getGroups()) + # Checks if we must ignore this match. + if self.ignoreLine(failRegex.getMatchedTupleLines()) \ + is not None: + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None + if ll <= 7: logSys.log(7, " Matched ignoreregex and was ignored") + if not self.checkAllRegex: + break + else: + continue + if date is None: + logSys.warning( + "Found a match for %r but no valid date/time " + "found for %r. Please try setting a custom " + "date pattern (see man page jail.conf(5)). " + "If format is complex, please " + "file a detailed issue on" + " https://github.com/fail2ban/fail2ban/issues " + "in order to get support for this format.", + "\n".join(failRegex.getMatchedLines()), timeText) + continue + self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None + # retrieve failure-id, host, etc from failure match: raw = returnRawHost if preGroups: fail = preGroups.copy() @@ -735,7 +739,7 @@ class Filter(JailThread): fail = self._mergeFailure(mlfid, fail, failRegex) # bypass if no-failure case: if fail.get('nofail'): - logSys.log(7, "Nofail by mlfid %r in regex %s: %s", + if ll <= 7: logSys.log(7, "Nofail by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure")) if not self.checkAllRegex: return failList else: @@ -764,7 +768,7 @@ class Filter(JailThread): cidr = IPAddr.CIDR_RAW # if mlfid case (not failure): if host is None: - logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", + if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) if not self.checkAllRegex: return failList ips = [None]