performance optimization: findFailure, search regex etc, handling with buffer/tuple-lines optimized (especially multi-regex resp. multi-lines filters)

fail2ban/server/failregex.py

@@ -185,6 +185,13 @@ class Regex:
 	def getRegex(self):
 		return self._regex
 	
+	##
+	# Returns string buffer using join of the tupleLines.
+	#
+	@staticmethod
+	def _tupleLinesBuf(tupleLines):
+		return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n"
+
 	##
 	# Searches the regular expression.
 	#
@@ -194,8 +201,10 @@ class Regex:
 	# @param a list of tupples. The tupples are ( prematch, datematch, postdatematch )
 	
 	def search(self, tupleLines, orgLines=None):
-		self._matchCache = self._regexObj.search(
-			"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
+		buf = tupleLines
+		if not isinstance(tupleLines, basestring):
+			buf = Regex._tupleLinesBuf(tupleLines)
+		self._matchCache = self._regexObj.search(buf)
 		if self._matchCache:
 			if orgLines is None: orgLines = tupleLines
 			# if single-line:

fail2ban/server/filter.py

@@ -582,8 +582,9 @@ class Filter(JailThread):
 	# @return: a boolean
 
 	def ignoreLine(self, tupleLines):
+		buf = Regex._tupleLinesBuf(tupleLines)
 		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
-			ignoreRegex.search(tupleLines)
+			ignoreRegex.search(buf, tupleLines)
 			if ignoreRegex.hasMatched():
 				return ignoreRegexIndex
 		return None
@@ -626,6 +627,7 @@ class Filter(JailThread):
 	def findFailure(self, tupleLine, date=None):
 		failList = list()
 
+		ll = logSys.getEffectiveLevel()
 		returnRawHost = self.returnRawHost
 		cidr = IPAddr.CIDR_UNSPEC
 		if self.__useDns == "raw":
@@ -635,7 +637,7 @@ class Filter(JailThread):
 		# Checks if we mut ignore this line.
 		if self.ignoreLine([tupleLine[::2]]) is not None:
 			# The ignoreregex matched. Return.
-			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
+			if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
 				"".join(tupleLine[::2]))
 			return failList
 
@@ -662,7 +664,7 @@ class Filter(JailThread):
 			date = self.__lastDate
 
 		if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
-			logSys.log(5, "Ignore line since time %s < %s - %s", 
+			if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", 
 				date, MyTime.time(), self.getFindTime())
 			return failList
 
@@ -671,40 +673,43 @@ class Filter(JailThread):
 				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
 		else:
 			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
-		logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		buf = Regex._tupleLinesBuf(self.__lineBuffer)
 
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
-				logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
-			self.__prefRegex.search(self.__lineBuffer)
+			if ll <= 5: logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
+			self.__prefRegex.search(buf, self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
-				logSys.log(5, "  Prefregex not matched")
+				if ll <= 5: logSys.log(5, "  Prefregex not matched")
 				return failList
 			preGroups = self.__prefRegex.getGroups()
-			logSys.log(7, "  Pre-filter matched %s", preGroups)
+			if ll <= 7: logSys.log(7, "  Pre-filter matched %s", preGroups)
 			repl = preGroups.get('content')
 			# Content replacement:
 			if repl:
 				del preGroups['content']
-				self.__lineBuffer = [('', '', repl)]
+				self.__lineBuffer, buf = [('', '', repl)], None
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
-				logSys.log(5, "  Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex())
-			failRegex.search(self.__lineBuffer, orgBuffer)
+			try:
+				# buffer from tuples if changed: 
+				if buf is None:
+					buf = Regex._tupleLinesBuf(self.__lineBuffer)
+				if ll <= 5: logSys.log(5, "  Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex())
+				failRegex.search(buf, orgBuffer)
 				if not failRegex.hasMatched():
 					continue
 				# The failregex matched.
-			logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, failRegex.getGroups())
+				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, failRegex.getGroups())
 				# Checks if we must ignore this match.
 				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
 						is not None:
 					# The ignoreregex matched. Remove ignored match.
-				self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-				logSys.log(7, "  Matched ignoreregex and was ignored")
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
+					if ll <= 7: logSys.log(7, "  Matched ignoreregex and was ignored")
 					if not self.checkAllRegex:
 						break
 					else:
@@ -720,9 +725,8 @@ class Filter(JailThread):
 						"in order to get support for this format.",
 						 "\n".join(failRegex.getMatchedLines()), timeText)
 					continue
-			self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+				self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
 				# retrieve failure-id, host, etc from failure match:
-			try:
 				raw = returnRawHost
 				if preGroups:
 					fail = preGroups.copy()
@@ -735,7 +739,7 @@ class Filter(JailThread):
 					fail = self._mergeFailure(mlfid, fail, failRegex)
 					# bypass if no-failure case:
 					if fail.get('nofail'):
-						logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
+						if ll <= 7: logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
 							mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure"))
 						if not self.checkAllRegex: return failList
 				else:
@@ -764,7 +768,7 @@ class Filter(JailThread):
 						cidr = IPAddr.CIDR_RAW
 				# if mlfid case (not failure):
 				if host is None:
-					logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
+					if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
 						mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier"))
 					if not self.checkAllRegex: return failList
 					ips = [None]