ENH+BF+TST: Filter now returns reference to failregex and ignoreregex

This avoids duplication of code across fail2ban-regex and samples test
cases. This also now more neatly resolves the issue of double counting
date templates matches in fail2ban-regex.
In addition, the samples test cases now also print a warning message
that not all regexs have samples for them, with future plan to change
this to an assertion.

fail2ban-regex

@@ -46,7 +46,6 @@ from client.configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 from server.filter import Filter
 from server.failregex import RegexException
-from server.datedetector import DateDetector
 
 from testcases.utils import FormatterWithTraceBack
 # Gets the instance of the logger.
@@ -130,7 +129,7 @@ class RegexStat(object):
 		return self._failregex
 
 	def appendIP(self, value):
-		self._ipList.extend(value)
+		self._ipList.append(value)
 
 	def getIPList(self):
 		return self._ipList
@@ -173,8 +172,6 @@ class Fail2banRegex(object):
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._line_stats = LineStats()
-		self._dateDetector = DateDetector()
-		self._dateDetector.addDefaultTemplate()
 
 
 	def readRegex(self, value, regextype):
@@ -204,53 +201,39 @@ class Fail2banRegex(object):
 			regex_values = [RegexStat(value)]
 
 		setattr(self, "_" + regex, regex_values)
+		for regex in regex_values:
+			getattr(
+				self._filter,
+				'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True
 
 	def testIgnoreRegex(self, line):
 		found = False
-		for regex in self._ignoreregex:
-			try:
-				self._filter.addIgnoreRegex(regex.getFailRegex())
 		try:
 			ret = self._filter.ignoreLine(line)
-					if ret:
+			if ret is not None:
 				found = True
-						regex.inc()
+				regex = self._ignoreregex[ret].inc()
 		except RegexException, e:
 			print e
 			return False
-			finally:
-				self._filter.delIgnoreRegex(0)
 		return found
 
 	def testRegex(self, line):
-		found = False
-		for regex in self._ignoreregex:
-			self._filter.addIgnoreRegex(regex.getFailRegex())
-		for regex in self._failregex:
 		try:
-				self._filter.addFailRegex(regex.getFailRegex())
+			ret = self._filter.processLine(line, checkAllRegex=True)
-				try:
+			for match in ret:
-					ret = self._filter.processLine(line)
+				match.append(len(ret)>1)
-					if len(ret):
+				regex = self._failregex[match[0]]
-						if found == True:
-							ret[0].append(True)
-						else:
-							found = True
-							ret[0].append(False)
 				regex.inc()
-						regex.appendIP(ret)
+				regex.appendIP(match)
 		except RegexException, e:
 			print e
 			return False
 		except IndexError:
 			print "Sorry, but no <host> found in regex"
 			return False
-			finally:
+		return len(ret) > 0
-				self._filter.delFailRegex(0)
-		for regex in self._ignoreregex:
-			self._filter.delIgnoreRegex(0)
-		return found
 
 
 	def process(self, test_lines):
@@ -259,9 +242,6 @@ class Fail2banRegex(object):
 			if line.startswith('#') or not line.strip():
 				# skip comment and empty lines
 				continue
-
-			self._dateDetector.matchTime(line)
-
 			is_ignored = fail2banRegex.testIgnoreRegex(line)
 			if is_ignored:
 				self._line_stats.ignored_lines.append(line)
@@ -302,10 +282,13 @@ class Fail2banRegex(object):
 
 				if self._verbose and len(failregex.getIPList()):
 					for ip in failregex.getIPList():
-						timeTuple = time.localtime(ip[1])
+						timeTuple = time.localtime(ip[2])
 						timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple)
-						out.append("    %s  %s%s" % (
+						out.append(
-							ip[0], timeString, ip[2] and " (already matched)" or ""))
+							"    %s  %s%s" % (
+								ip[1],
+								timeString,
+								ip[3] and " (multiple regex matched)" or ""))
 
 			print "\n%s: %d total" % (title, total)
 			pprint_list(out, " #) [# of hits] regular expression")
@@ -318,7 +301,7 @@ class Fail2banRegex(object):
 
 		print "\nDate template hits:"
 		out = []
-		for template in self._dateDetector.getTemplates():
+		for template in self._filter.dateDetector.getTemplates():
 			if self._verbose or template.getHits():
 				out.append("[%d] %s" % (template.getHits(), template.getName()))
 		pprint_list(out, "[# of hits] date format")

server/datedetector.py

@@ -174,6 +174,7 @@ class DateDetector:
 				match = template.matchDate(line)
 				if not match is None:
 					logSys.debug("Matched time template %s" % template.getName())
+					template.incHits()
 					return match
 			return None
 		finally:

server/datetemplate.py

@@ -60,10 +60,11 @@ class DateTemplate:
 	def getHits(self):
 		return self.__hits
 
+	def incHits(self):
+		self.__hits += 1
+	
 	def matchDate(self, line):
 		dateMatch = self.__cRegex.search(line)
-		if not dateMatch is None:
-			self.__hits += 1
 		return dateMatch
 	
 	def getDate(self, line):

server/filter.py

@@ -284,7 +284,7 @@ class Filter(JailThread):
 		return False
 
 
-	def processLine(self, line, returnRawHost=False):
+	def processLine(self, line, returnRawHost=False, checkAllRegex=False):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
 		try:
@@ -306,14 +306,15 @@ class Filter(JailThread):
 		else:
 			timeLine = l
 			logLine = l
-		return self.findFailure(timeLine, logLine, returnRawHost)
+		return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex)
 
 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
 		"""
 		for element in self.processLine(line):
-			ip = element[0]
+			failregex = element[0]
-			unixTime = element[1]
+			ip = element[1]
+			unixTime = element[2]
 			logSys.debug("Processing line with time:%s and ip:%s"
 						 % (unixTime, ip))
 			if unixTime < MyTime.time() - self.getFindTime():
@@ -335,11 +336,11 @@ class Filter(JailThread):
 	# @return: a boolean
 
 	def ignoreLine(self, line):
-		for ignoreRegex in self.__ignoreRegex:
+		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
 			ignoreRegex.search(line)
 			if ignoreRegex.hasMatched():
-				return True
+				return ignoreRegexIndex
-		return False
+		return None
 
 	##
 	# Finds the failure in a line given split into time and log parts.
@@ -348,18 +349,19 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, timeLine, logLine, returnRawHost=False):
+	def findFailure(self, timeLine, logLine,
+			returnRawHost=False, checkAllRegex=False):
 		failList = list()
 		# Checks if we must ignore this line.
-		if self.ignoreLine(logLine):
+		if self.ignoreLine(logLine) is not None:
 			# The ignoreregex matched. Return.
 			return failList
+		date = self.dateDetector.getUnixTime(timeLine)
 		# Iterates over all the regular expressions.
-		for failRegex in self.__failRegex:
+		for failRegexIndex, failRegex in enumerate(self.__failRegex):
 			failRegex.search(logLine)
 			if failRegex.hasMatched():
 				# The failregex matched.
-				date = self.dateDetector.getUnixTime(timeLine)
 				logSys.log(7, "Date: %r, message: %r",
 							  timeLine, logLine)
 				if date is None:
@@ -372,13 +374,15 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							failList.append([host, date])
+							failList.append([failRegexIndex, host, date])
+							if not checkAllRegex:
 								break
+						else:
 							ipMatch = DNSUtils.textToIp(host, self.__useDns)
 							if ipMatch:
 								for ip in ipMatch:
-								failList.append([ip, date])
+									failList.append([failRegexIndex, ip, date])
-							# We matched a regex, it is enough to stop.
+								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
 						logSys.error(e)

testcases/samplestestcase.py

@@ -82,6 +82,7 @@ def testSampleRegexsFactory(name):
 		logFile = fileinput.FileInput(
 			os.path.join(TEST_FILES_DIR, "logs", name))
 
+		regexsUsed = set()
 		for line in logFile:
 			jsonREMatch = re.match("^# ?failJSON:(.+)$", line)
 			if jsonREMatch:
@@ -96,7 +97,8 @@ def testSampleRegexsFactory(name):
 			else:
 				faildata = {}
 
-			ret = self.filter.processLine(line, returnRawHost=True)
+			ret = self.filter.processLine(
+				line, returnRawHost=True, checkAllRegex=True)
 			if not ret:
 				# Check line is flagged as none match
 				self.assertFalse(faildata.get('match', True),
@@ -107,15 +109,28 @@ def testSampleRegexsFactory(name):
 				self.assertTrue(faildata.get('match', False),
 					"Line matched when shouldn't have: %s:%i %r" %
 					(logFile.filename(), logFile.filelineno(), line))
-				self.assertEqual(len(ret), 1)
+				self.assertEqual(len(ret), 1, "Multiple regexs matched")
 				# Verify timestamp and host as expected
-				host, time = ret[0]
+				failregex, host, time = ret[0]
 				self.assertEqual(host, faildata.get("host", None))
 				self.assertEqual(
 					datetime.datetime.fromtimestamp(time),
 					datetime.datetime.strptime(
 						faildata.get("time", None), "%Y-%m-%dT%H:%M:%S"))
 
+				regexsUsed.add(failregex)
+
+		# TODO: Remove exception handling once all regexs have samples
+		for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()):
+			try:
+				self.assertTrue(
+					failRegexIndex in regexsUsed,
+					"Regex for filter '%s' has no samples: %i: %r" %
+						(name, failRegexIndex, failRegex))
+			except AssertionError:
+				print "I: Regex for filter '%s' has no samples: %i: %r" % (
+					name, failRegexIndex, failRegex)
+
 	return testFilter
 
 for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")):