Merge branch 'filter-failregex-return'

Conflicts:
	server/filter.py

DEVELOP

@@ -39,8 +39,9 @@ Filters
 
 * Include sample logs with 1.2.3.4 used for IP addresses and 
   example.com/example.org used for DNS names
-* Ensure ./fail2ban-regex testcases/files/logs/{samplelog} config/filter.d/{filter}.conf
+* Ensure sample log is provided in testcases/files/logs/ with same name as the
-  has matches for EVERY regex
+  filter. Each log line should include match meta data for time & IP above
+  every line (see other sample log files for examples)
 * Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if
   \d+ is sufficient
 * Use the functionality of regexs http://docs.python.org/2/library/re.html

fail2ban-regex

@@ -46,7 +46,6 @@ from client.configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 from server.filter import Filter
 from server.failregex import RegexException
-from server.datedetector import DateDetector
 
 from testcases.utils import FormatterWithTraceBack
 # Gets the instance of the logger.
@@ -130,7 +129,7 @@ class RegexStat(object):
 		return self._failregex
 
 	def appendIP(self, value):
-		self._ipList.extend(value)
+		self._ipList.append(value)
 
 	def getIPList(self):
 		return self._ipList
@@ -173,8 +172,6 @@ class Fail2banRegex(object):
 		self._ignoreregex = list()
 		self._failregex = list()
 		self._line_stats = LineStats()
-		self._dateDetector = DateDetector()
-		self._dateDetector.addDefaultTemplate()
 
 
 	def readRegex(self, value, regextype):
@@ -204,53 +201,41 @@ class Fail2banRegex(object):
 			regex_values = [RegexStat(value)]
 
 		setattr(self, "_" + regex, regex_values)
+		for regex in regex_values:
+			getattr(
+				self._filter,
+				'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True
 
 	def testIgnoreRegex(self, line):
 		found = False
-		for regex in self._ignoreregex:
+		try:
-			try:
+			ret = self._filter.ignoreLine(line)
-				self._filter.addIgnoreRegex(regex.getFailRegex())
+			if ret is not None:
-				try:
+				found = True
-					ret = self._filter.ignoreLine(line)
+				regex = self._ignoreregex[ret].inc()
-					if ret:
+		except RegexException, e:
-						found = True
+			print e
-						regex.inc()
+			return False
-				except RegexException, e:
-					print e
-					return False
-			finally:
-				self._filter.delIgnoreRegex(0)
 		return found
 
 	def testRegex(self, line):
-		found = False
+		try:
-		for regex in self._ignoreregex:
+			ret = self._filter.processLine(line, checkAllRegex=True)
-			self._filter.addIgnoreRegex(regex.getFailRegex())
+			for match in ret:
-		for regex in self._failregex:
+				# Append True/False flag depending if line was matched by
-			try:
+				# more than one regex
-				self._filter.addFailRegex(regex.getFailRegex())
+				match.append(len(ret)>1)
-				try:
+				regex = self._failregex[match[0]]
-					ret = self._filter.processLine(line)
+				regex.inc()
-					if len(ret):
+				regex.appendIP(match)
-						if found == True:
+		except RegexException, e:
-							ret[0].append(True)
+			print e
-						else:
+			return False
-							found = True
+		except IndexError:
-							ret[0].append(False)
+			print "Sorry, but no <host> found in regex"
-						regex.inc()
+			return False
-						regex.appendIP(ret)
+		return len(ret) > 0
-				except RegexException, e:
-					print e
-					return False
-				except IndexError:
-					print "Sorry, but no <host> found in regex"
-					return False
-			finally:
-				self._filter.delFailRegex(0)
-		for regex in self._ignoreregex:
-			self._filter.delIgnoreRegex(0)
-		return found
 
 
 	def process(self, test_lines):
@@ -259,9 +244,6 @@ class Fail2banRegex(object):
 			if line.startswith('#') or not line.strip():
 				# skip comment and empty lines
 				continue
-
-			self._dateDetector.matchTime(line)
-
 			is_ignored = fail2banRegex.testIgnoreRegex(line)
 			if is_ignored:
 				self._line_stats.ignored_lines.append(line)
@@ -302,10 +284,13 @@ class Fail2banRegex(object):
 
 				if self._verbose and len(failregex.getIPList()):
 					for ip in failregex.getIPList():
-						timeTuple = time.localtime(ip[1])
+						timeTuple = time.localtime(ip[2])
 						timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple)
-						out.append("    %s  %s%s" % (
+						out.append(
-							ip[0], timeString, ip[2] and " (already matched)" or ""))
+							"    %s  %s%s" % (
+								ip[1],
+								timeString,
+								ip[3] and " (multiple regex matched)" or ""))
 
 			print "\n%s: %d total" % (title, total)
 			pprint_list(out, " #) [# of hits] regular expression")
@@ -318,7 +303,7 @@ class Fail2banRegex(object):
 
 		print "\nDate template hits:"
 		out = []
-		for template in self._dateDetector.getTemplates():
+		for template in self._filter.dateDetector.getTemplates():
 			if self._verbose or template.getHits():
 				out.append("[%d] %s" % (template.getHits(), template.getName()))
 		pprint_list(out, "[# of hits] date format")

server/datedetector.py

@@ -174,6 +174,7 @@ class DateDetector:
 				match = template.matchDate(line)
 				if not match is None:
 					logSys.debug("Matched time template %s" % template.getName())
+					template.incHits()
 					return match
 			return None
 		finally:

server/datetemplate.py

@@ -59,11 +59,12 @@ class DateTemplate:
 	
 	def getHits(self):
 		return self.__hits
+
+	def incHits(self):
+		self.__hits += 1
 	
 	def matchDate(self, line):
 		dateMatch = self.__cRegex.search(line)
-		if not dateMatch is None:
-			self.__hits += 1
 		return dateMatch
 	
 	def getDate(self, line):

server/filter.py

@@ -284,7 +284,7 @@ class Filter(JailThread):
 		return False
 
 
-	def processLine(self, line, returnRawHost=False):
+	def processLine(self, line, returnRawHost=False, checkAllRegex=False):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
 		try:
@@ -306,14 +306,15 @@ class Filter(JailThread):
 		else:
 			timeLine = l
 			logLine = l
-		return self.findFailure(timeLine, logLine, returnRawHost)
+		return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex)
 
 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
 		"""
 		for element in self.processLine(line):
-			ip = element[0]
+			failregex = element[0]
-			unixTime = element[1]
+			ip = element[1]
+			unixTime = element[2]
 			logSys.debug("Processing line with time:%s and ip:%s"
 						 % (unixTime, ip))
 			if unixTime < MyTime.time() - self.getFindTime():
@@ -335,11 +336,11 @@ class Filter(JailThread):
 	# @return: a boolean
 
 	def ignoreLine(self, line):
-		for ignoreRegex in self.__ignoreRegex:
+		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
 			ignoreRegex.search(line)
 			if ignoreRegex.hasMatched():
-				return True
+				return ignoreRegexIndex
-		return False
+		return None
 
 	##
 	# Finds the failure in a line given split into time and log parts.
@@ -348,21 +349,22 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, timeLine, logLine, returnRawHost=False):
+	def findFailure(self, timeLine, logLine,
+			returnRawHost=False, checkAllRegex=False):
 		logSys.log(5, "Date: %r, message: %r", timeLine, logLine)
 		failList = list()
 		# Checks if we must ignore this line.
-		if self.ignoreLine(logLine):
+		if self.ignoreLine(logLine) is not None:
 			# The ignoreregex matched. Return.
 			logSys.log(7, "Matched ignoreregex and was ignored")
 			return failList
+		date = self.dateDetector.getUnixTime(timeLine)
 		# Iterates over all the regular expressions.
-		for failRegex in self.__failRegex:
+		for failRegexIndex, failRegex in enumerate(self.__failRegex):
 			failRegex.search(logLine)
 			if failRegex.hasMatched():
 				# The failregex matched.
 				logSys.log(7, "Matched %s", failRegex)
-				date = self.dateDetector.getUnixTime(timeLine)
 				if date is None:
 					logSys.debug("Found a match for %r but no valid date/time "
 								 "found for %r. Please file a detailed issue on"
@@ -373,14 +375,16 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							failList.append([host, date])
+							failList.append([failRegexIndex, host, date])
-							break
+							if not checkAllRegex:
-						ipMatch = DNSUtils.textToIp(host, self.__useDns)
+								break
-						if ipMatch:
+						else:
-							for ip in ipMatch:
+							ipMatch = DNSUtils.textToIp(host, self.__useDns)
-								failList.append([ip, date])
+							if ipMatch:
-							# We matched a regex, it is enough to stop.
+								for ip in ipMatch:
-							break
+									failList.append([failRegexIndex, ip, date])
+								if not checkAllRegex:
+									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
 		return failList

testcases/samplestestcase.py

@@ -82,6 +82,7 @@ def testSampleRegexsFactory(name):
 		logFile = fileinput.FileInput(
 			os.path.join(TEST_FILES_DIR, "logs", name))
 
+		regexsUsed = set()
 		for line in logFile:
 			jsonREMatch = re.match("^# ?failJSON:(.+)$", line)
 			if jsonREMatch:
@@ -96,7 +97,8 @@ def testSampleRegexsFactory(name):
 			else:
 				faildata = {}
 
-			ret = self.filter.processLine(line, returnRawHost=True)
+			ret = self.filter.processLine(
+				line, returnRawHost=True, checkAllRegex=True)
 			if not ret:
 				# Check line is flagged as none match
 				self.assertFalse(faildata.get('match', True),
@@ -107,15 +109,28 @@ def testSampleRegexsFactory(name):
 				self.assertTrue(faildata.get('match', False),
 					"Line matched when shouldn't have: %s:%i %r" %
 					(logFile.filename(), logFile.filelineno(), line))
-				self.assertEqual(len(ret), 1)
+				self.assertEqual(len(ret), 1, "Multiple regexs matched")
 				# Verify timestamp and host as expected
-				host, time = ret[0]
+				failregex, host, time = ret[0]
 				self.assertEqual(host, faildata.get("host", None))
 				self.assertEqual(
 					datetime.datetime.fromtimestamp(time),
 					datetime.datetime.strptime(
 						faildata.get("time", None), "%Y-%m-%dT%H:%M:%S"))
 
+				regexsUsed.add(failregex)
+
+		# TODO: Remove exception handling once all regexs have samples
+		for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()):
+			try:
+				self.assertTrue(
+					failRegexIndex in regexsUsed,
+					"Regex for filter '%s' has no samples: %i: %r" %
+						(name, failRegexIndex, failRegex))
+			except AssertionError:
+				print "I: Regex for filter '%s' has no samples: %i: %r" % (
+					name, failRegexIndex, failRegex)
+
 	return testFilter
 
 for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")):