Merge pull request #527 from kwirk/systemd-backend-datetime

ENH: Pass date time straight from systemd backend

bin/fail2ban-regex

@@ -239,7 +239,9 @@ class Fail2banRegex(object):
 		if not self._datepattern_set:
 			self._filter.setDatePattern(pattern)
 			self._datepattern_set = True
-			print "Use      datepattern : %s" % self._filter.getDatePattern()[1]
+			if pattern is not None:
+				print "Use      datepattern : %s" % (
+					self._filter.getDatePattern()[1], )
 
 	def setMaxLines(self, v):
 		if not self._maxlines_set:
@@ -318,11 +320,11 @@ class Fail2banRegex(object):
 			return False
 		return found
 
-	def testRegex(self, line):
+	def testRegex(self, line, date=None):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
 		try:
-			line, ret = self._filter.processLine(line, checkAllRegex=True)
+			line, ret = self._filter.processLine(line, date, checkAllRegex=True)
 			for match in ret:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
@@ -353,11 +355,16 @@ class Fail2banRegex(object):
 	def process(self, test_lines):
 
 		for line_no, line in enumerate(test_lines):
-			line = line.strip('\r\n')
-			if line.startswith('#') or not line:
-				# skip comment and empty lines
-				continue
-			line_datetimestripped, ret = fail2banRegex.testRegex(line)
+			if isinstance(line, tuple):
+				line_datetimestripped, ret = fail2banRegex.testRegex(
+					line[0], line[1])
+				line = "".join(line[0])
+			else:
+				line = line.rstrip('\r\n')
+				if line.startswith('#') or not line:
+					# skip comment and empty lines
+					continue
+				line_datetimestripped, ret = fail2banRegex.testRegex(line)
 			is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped)
 
 			if is_ignored:
@@ -373,7 +380,7 @@ class Fail2banRegex(object):
 					self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
 			self._line_stats.tested += 1
 
-			if line_no % 10 == 0:
+			if line_no % 10 == 0 and self._filter.dateDetector is not None:
 				self._filter.dateDetector.sortTemplate()
 
 
@@ -439,12 +446,14 @@ class Fail2banRegex(object):
 		_ = print_failregexes("Ignoreregex", self._ignoreregex)
 
 
-		print "\nDate template hits:"
-		out = []
-		for template in self._filter.dateDetector.getTemplates():
-			if self._verbose or template.getHits():
-				out.append("[%d] %s" % (template.getHits(), template.getName()))
-		pprint_list(out, "[# of hits] date format")
+		if self._filter.dateDetector is not None:
+			print "\nDate template hits:"
+			out = []
+			for template in self._filter.dateDetector.getTemplates():
+				if self._verbose or template.getHits():
+					out.append("[%d] %s" % (
+						template.getHits(), template.getName()))
+			pprint_list(out, "[# of hits] date format")
 
 		print "\nLines: %s" % self._line_stats
 
@@ -523,7 +532,7 @@ if __name__ == "__main__":
 			sys.exit(-1)
 		myjournal = journal.Reader(converters={'__CURSOR': lambda x: x})
 		journalmatch = fail2banRegex._journalmatch
-		fail2banRegex.setDatePattern("ISO8601")
+		fail2banRegex.setDatePattern(None)
 		if journalmatch:
 			try:
 				for element in journalmatch:

fail2ban/client/beautifier.py

@@ -136,7 +136,7 @@ class Beautifier:
 			elif inC[2] == "datepattern":
 				msg = "Current date pattern set to: "
 				if response is None:
-					msg = msg + "Default Detectors"
+					msg = msg + "Not set/required"
 				elif response[0] is None:
 					msg = msg + "%s" % response[1]
 				else:

fail2ban/server/filter.py

@@ -199,8 +199,10 @@ class Filter(JailThread):
 	# @param pattern the date template pattern
 
 	def setDatePattern(self, pattern):
-		dateDetector = DateDetector()
-		if pattern.upper() == "ISO8601":
+		if pattern is None:
+			self.dateDetector = None
+			return
+		elif pattern.upper() == "ISO8601":
 			template = DateISO8601()
 			template.setName("ISO8601")
 		elif pattern.upper() == "EPOCH":
@@ -215,8 +217,8 @@ class Filter(JailThread):
 				template.setPattern(pattern[1:], anchor=True)
 			else:
 				template.setPattern(pattern, anchor=False)
-		dateDetector.appendTemplate(template)
-		self.dateDetector = dateDetector
+		self.dateDetector = DateDetector()
+		self.dateDetector.appendTemplate(template)
 		logSys.info("Date pattern set to `%r`: `%s`" %
 			(pattern, template.getName()))
 		logSys.debug("Date pattern regex for %r: %s" %
@@ -228,17 +230,18 @@ class Filter(JailThread):
 	# @return pattern of the date template pattern
 
 	def getDatePattern(self):
-		templates = self.dateDetector.getTemplates()
-		if len(templates) > 1:
-			return None # Default Detectors in use
-		elif len(templates) == 1:
-			if hasattr(templates[0], "getPattern"):
-				pattern =  templates[0].getPattern()
-				if templates[0].getRegex()[0] == "^":
-					pattern = "^" + pattern
-			else:
-				pattern = None
-			return pattern, templates[0].getName()
+		if self.dateDetector is not None:
+			templates = self.dateDetector.getTemplates()
+			if len(templates) > 1:
+				return None, "Default Detectors"
+			elif len(templates) == 1:
+				if hasattr(templates[0], "getPattern"):
+					pattern =  templates[0].getPattern()
+					if templates[0].getRegex()[0] == "^":
+						pattern = "^" + pattern
+				else:
+					pattern = None
+				return pattern, templates[0].getName()
 
 	##
 	# Set the maximum retry value.
@@ -361,28 +364,32 @@ class Filter(JailThread):
 		return False
 
 
-	def processLine(self, line, returnRawHost=False, checkAllRegex=False):
+	def processLine(self, line, date=None, returnRawHost=False,
+		checkAllRegex=False):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
-		l = line.rstrip('\r\n')
-		logSys.log(7, "Working on line %r", line)
-
-		timeMatch = self.dateDetector.matchTime(l)
-		if timeMatch:
-			tupleLine  = (
-				l[:timeMatch.start()],
-				l[timeMatch.start():timeMatch.end()],
-				l[timeMatch.end():])
+		if date:
+			tupleLine = line
 		else:
-			tupleLine = (l, "", "")
+			l = line.rstrip('\r\n')
+			logSys.log(7, "Working on line %r", line)
+
+			timeMatch = self.dateDetector.matchTime(l)
+			if timeMatch:
+				tupleLine  = (
+					l[:timeMatch.start()],
+					l[timeMatch.start():timeMatch.end()],
+					l[timeMatch.end():])
+			else:
+				tupleLine = (l, "", "")
 
 		return "".join(tupleLine[::2]), self.findFailure(
-			tupleLine, returnRawHost, checkAllRegex)
+			tupleLine, date, returnRawHost, checkAllRegex)
 
-	def processLineAndAdd(self, line):
+	def processLineAndAdd(self, line, date=None):
 		"""Processes the line for failures and populates failManager
 		"""
-		for element in self.processLine(line)[1]:
+		for element in self.processLine(line, date)[1]:
 			failregex = element[0]
 			ip = element[1]
 			unixTime = element[2]
@@ -421,7 +428,8 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, tupleLine, returnRawHost=False, checkAllRegex=False):
+	def findFailure(self, tupleLine, date=None, returnRawHost=False,
+		checkAllRegex=False):
 		failList = list()
 
 		# Checks if we must ignore this line.
@@ -432,7 +440,10 @@ class Filter(JailThread):
 			return failList
 
 		timeText = tupleLine[1]
-		if timeText:
+		if date:
+			self.__lastTimeText = timeText
+			self.__lastDate = date
+		elif timeText:
 
 			dateTimeMatch = self.dateDetector.getTime(timeText)
 

fail2ban/server/filtersystemd.py

@@ -22,7 +22,7 @@ __author__ = "Steven Hiscocks"
 __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
-import logging, datetime
+import logging, datetime, time
 from distutils.version import LooseVersion
 
 from systemd import journal
@@ -57,7 +57,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		# Initialise systemd-journal connection
 		self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x})
 		self.__matches = []
-		self.setDatePattern("ISO8601")
+		self.setDatePattern(None)
 		logSys.debug("Created FilterSystemd")
 
 
@@ -162,8 +162,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 	@staticmethod
 	def formatJournalEntry(logentry):
-		logelements = [logentry.get('_SOURCE_REALTIME_TIMESTAMP',
-			logentry.get('__REALTIME_TIMESTAMP')).isoformat()]
+		logelements = [""]
 		if logentry.get('_HOSTNAME'):
 			logelements.append(logentry['_HOSTNAME'])
 		if logentry.get('SYSLOG_IDENTIFIER'):
@@ -188,18 +187,22 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			logelements.append(logentry.get('MESSAGE', ''))
 
 		try:
-			logline = u" ".join(logelements) + u"\n"
+			logline = u" ".join(logelements)
 		except UnicodeDecodeError:
 			# Python 2, so treat as string
-			logline = " ".join([str(logline) for logline in logelements]) + "\n"
+			logline = " ".join([str(logline) for logline in logelements])
 		except TypeError:
 			# Python 3, one or more elements bytes
 			logSys.warning("Error decoding log elements from journal: %s" %
 				repr(logelements))
-			logline =  self._joinStrAndBytes(logelements) + "\n"
+			logline =  self._joinStrAndBytes(logelements)
 
-		logSys.debug("Read systemd journal entry: %s" % repr(logline))
-		return logline
+		date = logentry.get('_SOURCE_REALTIME_TIMESTAMP',
+				logentry.get('__REALTIME_TIMESTAMP'))
+		logSys.debug("Read systemd journal entry: %r" %
+			"".join([date.isoformat(), logline]))
+		return (('', date.isoformat(), logline),
+			time.mktime(date.timetuple()) + date.microsecond/1.0E6)
 
 	##
 	# Main loop.
@@ -232,7 +235,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 						continue
 					if logentry:
 						self.processLineAndAdd(
-							self.formatJournalEntry(logentry))
+							*self.formatJournalEntry(logentry))
 						self.__modified = True
 					else:
 						break
@@ -243,7 +246,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 							self.jail.putFailTicket(ticket)
 					except FailManagerEmpty:
 						self.failManager.cleanup(MyTime.time())
-					self.dateDetector.sortTemplate()
 					self.__modified = False
 			self.__journal.wait(self.getSleepTime())
 		logSys.debug((self.jail is not None and self.jail.getName()

fail2ban/tests/filtertestcase.py

@@ -203,7 +203,8 @@ class BasicFilter(unittest.TestCase):
 		self.assertEqual(self.filter.getUseDns(), 'no')
 
 	def testGetSetDatePattern(self):
-		self.assertEqual(self.filter.getDatePattern(), None)
+		self.assertEqual(self.filter.getDatePattern(),
+			(None, "Default Detectors"))
 		self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z")
 		self.assertEqual(self.filter.getDatePattern(),
 			("^%Y-%m-%d-%H%M%S.%f %z",