Merge branch 'datepatterns' into datepatterns-dateregex

Conflicts:
	bin/fail2ban-regex
	fail2ban/client/beautifier.py
	fail2ban/server/datedetector.py

bin/fail2ban-regex

@@ -88,6 +88,8 @@ IGNOREREGEX:
 				version="%prog " + version)
 
 	p.add_options([
+		Option("-d", "--datepattern",
+			   help="set custom pattern used to match date/times"),
 		Option("-e", "--encoding",
 			   help="File encoding. Default: system locale"),
 		Option("-L", "--maxlines", type=int, default=0,
@@ -179,6 +181,9 @@ class Fail2banRegex(object):
 		self._maxlines_set = False		  # so we allow to override maxlines in cmdline
 		self._journalmatch = None
 
+		if opts.datepattern:
+			self.setDatePattern(opts.datepattern)
+
 		if opts.encoding:
 			self.encoding = opts.encoding
 		else:
@@ -194,6 +199,8 @@ class Fail2banRegex(object):
 		if opts.journalmatch is not None:
 			self.setJournalMatch(opts.journalmatch.split())
 
+	def setDatePattern(self, pattern):
+		self._filter.setDatePattern(pattern)
 
 	def setMaxLines(self, v):
 		if not self._maxlines_set:

fail2ban/client/beautifier.py

@@ -119,6 +119,12 @@ class Beautifier:
 				else:
 					msg = "Current match filter:\n"
 					msg += ' + '.join(" ".join(res) for res in response)
+			elif inC[2] == "datepattern":
+				msg = "Current date pattern set to: "
+				if response is None:
+					msg = msg + "Default Detectors"
+				else:
+					msg = msg + "%s (%s)" % response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"

fail2ban/protocol.py

@@ -63,6 +63,7 @@ protocol = [
 ["set <JAIL> delignoreregex <INDEX>", "removes the regular expression at <INDEX> for ignoreregex"], 
 ["set <JAIL> findtime <TIME>", "sets the number of seconds <TIME> for which the filter will look back for <JAIL>"], 
 ["set <JAIL> bantime <TIME>", "sets the number of seconds <TIME> a host will be banned for <JAIL>"], 
+["set <JAIL> datepattern <PATTERN>", "sets the <PATTERN> used to match date/times for <JAIL>"],
 ["set <JAIL> usedns <VALUE>", "sets the usedns mode for <JAIL>"],
 ["set <JAIL> banip <IP>", "manually Ban <IP> for <JAIL>"], 
 ["set <JAIL> unbanip <IP>", "manually Unban <IP> in <JAIL>"], 
@@ -87,6 +88,7 @@ protocol = [
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],
 ["get <JAIL> findtime", "gets the time for which the filter will look back for failures for <JAIL>"],
 ["get <JAIL> bantime", "gets the time a host is banned for <JAIL>"],
+["get <JAIL> datepattern", "gets the patern used to match date/times for <JAIL>"],
 ["get <JAIL> usedns", "gets the usedns setting for <JAIL>"],
 ["get <JAIL> maxretry", "gets the number of failures allowed for <JAIL>"],
 ["get <JAIL> maxlines", "gets the number of lines to buffer for <JAIL>"],

fail2ban/server/datedetector.py

@@ -23,7 +23,7 @@ __license__ = "GPL"
 
 import time, logging
 
-from datetemplate import DateStrptime, DateTai64n, DateEpoch, DateISO8601
+from datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601
 from threading import Lock
 
 # Gets the instance of the logger.
@@ -43,130 +43,63 @@ class DateDetector:
 		self.__known_names.add(name)
 		self.__templates.append(template)
 	
+	def appendTemplate(self, template, **kwargs):
+		if isinstance(template, str):
+			template = DatePatternRegex(template, **kwargs)
+		else:
+			assert not kwargs
+		DateDetector._appendTemplate(self, template)
+
 	def addDefaultTemplate(self):
 		self.__lock.acquire()
 		try:
-			# standard
-			template = DateStrptime()
-			template.setName("MONTH Day Hour:Minute:Second")
-			template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%b %d %H:%M:%S")
-			self._appendTemplate(template)
 			# asctime
-			template = DateStrptime()
+			self.appendTemplate("%a %b %d %H:%M:%S %Y")
-			template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year")
-			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
-			template.setPattern("%a %b %d %H:%M:%S %Y")
-			self._appendTemplate(template)
 			# asctime without year
-			template = DateStrptime()
+			self.appendTemplate("%a %b %d %H:%M:%S")
-			template.setName("WEEKDAY MONTH Day Hour:Minute:Second")
+			# standard
-			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+			self.appendTemplate("%b %d %H:%M:%S")
-			template.setPattern("%a %b %d %H:%M:%S")
-			self._appendTemplate(template)
 			# simple date
-			template = DateStrptime()
+			self.appendTemplate("%Y/%m/%d %H:%M:%S")
-			template.setName("Year/Month/Day Hour:Minute:Second")
-			template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%Y/%m/%d %H:%M:%S")
-			self._appendTemplate(template)
 			# simple date too (from x11vnc)
-			template = DateStrptime()
+			self.appendTemplate("%d/%m/%Y %H:%M:%S")
-			template.setName("Day/Month/Year Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%d/%m/%Y %H:%M:%S")
-			self._appendTemplate(template)
 			# previous one but with year given by 2 digits
 			# (See http://bugs.debian.org/537610)
-			template = DateStrptime()
+			self.appendTemplate("%d/%m/%y %H:%M:%S")
-			template.setName("Day/Month/Year2 Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%d/%m/%y %H:%M:%S")
-			self._appendTemplate(template)
 			# Apache format [31/Oct/2006:09:22:55 -0000]
-			template = DateStrptime()
+			self.appendTemplate("%d/%b/%Y:%H:%M:%S")
-			template.setName("Day/MONTH/Year:Hour:Minute:Second")
-			template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
-			template.setPattern("%d/%b/%Y:%H:%M:%S")
-			self._appendTemplate(template)
 			# CPanel 05/20/2008:01:57:39
-			template = DateStrptime()
+			self.appendTemplate("%m/%d/%Y:%H:%M:%S")
-			template.setName("Month/Day/Year:Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
-			template.setPattern("%m/%d/%Y:%H:%M:%S")
-			self._appendTemplate(template)
-			# Exim 2006-12-21 06:43:20
-			template = DateStrptime()
-			template.setName("Year-Month-Day Hour:Minute:Second")
-			template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%Y-%m-%d %H:%M:%S")
-			self._appendTemplate(template)
 			# custom for syslog-ng 2006.12.21 06:43:20
-			template = DateStrptime()
+			self.appendTemplate("%Y.%m.%d %H:%M:%S")
-			template.setName("Year.Month.Day Hour:Minute:Second")
-			template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%Y.%m.%d %H:%M:%S")
-			self._appendTemplate(template)
 			# named 26-Jul-2007 15:20:52.252 
-			template = DateStrptime()
+			self.appendTemplate("%d-%b-%Y %H:%M:%S")
-			template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%d-%b-%Y %H:%M:%S")
-			self._appendTemplate(template)
 			# 17-07-2008 17:23:25
-			template = DateStrptime()
+			self.appendTemplate("%d-%m-%Y %H:%M:%S")
-			template.setName("Day-Month-Year Hour:Minute:Second")
-			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%d-%m-%Y %H:%M:%S")
-			self._appendTemplate(template)
 			# 01-27-2012 16:22:44.252
-			template = DateStrptime()
+			self.appendTemplate("%m-%d-%Y %H:%M:%S")
-			template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%m-%d-%Y %H:%M:%S")
-			self._appendTemplate(template)
 			# TAI64N
 			template = DateTai64n()
 			template.setName("TAI64N")
-			self._appendTemplate(template)
+			self.appendTemplate(template)
 			# Epoch
 			template = DateEpoch()
 			template.setName("Epoch")
-			self._appendTemplate(template)
+			self.appendTemplate(template)
 			# ISO 8601
 			template = DateISO8601()
 			template.setName("ISO 8601")
-			self._appendTemplate(template)
+			self.appendTemplate(template)
 			# Only time information in the log
-			template = DateStrptime()
+			self.appendTemplate("%H:%M:%S", anchor=True)
-			template.setName("Hour:Minute:Second")
-			template.setRegex("^\d{2}:\d{2}:\d{2}")
-			template.setPattern("%H:%M:%S")
-			self._appendTemplate(template)
 			# <09/16/08@05:03:30>
-			template = DateStrptime()
+			self.appendTemplate("<%m/%d/%y@%H:%M:%S>", anchor=True)
-			template.setName("<Month/Day/Year@Hour:Minute:Second>")
-			template.setRegex("^<\d{2}/\d{2}/\d{2}@\d{2}:\d{2}:\d{2}>")
-			template.setPattern("<%m/%d/%y@%H:%M:%S>")
-			self._appendTemplate(template)
 			# MySQL: 130322 11:46:11
-			template = DateStrptime()
+			self.appendTemplate("%y%m%d %H:%M:%S", anchor=True)
-			template.setName("MonthDayYear Hour:Minute:Second")
-			template.setRegex("^\d{2}\d{2}\d{2} +\d{1,2}:\d{2}:\d{2}")
-			template.setPattern("%y%m%d %H:%M:%S")
-			self._appendTemplate(template)
 			# Apache Tomcat
-			template = DateStrptime()
+			self.appendTemplate("%b %d, %Y %I:%M:%S %p")
-			template.setName("MONTH Day, Year 12hour:Minute:Second AM/PM")
-			template.setRegex("\S{3}\s{1,2}\d{1,2}, \d{4} \d{1,2}:\d{2}:\d{2} [AP]M")
-			template.setPattern("%b %d, %Y %I:%M:%S %p")
-			self._appendTemplate(template)
 			# ASSP: Apr-27-13 02:33:06
-			template = DateStrptime()
+			self.appendTemplate("%b-%d-%y %H:%M:%S", anchor=True)
-			template.setName("Month-Day-Year Hour:Minute:Second")
-			template.setRegex("^[a-zA-Z]{3}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
-			template.setPattern("%b-%d-%y %H:%M:%S")
-			self._appendTemplate(template)
 		finally:
 			self.__lock.release()
 	

fail2ban/server/datetemplate.py

@@ -52,7 +52,7 @@ class DateTemplate:
 		if (wordBegin and not re.search(r'^\^', regex)):
 			regex = r'\b' + regex
 		self.__regex = regex
-		self.__cRegex = re.compile(regex)
+		self.__cRegex = re.compile(regex, re.UNICODE)
 		
 	def getRegex(self):
 		return self.__regex
@@ -178,6 +178,51 @@ class DateStrptime(DateTemplate):
 					date[2] = MyTime.gmtime()[2]
 		return date
 
+class DatePatternRegex(DateStrptime):
+	_reEscape = r"([\\.^$*+?\(\){}\[\]|])"
+	_patternRE = r"%(%|[aAbBdHIjmMpSUwWyY])"
+	_patternName = {
+		'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
+		'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
+		'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
+		'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%"}
+	_patternRegex = {
+		'a': r"\w{3}", 'A': r"\w+", 'b': r"\w{3}", 'B': r"\w+",
+		'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)",
+		'I': r"(?:1[0-2]|[ 0]?\d)",
+		'j': r"(?:36[0-6]3[0-5]\d|[1-2]\d\d|[ 0]?\d\d|[ 0]{0,2}\d)",
+		'm': r"(?:1[0-2]|[ 0]?[1-9])", 'M': r"[0-5]\d", 'p': r"[AP]M",
+		'S': r"(?:6[01]|[0-5]\d)", 'U': r"(?:5[0-3]|[1-4]\d|[ 0]?\d)",
+		'w': r"[0-6]", 'W': r"(?:5[0-3]|[ 0]?\d)", 'y': r"\d{2}",
+		'Y': r"\d{4}", '%': "%"}
+
+	def __init__(self, pattern=None, **kwargs):
+		DateStrptime.__init__(self)
+		if pattern:
+			self.setPattern(pattern, **kwargs)
+
+	def setPattern(self, pattern, anchor=False, **kwargs):
+		self.__pattern = pattern.strip()
+
+		name = re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName
+		DateStrptime.setName(self, name)
+
+		# Custom escape as don't want to escape "%"
+		pattern = re.sub(self._reEscape, r'\\\1', pattern)
+		regex = re.sub(
+			self._patternRE, r'%(\1)s', pattern) % self._patternRegex
+		if anchor:
+			regex = r"^" + regex
+		DateStrptime.setRegex(self, regex, **kwargs)
+
+	def getPattern(self):
+		return self.__pattern
+
+	def setRegex(self, line):
+		raise NotImplementedError("Regex derived from pattern")
+
+	def setName(self, line):
+		raise NotImplementedError("Name derived from pattern")
 
 class DateTai64n(DateTemplate):
 	

fail2ban/server/filter.py

@@ -28,6 +28,7 @@ from failmanager import FailManager
 from ticket import FailTicket
 from jailthread import JailThread
 from datedetector import DateDetector
+from datetemplate import DatePatternRegex
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
 
@@ -191,6 +192,40 @@ class Filter(JailThread):
 	def getFindTime(self):
 		return self.__findTime
 
+	##
+	# Set the date detector pattern, removing Defaults
+	#
+	# @param pattern the date template pattern
+
+	def setDatePattern(self, pattern):
+		dateDetector = DateDetector()
+		template = DatePatternRegex()
+		if pattern[0] == "^": # Special extra to enable anchor
+			template.setPattern(pattern[1:], anchor=True)
+		else:
+			template.setPattern(pattern, anchor=False)
+		dateDetector.appendTemplate(template)
+		self.dateDetector = dateDetector
+		logSys.info("Date pattern set to `%r`: `%s`" %
+			(pattern, template.getName()))
+		logSys.debug("Date pattern regex for %r: %s" %
+			(pattern, template.getRegex()))
+
+	##
+	# Get the date detector pattern, or Default Detectors if not changed
+	#
+	# @return pattern of the date template pattern
+
+	def getDatePattern(self):
+		templates = self.dateDetector.getTemplates()
+		if len(templates) > 1:
+			return None # Default Detectors in use
+		elif len(templates) == 1:
+			pattern =  templates[0].getPattern()
+			if templates[0].getRegex()[0] == "^":
+				pattern = "^" + pattern
+			return pattern, templates[0].getName()
+
 	##
 	# Set the maximum retry value.
 	#
@@ -394,7 +429,9 @@ class Filter(JailThread):
 				logSys.log(7, "Matched %s", failRegex)
 				if date is None:
 					logSys.debug("Found a match for %r but no valid date/time "
-								 "found for %r. Please file a detailed issue on"
+								 "found for %r. Please try setting a custom "
+								 "date pattern. If format is complex, please "
+								 "file a detailed issue on"
 								 " https://github.com/fail2ban/fail2ban/issues "
 								 "in order to get support for this format."
 								 % (logLine, timeLine))

fail2ban/server/server.py

@@ -222,6 +222,12 @@ class Server:
 	def getFindTime(self, name):
 		return self.__jails.getFilter(name).getFindTime()
 
+	def setDatePattern(self, name, pattern):
+		self.__jails.getFilter(name).setDatePattern(pattern)
+
+	def getDatePattern(self, name):
+		return self.__jails.getFilter(name).getDatePattern()
+
 	def addFailRegex(self, name, value):
 		self.__jails.getFilter(name).addFailRegex(value)
 	

fail2ban/server/transmitter.py

@@ -176,6 +176,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setFindTime(name, int(value))
 			return self.__server.getFindTime(name)
+		elif command[1] == "datepattern":
+			value = command[2]
+			self.__server.setDatePattern(name, value)
+			return self.__server.getDatePattern(name)
 		elif command[1] == "maxretry":
 			value = command[2]
 			self.__server.setMaxRetry(name, int(value))
@@ -270,6 +274,8 @@ class Transmitter:
 			return self.__server.getUseDns(name)
 		elif command[1] == "findtime":
 			return self.__server.getFindTime(name)
+		elif command[1] == "datepattern":
+			return self.__server.getDatePattern(name)
 		elif command[1] == "maxretry":
 			return self.__server.getMaxRetry(name)
 		elif command[1] == "maxlines":

fail2ban/tests/datedetectortestcase.py

@@ -104,7 +104,7 @@ class DateDetectorTest(unittest.TestCase):
 			self.assertEqual(old_name, n.getName()) # "Sort must be stable"
 
 	def testAllUniqueTemplateNames(self):
-		self.assertRaises(ValueError, self.__datedetector._appendTemplate,
+		self.assertRaises(ValueError, self.__datedetector.appendTemplate,
 						  self.__datedetector.getTemplates()[0])
 
 	def testFullYearMatch_gh130(self):

fail2ban/tests/servertestcase.py

@@ -235,6 +235,12 @@ class Transmitter(TransmitterBase):
 		self.setGetTest("bantime", "-50", -50, jail=self.jailName)
 		self.setGetTestNOK("bantime", "Cat", jail=self.jailName)
 
+	def testDatePattern(self):
+		self.setGetTest("datepattern", "%%%Y%m%d%H%M%S",
+			("%%%Y%m%d%H%M%S", "%YearMonthDay24hourMinuteSecond"),
+			jail=self.jailName)
+		self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
+
 	def testJailUseDNS(self):
 		self.setGetTest("usedns", "yes", jail=self.jailName)
 		self.setGetTest("usedns", "warn", jail=self.jailName)