From e735f8f56835abeda6135c2088b54753193de7c9 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 6 Oct 2016 15:27:49 +0200
Subject: [PATCH] default non-unicode and case-sensitive matching (by pattern
 templates automatically add `(?iu)` for "ignore case" and "unicode" if
 expected)

---
 fail2ban/server/datedetector.py        | 47 ++++++++++++++------------
 fail2ban/server/datetemplate.py        | 25 ++++++++++----
 fail2ban/tests/datedetectortestcase.py | 45 +++++++++++++++++++++---
 3 files changed, 85 insertions(+), 32 deletions(-)

diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 0aac4337..f256a425 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -254,27 +254,32 @@ class DateDetector(object):
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
-			distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
-			if logSys.getEffectiveLevel() <= logLevel-1:
-				logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
-					i, distance, endpos, 
-					line[distance-1:distance], self.__lastPos[1],
-					line[distance:endpos],
-					line[endpos:endpos+1], self.__lastEndPos[1])
-			# check same boundaries left/right, otherwise possible collision/pattern switch:
-			if (line[distance-1:distance] == self.__lastPos[1] and 
-					line[endpos:endpos+1] == self.__lastEndPos[1]
-			):
-				match = template.matchDate(line, distance, endpos)
-				if match:
-					distance = match.start()
-					endpos = match.end()
-					# if different position, possible collision/pattern switch:
-					if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
-						logSys.log(logLevel, "  matched last time template #%02i", i)
-					else:
-						logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
-						match = None
+			if template.flags & DateTemplate.LINE_BEGIN:
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
+					match = template.matchDate(line)
+			else:
+				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
+						i, distance, endpos, 
+						line[distance-1:distance], self.__lastPos[1],
+						line[distance:endpos],
+						line[endpos:endpos+1], self.__lastEndPos[1])
+				# check same boundaries left/right, otherwise possible collision/pattern switch:
+				if (line[distance-1:distance] == self.__lastPos[1] and 
+						line[endpos:endpos+1] == self.__lastEndPos[1]
+				):
+					match = template.matchDate(line, distance, endpos)
+			if match:
+				distance = match.start()
+				endpos = match.end()
+				# if different position, possible collision/pattern switch:
+				if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
+					logSys.log(logLevel, "  matched last time template #%02i", i)
+				else:
+					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
+					match = None
 		# search template and better match:
 		if not match:
 			self.__lastTemplIdx = 0x7fffffff
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index fcd51f99..9524e373 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -32,13 +32,18 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
-RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)')
 RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
-RE_DEL_WRD_BOUNDS =   re.compile(r'^\*\*|(?<!\\)\*\*$')
+RE_DEL_WRD_BOUNDS = ( re.compile(r'^(?:\(\?\w+\))?\*\*|(?<!\\)\*\*()$'), 
+	                    lambda m: m.group().replace('**', '') )
 
-RE_LINE_BOUND_BEG = re.compile(r'^(?:\^|\(\?:\^(?!\|))')
+RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
 RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
+RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
+
+_Templ_RECache = {}
+
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -104,7 +109,7 @@ class DateTemplate(object):
 		if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
 		if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
-		regex = RE_DEL_WRD_BOUNDS.sub('', regex)
+		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
 		self._cRegex = None
 
@@ -116,7 +121,11 @@ class DateTemplate(object):
 		"""Compile regex by first usage.
 		"""
 		if not self._cRegex:
-			self._cRegex = re.compile(self.regex, re.UNICODE | re.IGNORECASE)
+			try:
+				self._cRegex = re.compile(self.regex)
+			except Exception as e:
+				logSys.error('Compile %r failed, expression %r', self.name, self.regex)
+				raise e
 
 	def matchDate(self, line, *args):
 		"""Check if regex for date matches on a log line.
@@ -235,7 +244,11 @@ class DatePatternRegex(DateTemplate):
 		self._pattern = pattern
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
 		self.name = fmt % self._patternName
-		super(DatePatternRegex, self).setRegex(fmt % timeRE, wordBegin, wordEnd)
+		regex = fmt % timeRE
+		# if expected add (?iu) for "ignore case" and "unicode":
+		if RE_ALPHA_PATTERN.search(pattern):
+			regex = r'(?iu)' + regex
+		super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index a4c9b645..6bf92582 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -195,11 +195,46 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 
 	def testDateTemplate(self):
-			t = DateTemplate()
-			t.setRegex('^a{3,5}b?c*$')
-			self.assertEqual(t.getRegex(), '^a{3,5}b?c*$')
-			self.assertRaises(Exception, t.getDate, '')
-			self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+		t = DateTemplate()
+		t.setRegex('^a{3,5}b?c*$')
+		self.assertEqual(t.regex, '^a{3,5}b?c*$')
+		self.assertRaises(Exception, t.getDate, '')
+		self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+
+		## no word boundaries left and right:
+		t = DatePatternRegex()
+		t.pattern = '(?iu)**time:%ExY%Exm%ExdT%ExH%ExM%ExS**'
+		# ** was removed from end-regex:
+		self.assertFalse('**' in t.regex)
+		# match date:
+		dt = 'TIME:20050102T010203'
+		self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt)
+		self.assertEqual(t.matchDate(dt).group(), dt)
+		# wrong year (for exact %ExY):
+		dt = 'TIME:50050102T010203'
+		self.assertFalse(t.matchDate(dt))
+
+		## start boundary left and word boundary right:
+		t = DatePatternRegex()
+		t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS'
+		self.assertTrue('^' in t.regex)
+		# try match date:
+		dt = 'time:20050102T010203'
+		self.assertFalse(t.matchDate('X' + dt))
+		self.assertFalse(t.matchDate(dt + 'X'))
+		self.assertEqual(t.matchDate('##' + dt + '...').group(), dt)
+		self.assertEqual(t.matchDate(dt).group(), dt)
+		# case sensitive:
+		dt = 'TIME:20050102T010203'
+		self.assertFalse(t.matchDate(dt))
+
+		## auto-switching "ignore case" and "unicode"
+		t = DatePatternRegex()
+		t.pattern = '^%Y %b %d'
+		self.assertTrue('(?iu)' in t.regex)
+		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt)
 
 	def testAmbiguousInOrderedTemplates(self):
 		dd = self.datedetector