more precise date template handling (WARNING: this commit creates possible incompatibilities):

- datedetector rewritten more strict as earlier;
  - default templates can be specified exacter using prefix/suffix syntax (via `datepattern`);
  - more as one date pattern can be specified using option `datepattern` now (new-line separated);
  - some default options like `datepattern` can be specified directly in section `[Definition]`, that avoids contrary usage of unnecessarily `[Init]` section, because of performance (each extra section costs time);
  - option `datepattern` can be specified in jail also (jails without filters);
  - if first group specified, only this will be cut out from search log-line (e. g.: `^date:[({DATE})]` will cut out only datetime match pattern, and leaves `date:[] failure ip...` for searching in filter);
  - faster match and fewer searching of appropriate templates (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
  - standard filters extended with exact prefixed or anchored date templates;

template cache introduced (in opposition to default template cache, holds custom templates cached by pattern for possible common usage of same template/regex);

config/filter.d/apache-badbots.conf

@@ -14,6 +14,9 @@ failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 # DEV Notes:
 # List of bad bots fetched from http://www.user-agents.org
 # Generated on Thu Nov  7 14:23:35 PST 2013 by files/gen_badbots.

config/filter.d/apache-fakegooglebot.conf

@@ -6,6 +6,8 @@ failregex = ^<HOST> .*Googlebot.*$
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 #

config/filter.d/apache-pass.conf

@@ -9,6 +9,9 @@ failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[
 
 ignoreregex =
 
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}
+
 [Init]
 
 knocking_url = /knocking/

config/filter.d/assp.conf

@@ -20,7 +20,8 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
 
 ignoreregex = 
 
-datepattern = {^LN-BEG}
+datepattern = {^LN-BEG}%%b-%%d-%%Exy %%H:%%M:%%S
+              {^LN-BEG}
 
 # DEV Notes:
 # V1 Examples matches:

config/filter.d/dovecot.conf

@@ -17,10 +17,11 @@ failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentica
 
 ignoreregex = 
 
-[Init]
-
 journalmatch = _SYSTEMD_UNIT=dovecot.service
 
+datepattern = {^LN-BEG}TAI64N
+              {^LN-BEG}
+
 # DEV Notes:
 # * the first regex is essentially a copy of pam-generic.conf
 # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016)

config/filter.d/ejabberd-auth.conf

@@ -25,8 +25,6 @@ failregex = ^=INFO REPORT====  ===\nI\(<0\.\d+\.0>:ejabberd_c2s:\d+\) : \([^)]+\
 #
 ignoreregex = 
 
-[Init]
-
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
 
@@ -35,3 +33,5 @@ maxlines = 2
 # Values:  TEXT
 #
 journalmatch = 
+
+datepattern = ^(?:=[^=]+={3,} )?({DATE})

config/filter.d/guacamole.conf

@@ -17,6 +17,9 @@ failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" fa
 #
 ignoreregex = 
 
-[Init]
 # "maxlines" is number of log lines to buffer for multi-line regex searches
 maxlines = 2
+
+datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p
+              ^WARNING:()**
+              {^LN-BEG}

config/filter.d/nginx-botsearch.conf

@@ -13,6 +13,9 @@ failregex = ^<HOST> \- \S+ \[\] \"(GET|POST|HEAD) \/<block> \S+\" 404 .+$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # DEV Notes:
 # Based on apache-botsearch filter

config/filter.d/nsd.conf

@@ -26,3 +26,6 @@ failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
              ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =
+
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}

config/filter.d/php-url-fopen.conf

@@ -18,3 +18,6 @@ ignoreregex =
 # http://blogs.buanzo.com.ar/2009/04/fail2ban-filter-for-php-injection-attacks.html#comment-1489
 #
 # Author: Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
+
+datepattern = ^[^\[]*\[({DATE})
+              {^LN-BEG}

config/filter.d/portsentry.conf

@@ -8,5 +8,8 @@ failregex = \/<HOST> Port\: [0-9]+ (TCP|UDP) Blocked$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Pacop <pacoparu@gmail.com>
 

config/filter.d/sogo-auth.conf

@@ -8,7 +8,10 @@ failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' migh
 
 ignoreregex = "^<ADDR>"
 
-datepattern = {^LN-BEG}
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)?
+              ^[^\[]*\[({DATE})
+              {^LN-BEG}
 
 # 
 # DEV Notes:

config/filter.d/squid.conf

@@ -9,5 +9,8 @@ failregex = ^\s+\d\s<HOST>\s+[A-Z_]+_DENIED/403 .*$
 
 ignoreregex =
 
+datepattern = {^LN-BEG}Epoch
+              {^LN-BEG}
+
 # Author: Daniel Black
 

config/filter.d/tine20.conf

@@ -10,6 +10,9 @@ failregex =  ^[\da-f]{5,} [\da-f]{5,} (-- none --|.*?)( \d+(\.\d+)?(h|m|s|ms)){0
 
 ignoreregex = 
 
+datepattern = ^[^-]+ -- [^-]+ -- - ({DATE})
+              {^LN-BEG}
+
 # Author: Mika (mkl) from Tine20.org forum: https://www.tine20.org/forum/viewtopic.php?f=2&t=15688&p=54766
 # Editor: Daniel Black
 # Advisor: Lars Kneschke

fail2ban/client/jailreader.py

@@ -112,6 +112,7 @@ class JailReader(ConfigReader):
 				["string", "ignorecommand", None],
 				["string", "ignoreip", None],
 				["string", "filter", ""],
+				["string", "datepattern", None],
 				["string", "action", ""]]
 
 		# Before interpolation (substitution) add static options always available as default:
@@ -195,6 +196,8 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
+		if self.__filter:
+			stream.extend(self.__filter.convert())
 		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					not self.__opts.get('backend', None).startswith("systemd"):
@@ -216,17 +219,9 @@ class JailReader(ConfigReader):
 				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
 				backend = value
-			elif opt == "maxretry":
-				stream.append(["set", self.__name, "maxretry", value])
 			elif opt == "ignoreip":
 				for ip in splitwords(value):
 					stream.append(["set", self.__name, "addignoreip", ip])
-			elif opt == "findtime":
-				stream.append(["set", self.__name, "findtime", value])
-			elif opt == "bantime":
-				stream.append(["set", self.__name, "bantime", value])
-			elif opt == "usedns":
-				stream.append(["set", self.__name, "usedns", value])
 			elif opt in ("failregex", "ignoreregex"):
 				multi = []
 				for regex in value.split('\n'):
@@ -237,10 +232,8 @@ class JailReader(ConfigReader):
 					stream.append(["multi-set", self.__name, "add" + opt, multi])
 				elif len(multi):
 					stream.append(["set", self.__name, "add" + opt, multi[0]])
-			elif opt == "ignorecommand":
+			elif opt not in ('action', 'filter', 'enabled'):
-				stream.append(["set", self.__name, "ignorecommand", value])
+				stream.append(["set", self.__name, opt, value])
-		if self.__filter:
-			stream.extend(self.__filter.convert())
 		for action in self.__actions:
 			if isinstance(action, (ConfigReaderUnshared, ConfigReader)):
 				stream.extend(action.convert())

fail2ban/server/datedetector.py

@@ -27,6 +27,7 @@ import time
 from threading import Lock
 
 from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .utils import Utils
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -90,7 +91,7 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
@@ -112,8 +113,6 @@ class DateDetectorCache(object):
 		# subseconds explicit to avoid possible %m<->%d confusion
 		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
 		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
-		# TAI64N
-		self._cacheTemplate(DateTai64n())
 		# Epoch
 		self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
 		self._cacheTemplate(DateEpoch())
@@ -132,6 +131,10 @@ class DateDetectorCache(object):
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
 		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		# +00:00 Jan 23 21:59:59.011 2005 
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		# TAI64N
+		self._cacheTemplate(DateTai64n())
 		#
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
@@ -168,6 +171,7 @@ class DateDetector(object):
 	templates
 	"""
 	_defCache = DateDetectorCache()
+	_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
 
 	def __init__(self):
 		self.__templates = list()
@@ -183,9 +187,10 @@ class DateDetector(object):
 		# pre-match pattern:
 		self.__preMatch = None
 
-	def _appendTemplate(self, template):
+	def _appendTemplate(self, template, ignoreDup=False):
 		name = template.name
 		if name in self.__known_names:
+			if ignoreDup: return
 			raise ValueError(
 				"There is already a template with name %s" % name)
 		self.__known_names.add(name)
@@ -207,24 +212,56 @@ class DateDetector(object):
 			If a template already exists with the same name.
 		"""
 		if isinstance(template, str):
-			template = DatePatternRegex(template)
+			key = pattern = template
+			if '%' not in pattern:
+				key = pattern.upper()
+			template = DateDetector._patternCache.get(key)
+
+			if not template:
+				if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
+					template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
+				elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
+					template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
+				elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
+					flt = \
+						lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
+						lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
+						None
+					self.addDefaultTemplate(flt)
+					return
+				elif "{DATE}" in key:
+					self.addDefaultTemplate(
+						lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
+					return
+				else:
+					template = DatePatternRegex(pattern)
+
+			DateDetector._patternCache.set(key, template)
+
 		self._appendTemplate(template)
+		logSys.info("  date pattern `%r`: `%s`",
+			getattr(template, 'pattern', ''), template.name)
+		logSys.debug("  date pattern regex for %r: %s",
+			getattr(template, 'pattern', ''), template.regex)
 
 	def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
 		"""Add Fail2Ban's default set of date templates.
 		"""
-		for template in sorted(DateDetector._defCache.templates,
+		ignoreDup = len(self.__templates) > 0
-			lambda a,b: b.hits - a.hits
+		for template in DateDetector._defCache.templates:
-		):
 			# filter if specified:
 			if filterTemplate is not None and not filterTemplate(template): continue
 			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
 			if preMatch is not None:
-				regex = getattr(template, 'pattern', template.regex)
+				deftemplate = template
-				template = copy.copy(template)
+				template = DateDetector._patternCache.get((preMatch, deftemplate.name))
+				if not template:
+					regex = getattr(deftemplate, 'pattern', deftemplate.regex)
+					template = copy.copy(deftemplate)
 					template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
-			# append date detector template:
+				DateDetector._patternCache.set((preMatch, deftemplate.name), template)
-			self._appendTemplate(template)
+			# append date detector template (ignore duplicate if some was added before default):
+			self._appendTemplate(template, ignoreDup=ignoreDup)
 
 	@property
 	def templates(self):
@@ -250,17 +287,22 @@ class DateDetector(object):
 			The regex match returned from the first successfully matched
 			template.
 		"""
-		#logSys.log(logLevel, "try to match time for line: %.120s", line)
+		# if no templates specified - default templates should be used:
+		if not len(self.__templates):
+			self.addDefaultTemplate()
+		logSys.log(logLevel-1, "try to match time for line: %.120s", line)
 		match = None
 		# first try to use last template with same start/end position:
+		ignoreBySearch = 0x7fffffff
 		i = self.__lastTemplIdx
 		if i < len(self.__templates):
 			ddtempl = self.__templates[i]
 			template = ddtempl.template
-			if template.flags & DateTemplate.LINE_BEGIN:
+			if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 				if logSys.getEffectiveLevel() <= logLevel-1:
 					logSys.log(logLevel-1, "  try to match last anchored template #%02i ...", i)
 					match = template.matchDate(line)
+					ignoreBySearch = i
 			else:
 				distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
 				if logSys.getEffectiveLevel() <= logLevel-1:
@@ -278,18 +320,28 @@ class DateDetector(object):
 				distance = match.start()
 				endpos = match.end()
 				# if different position, possible collision/pattern switch:
-				if distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]:
+				if (
+					template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or 
+					(distance == self.__lastPos[0] and endpos == self.__lastEndPos[0])
+				):
 					logSys.log(logLevel, "  matched last time template #%02i", i)
 				else:
 					logSys.log(logLevel, "  ** last pattern collision - pattern change, search ...")
 					match = None
+			else:
+				logSys.log(logLevel, "  ** last pattern not found - pattern change, search ...")
 		# search template and better match:
 		if not match:
 			self.__lastTemplIdx = 0x7fffffff
-			logSys.log(logLevel, " search template ...")
+			logSys.log(logLevel, " search template (%i) ...", len(self.__templates))
 			found = None, 0x7fffffff, -1
 			i = 0
 			for ddtempl in self.__templates:
+				if logSys.getEffectiveLevel() <= logLevel-1:
+					logSys.log(logLevel-1, "  try template #%02i: %s", i, ddtempl.name)
+				if i == ignoreBySearch:
+					i += 1
+					continue
 				template = ddtempl.template
 				match = template.matchDate(line)
 				if match:
@@ -298,15 +350,18 @@ class DateDetector(object):
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "  matched time template #%02i (at %r <= %r, %r) %s",
 							i, distance, ddtempl.distance, self.__lastPos[0], template.name)
+					## last (or single) template - fast stop:
+					if i+1 >= len(self.__templates):
+						break
 					## if line-begin/end anchored - stop searching:
 					if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
 						break
+					## stop searching if next template still unused, but we had already hits:
+					if (distance == 0 and ddtempl.hits) and not self.__templates[i+1].template.hits:
+						break
 					## [grave] if distance changed, possible date-match was found somewhere 
 					## in body of message, so save this template, and search further:
-					if (
+					if distance > ddtempl.distance or distance > self.__lastPos[0]:
-						(distance > ddtempl.distance or distance > self.__lastPos[0]) and
-						len(self.__templates) > 1
-					):
 						logSys.log(logLevel, "  ** distance collision - pattern change, reserve")
 						## shortest of both:
 						if distance < found[1]:
@@ -374,7 +429,7 @@ class DateDetector(object):
 				if date is not None:
 					if logSys.getEffectiveLevel() <= logLevel:
 						logSys.log(logLevel, "  got time %f for %r using template %s",
-							date[0], date[1].group(), template.name)
+							date[0], date[1].group(1), template.name)
 					return date
 			except ValueError:
 				pass

fail2ban/server/datetemplate.py

@@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import re
+import re, time
 from abc import abstractmethod
 
 from .strptime import reGroupDictStrptime, timeRE, getTimePatternRE
@@ -32,9 +32,14 @@ from ..helpers import getLogger
 
 logSys = getLogger(__name__)
 
-RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\*\*|\(\?:\^)')
+# check already grouped contains "(", but ignores char "\(" and conditional "(?(id)...)":
-RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
+RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
-RE_DEL_WRD_BOUNDS = ( re.compile(r'^(?:\(\?\w+\))?\*\*|(?<!\\)\*\*()$'), 
+RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
+
+RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
+RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)')
+RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*\)*)$')
+RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), 
 	                    lambda m: m.group().replace('**', '') )
 
 RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))')
@@ -42,8 +47,6 @@ RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
 RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
 
-_Templ_RECache = {}
-
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -66,6 +69,7 @@ class DateTemplate(object):
 		self.weight = 1.0
 		self.flags = 0
 		self.hits = 0
+		self.time = 0
 		self._regex = ""
 		self._cRegex = None
 
@@ -95,15 +99,28 @@ class DateTemplate(object):
 		re.error
 			If regular expression fails to compile
 		"""
+		# Warning: don't use lookahead for line-begin boundary, 
+		# (e. g. r"^(?:\W{0,2})?" is much faster as r"(?:^|(?<=^\W)|(?<=^\W{2}))")
+		# because it may be very slow in negative case (by long log-lines not matching pattern)
+
 		regex = regex.strip()
+		boundBegin = wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex)
+		boundEnd = wordEnd and not RE_NO_WRD_BOUND_END.search(regex)
+		# if no group add it now, should always have a group(1):
+		if not RE_GROUPED.search(regex):
+			regex = RE_GROUP[0].sub(RE_GROUP[1], regex)
 		self.flags = 0
 		# if word or line start boundary:
-		if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
+		if boundBegin:
 			self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
-			regex = (r'(?=^|\b|\W)' if wordBegin != 'start' else r"(?:^|(?<=^\W)|(?<=^\W{2}))") + regex
+			if wordBegin != 'start':
-			self.name = ('{*WD-BEG}' if wordBegin != 'start' else '{^LN-BEG}') + self.name
+				regex = r'(?:^|\b|\W)' + regex
+				self.name = '{*WD-BEG}' + self.name
+			else:
+				regex = r"^(?:\W{0,2})?" + regex
+				self.name = '{^LN-BEG}' + self.name
 		# if word end boundary:
-		if wordEnd and not RE_NO_WRD_BOUND_END.search(regex):
+		if boundEnd:
 			self.flags |= DateTemplate.WORD_END
 			regex += r'(?=\b|\W|$)'
 			self.name += '{*WD-END}'
@@ -112,6 +129,7 @@ class DateTemplate(object):
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
+		logSys.debug('  constructed regex %s', regex)
 		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
@@ -123,6 +141,7 @@ class DateTemplate(object):
 		"""
 		if not self._cRegex:
 			try:
+				# print('*'*10 + (' compile - %-30.30s -- %s' % (getattr(self, 'pattern', self.regex), self.name)))
 				self._cRegex = re.compile(self.regex)
 			except Exception as e:
 				logSys.error('Compile %r failed, expression %r', self.name, self.regex)
@@ -136,6 +155,7 @@ class DateTemplate(object):
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		if dateMatch:
 			self.hits += 1
+		# print('*'*10 + ('[%s] - %-30.30s -- %s' % ('*' if dateMatch else ' ', getattr(self, 'pattern', self.regex), self.name)))
 		return dateMatch
 
 	@abstractmethod
@@ -175,10 +195,10 @@ class DateEpoch(DateTemplate):
 		DateTemplate.__init__(self)
 		self.name = "Epoch"
 		if not lineBeginOnly:
-			regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
+			regex = r"((?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
 			self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
 		else:
-			regex = r"(?P<square>(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))"
+			regex = r"((?P<square>(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))"
 			self.setRegex(regex, wordBegin='start', wordEnd=True)
 
 	def getDate(self, line, dateMatch=None):
@@ -199,7 +219,7 @@ class DateEpoch(DateTemplate):
 			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
-			return (float(dateMatch.group()), dateMatch)
+			return (float(dateMatch.group(1)), dateMatch)
 		return None
 
 
@@ -244,7 +264,13 @@ class DatePatternRegex(DateTemplate):
 		self.setRegex(pattern)
 
 	def setRegex(self, pattern, wordBegin=True, wordEnd=True):
+		# original pattern:
 		self._pattern = pattern
+		# if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin:
+		if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern):
+			pattern = RE_EXLINE_BOUND_BEG.sub('', pattern)
+			wordBegin = 'start'
+		# wrap to regex:
 		fmt = self._patternRE.sub(r'%(\1)s', pattern)
 		self.name = fmt % self._patternName
 		regex = fmt % timeRE
@@ -285,12 +311,11 @@ class DateTai64n(DateTemplate):
 	regex
 	"""
 
-	def __init__(self):
+	def __init__(self, wordBegin=False):
 		DateTemplate.__init__(self)
 		self.name = "TAI64N"
 		# We already know the format for TAI64N
-		# yoh: we should not add an additional front anchor
+		self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin)
-		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
 
 	def getDate(self, line, dateMatch=None):
 		"""Method to return the date for a log line.
@@ -310,7 +335,7 @@ class DateTai64n(DateTemplate):
 			dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
-			value = dateMatch.group()
+			value = dateMatch.group(1)
 			seconds_since_epoch = value[2:17]
 			# convert seconds from HEX into local time stamp
 			return (int(seconds_since_epoch, 16), dateMatch)

fail2ban/server/filter.py

@@ -35,7 +35,6 @@ from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
-from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
@@ -94,7 +93,6 @@ class Filter(JailThread):
 		self.ticks = 0
 
 		self.dateDetector = DateDetector()
-		self.dateDetector.addDefaultTemplate()
 		logSys.debug("Created %s" % self)
 
 	def __repr__(self):
@@ -258,34 +256,12 @@ class Filter(JailThread):
 			self.dateDetector = None
 			return
 		else:
-			key = pattern.upper()
+			dd = DateDetector()
-			if key == "EPOCH":
+			if not isinstance(pattern, (list, tuple)):
-				template = DateEpoch()
+				pattern = filter(bool, map(str.strip, re.split('\n+', pattern)))
-				template.name = "Epoch"
+			for pattern in pattern:
-			elif key == "TAI64N":
+				dd.appendTemplate(pattern)
-				template = DateTai64n()
+			self.dateDetector = dd
-				template.name = "TAI64N"
-			elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
-				self.dateDetector = DateDetector()
-				flt = \
-					lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
-					lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
-					None
-				self.dateDetector.addDefaultTemplate(flt)
-				return
-			elif "{DATE}" in key:
-				self.dateDetector = DateDetector()
-				self.dateDetector.addDefaultTemplate(
-					lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
-				return
-			else:
-				template = DatePatternRegex(pattern)
-		self.dateDetector = DateDetector()
-		self.dateDetector.appendTemplate(template)
-		logSys.info("  date pattern `%r`: `%s`",
-			pattern, template.name)
-		logSys.debug("  date pattern regex for %r: %s",
-			pattern, template.regex)
 
 	##
 	# Get the date detector pattern, or Default Detectors if not changed
@@ -295,7 +271,8 @@ class Filter(JailThread):
 	def getDatePattern(self):
 		if self.dateDetector is not None:
 			templates = self.dateDetector.templates
-			if len(templates) > 2:
+			# lazy template init, by first match
+			if not len(templates) or len(templates) > 2:
 				return None, "Default Detectors"
 			elif len(templates):
 				if hasattr(templates[0], "pattern"):
@@ -303,6 +280,7 @@ class Filter(JailThread):
 				else:
 					pattern = None
 				return pattern, templates[0].name
+		return None
 
 	##
 	# Set the maximum retry value.
@@ -483,9 +461,9 @@ class Filter(JailThread):
 			(timeMatch, template) = self.dateDetector.matchTime(l)
 			if timeMatch:
 				tupleLine  = (
-					l[:timeMatch.start()],
+					l[:timeMatch.start(1)],
-					l[timeMatch.start():timeMatch.end()],
+					l[timeMatch.start(1):timeMatch.end(1)],
-					l[timeMatch.end():],
+					l[timeMatch.end(1):],
 					(timeMatch, template)
 				)
 			else:

fail2ban/server/strptime.py

@@ -55,8 +55,6 @@ timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
 # respect possible run in the test-cases (alternate date used there):
 timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3)
 timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3)
-# Special pattern "start of the line", analogous to `wordBegin='start'` of default templates:
-timeRE['ExLB'] = r"(?:^|(?<=^\W)|(?<=^\W{2}))"
 
 def getTimePatternRE():
 	keys = timeRE.keys()
@@ -70,7 +68,6 @@ def getTimePatternRE():
 		'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
 		'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%",
 		'z': "Zone offset", 'f': "Microseconds", 'Z': "Zone name",
-		'ExLB': '{^LN-BEG}',
 	}
 	for key in set(keys) - set(names): # may not have them all...
 		if key.startswith('Ex'):

fail2ban/server/transmitter.py

@@ -303,7 +303,7 @@ class Transmitter:
 					actionvalue = command[4]
 					setattr(action, actionkey, actionvalue)
 					return getattr(action, actionkey)
-		raise Exception("Invalid command (no set action or not yet implemented)")
+		raise Exception("Invalid command %r (no set action or not yet implemented)" % (command[1],))
 	
 	def __commandGet(self, command):
 		name = command[0]

fail2ban/tests/action_d/test_badips.py

@@ -32,6 +32,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 		
 		def setUp(self):
 			"""Call before every test case."""
+			super(BadIPsActionTest, self).setUp()
 			unittest.F2B.SkipIfNoNetwork()
 
 			self.jail = DummyJail()

fail2ban/tests/action_d/test_smtp.py

@@ -45,6 +45,7 @@ class SMTPActionTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(SMTPActionTest, self).setUp()
 		self.jail = DummyJail()
 		pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")
 		pythonModuleName = os.path.basename(pythonModule.rstrip(".py"))

fail2ban/tests/banmanagertestcase.py

@@ -32,6 +32,7 @@ from ..server.ticket import BanTicket
 class AddFailure(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
+		super(AddFailure, self).setUp()
 		self.__ticket = BanTicket('193.168.0.128', 1167605999.0)
 		self.__banManager = BanManager()
 
@@ -134,6 +135,7 @@ class AddFailure(unittest.TestCase):
 class StatusExtendedCymruInfo(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
+		super(StatusExtendedCymruInfo, self).setUp()
 		unittest.F2B.SkipIfNoNetwork()
 		self.__ban_ip = "93.184.216.34"
 		self.__asn = "15133"

fail2ban/tests/clientbeautifiertestcase.py

@@ -32,6 +32,7 @@ class BeautifierTest(unittest.TestCase):
 
 	def setUp(self):
 		""" Call before every test case """
+		super(BeautifierTest, self).setUp()
 		self.b = Beautifier()
 
 	def tearDown(self):

fail2ban/tests/clientreadertestcase.py

@@ -55,6 +55,7 @@ class ConfigReaderTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(ConfigReaderTest, self).setUp()
 		self.d = tempfile.mkdtemp(prefix="f2b-temp")
 		self.c = ConfigReaderUnshared(basedir=self.d)
 

fail2ban/tests/config/filter.d/zzz-generic-example.conf

@@ -20,3 +20,8 @@ failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
 # just to test multiple ignoreregex:
 ignoreregex = ^%(__prefix_line)sF2B: error from 192.0.2.251$
               ^%(__prefix_line)sF2B: error from 192.0.2.252$
+
+# specify only exact date patterns, +1 with %%Y to test usage of last known date by wrong dates like 0000-00-00...
+datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?
+              {^LN-BEG}(?:%%a )?%%b %%d %%H:%%M:%%S(?:\.%%f)?(?: %%ExY)?
+              {^LN-BEG}%%Y(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)?

fail2ban/tests/datedetectortestcase.py

@@ -58,6 +58,8 @@ class DateDetectorTest(LogCaptureTestCase):
 		return self.__datedetector
 	
 	def testGetEpochTime(self):
+		self.__datedetector = DateDetector()
+		self.__datedetector.appendTemplate('EPOCH')
 		# correct epoch time, using all variants:
 		for dateUnix in (1138049999, 32535244799):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
@@ -67,7 +69,7 @@ class DateDetectorTest(LogCaptureTestCase):
 				self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,))
 				( datelog, matchlog ) = datelog
 				self.assertEqual(int(datelog), dateUnix)
-				self.assertIn(matchlog.group(), (str(dateUnix), str(dateUnix)+'.555'))
+				self.assertIn(matchlog.group(1), (str(dateUnix), str(dateUnix)+'.555'))
 		# wrong, no epoch time (< 10 digits, more as 11 digits, begin/end of word) :
 		for dateUnix in ('123456789', '9999999999999999', '1138049999A', 'A1138049999'):
 			for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"):
@@ -85,7 +87,7 @@ class DateDetectorTest(LogCaptureTestCase):
 		#      of fail2ban -- we just ignore incorrect day of the week
 		( datelog, matchlog ) = self.datedetector.getTime(log)
 		self.assertEqual(datelog, dateUnix)
-		self.assertEqual(matchlog.group(), 'Jan 23 21:59:59')
+		self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
 
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
@@ -150,7 +152,7 @@ class DateDetectorTest(LogCaptureTestCase):
 					( logUnix, logMatch ) = logtime
 					self.assertEqual(logUnix, dateUnix,
 						"getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
-					self.assertEqual(logMatch.group(), rdate)
+					self.assertEqual(logMatch.group(1), rdate)
 				else:
 					self.assertEqual(logtime, None,
 						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
@@ -164,7 +166,7 @@ class DateDetectorTest(LogCaptureTestCase):
 					( logUnix, logMatch ) = logtime
 					self.assertEqual(logUnix, dateUnix,
 						"getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix))
-					self.assertEqual(logMatch.group(), rdate)
+					self.assertEqual(logMatch.group(1), rdate)
 				else:
 					self.assertEqual(logtime, None,
 						"getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime))
@@ -183,23 +185,23 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertNotEqual(logdate, None)
 		( logTime, logMatch ) = logdate
 		self.assertEqual(logTime, mu)
-		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
+		self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17')
 		# confuse it with year being at the end
 		for i in xrange(10):
 			( logTime, logMatch ) =	self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
 			self.assertEqual(logTime, mu)
-			self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
+			self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17')
 		# and now back to the original
 		( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
 		self.assertEqual(logTime, mu)
-		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
+		self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17')
 
 	def testDateTemplate(self):
 		t = DateTemplate()
 		t.setRegex('^a{3,5}b?c*$')
-		self.assertEqual(t.regex, '^a{3,5}b?c*$')
+		self.assertEqual(t.regex, '^(a{3,5}b?c*)$')
 		self.assertRaises(Exception, t.getDate, '')
-		self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
+		self.assertEqual(t.matchDate('aaaac').group(1), 'aaaac')
 
 		## no word boundaries left and right:
 		t = DatePatternRegex()
@@ -208,22 +210,22 @@ class DateDetectorTest(LogCaptureTestCase):
 		self.assertFalse('**' in t.regex)
 		# match date:
 		dt = 'TIME:20050102T010203'
-		self.assertEqual(t.matchDate('X' + dt + 'X').group(), dt)
+		self.assertEqual(t.matchDate('X' + dt + 'X').group(1), dt)
-		self.assertEqual(t.matchDate(dt).group(), dt)
+		self.assertEqual(t.matchDate(dt).group(1), dt)
 		# wrong year (for exact %ExY):
 		dt = 'TIME:50050102T010203'
 		self.assertFalse(t.matchDate(dt))
 
-		## start boundary left and word boundary right:
+		## start boundary left and word boundary right (automatically if not **):
 		t = DatePatternRegex()
-		t.pattern = '%ExLBtime:%ExY%Exm%ExdT%ExH%ExM%ExS'
+		t.pattern = '{^LN-BEG}time:%ExY%Exm%ExdT%ExH%ExM%ExS'
 		self.assertTrue('^' in t.regex)
 		# try match date:
 		dt = 'time:20050102T010203'
 		self.assertFalse(t.matchDate('X' + dt))
 		self.assertFalse(t.matchDate(dt + 'X'))
-		self.assertEqual(t.matchDate('##' + dt + '...').group(), dt)
+		self.assertEqual(t.matchDate('##' + dt + '...').group(1), dt)
-		self.assertEqual(t.matchDate(dt).group(), dt)
+		self.assertEqual(t.matchDate(dt).group(1), dt)
 		# case sensitive:
 		dt = 'TIME:20050102T010203'
 		self.assertFalse(t.matchDate(dt))
@@ -232,9 +234,9 @@ class DateDetectorTest(LogCaptureTestCase):
 		t = DatePatternRegex()
 		t.pattern = '^%Y %b %d'
 		self.assertTrue('(?iu)' in t.regex)
-		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
-		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
-		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(), dt)
+		dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt)
 
 	def testAmbiguousInOrderedTemplates(self):
 		dd = self.datedetector
@@ -259,7 +261,7 @@ class DateDetectorTest(LogCaptureTestCase):
 				logSys.debug('Line: %s', line)
 				match, template = dd.matchTime(line)
 				self.assertTrue(match)
-				self.assertEqual(match.group(), debit)
+				self.assertEqual(match.group(1), debit)
 
 	def testLowLevelLogging(self):
 		# test coverage for the deep (heavy) debug messages:
@@ -363,31 +365,30 @@ class CustomDateFormatsTest(unittest.TestCase):
 			('200333 010203',   r'%Y%m%d %H%M%S',             "text:200333 010203 | date:20031230 010203"),
 			('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
 			('20031230 010203', None,                         "text:200333 010203 | date:20031230 010203"),
-			# Explicit bound in start of the line using %ExLB key,
+			# Explicit bound in start of the line using {^LN-BEG} key,
 			# (negative) in the 1st case without line begin boundary - wrong date may be found,
 			# (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found)
 			# (positive) and in 3th case with line begin boundary - find the correct date
 			("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS",      "00001230 010203 - 20030101 000000"),
-			(None,              "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
+			(None,              "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
-			("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
+			("20031230 010203", "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
-			# Explicit bound in start of the line using %ExLB key, 
+			# Explicit bound in start of the line using {^LN-BEG} key, 
 			# up to 2 non-alphanumeric chars front, ** - no word boundary on the right
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
-			("20031230010203",  "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
+			("20031230010203",  "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS",   "[20031230010203]20030101000000"),
 		):
 			logSys.debug('== test: %r', (matched, dp, line))
 			if dp is None:
 				dd = defDD
 			else:
-				dp = DatePatternRegex(dp)
 				dd = DateDetector()
 				dd.appendTemplate(dp)
 			date = dd.getTime(line)
 			if matched:
 				self.assertTrue(date)
-				self.assertEqual(matched, date[1].group())
+				self.assertEqual(matched, date[1].group(1))
 			else:
 				self.assertEqual(date, None)
 

fail2ban/tests/fail2banclienttestcase.py

@@ -743,6 +743,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"maxretry = 3",
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",
+				"datepattern = {^LN-BEG}EPOCH",
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",

fail2ban/tests/fail2banregextestcase.py

@@ -101,6 +101,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWrongIngnoreRE(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "{^LN-BEG}EPOCH",
 			"test", r".*? from <HOST>$", r".**"
 		)
 		self.assertFalse(fail2banRegex.start(opts, args))
@@ -108,6 +109,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectFound(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--print-no-missed",
 			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0",
 			r"Authentication failure for .*? from <HOST>$"
@@ -136,6 +138,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -151,6 +154,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1raw(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--raw",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -160,6 +164,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_1raw_noDns(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched", "--raw", "--usedns=no",
 			Fail2banRegexTest.FILENAME_01, 
 			Fail2banRegexTest.RE_00
@@ -169,6 +174,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testDirectRE_2(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--print-all-matched",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
@@ -178,6 +184,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testVerbose(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--verbose", "--verbose-date", "--print-no-missed",
 			Fail2banRegexTest.FILENAME_02, 
 			Fail2banRegexTest.RE_00
@@ -190,6 +197,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWronChar(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)
 		self.assertTrue(fail2banRegex.start(opts, args))
@@ -203,6 +211,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 
 	def testWronCharDebuggex(self):
 		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?",
 			"--debuggex", "--print-all-matched",
 			Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD
 		)

fail2ban/tests/failmanagertestcase.py

@@ -36,6 +36,7 @@ class AddFailure(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(AddFailure, self).setUp()
 		self.__items = None
 		self.__failManager = FailManager()
 

fail2ban/tests/files/logs/zzz-generic-example

@@ -30,8 +30,8 @@ Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-
 # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
 [Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
 
-# -- wrong time direct in journal-line (using precise year pattern):
+# -- wrong time direct in journal-line (used last known date):
-# failJSON: { "match": false}
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.1
 # -- wrong time after newline in message (plist without escaped newlines):
 # failJSON: { "match": false }
@@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={
              applicationDate = "0000-12-30 00:00:00 +0000";
 # failJSON: { "match": false }
 }
-# -- wrong time direct in journal-line (using precise year pattern):
+# -- wrong time direct in journal-line (used last known date):
-# failJSON: { "match": false}
+# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" }
 0000-12-30 00:00:00 server test-demo[47831]:  F2B: failure from 192.0.2.2
 
 # failJSON: { "time": "2005-06-21T16:56:02", "match": true , "host": "192.0.2.250" }

fail2ban/tests/filtertestcase.py

@@ -270,6 +270,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p
 class BasicFilter(unittest.TestCase):
 
 	def setUp(self):
+		super(BasicFilter, self).setUp()
 		self.filter = Filter('name')
 
 	def testGetSetUseDNS(self):
@@ -363,6 +364,7 @@ class IgnoreIP(LogCaptureTestCase):
 		setUpMyTime()
 		self.filter.addIgnoreIP('192.168.1.0/25')
 		self.filter.addFailRegex('<HOST>')
+		self.filter.setDatePattern('{^LN-BEG}EPOCH')
 		self.filter.processLineAndAdd('1387203300.222 192.168.1.32')
 		self.assertLogged('Ignore 192.168.1.32')
 		tearDownMyTime()
@@ -461,6 +463,7 @@ class LogFileFilterPoll(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(LogFileFilterPoll, self).setUp()
 		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(LogFileFilterPoll.FILENAME)
 
@@ -653,6 +656,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		self.assertLogged('Unable to open %s' % self.name)
 
 	def testErrorProcessLine(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 		self.filter.sleeptime /= 1000.0
 		## produce error with not callable processLine:
 		_org_processLine = self.filter.processLine
@@ -715,6 +720,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		pass
 
 	def testNewChangeViaGetFailures_simple(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		# suck in lines from this sample log file
 		self.filter.getFailures(self.name)
 		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
@@ -730,6 +737,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01)
 
 	def testNewChangeViaGetFailures_rewrite(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		#
 		# if we rewrite the file at once
 		self.file.close()
@@ -748,6 +757,8 @@ class LogFileMonitor(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01)
 
 	def testNewChangeViaGetFailures_move(self):
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		#
 		# if we move file into a new location while it has been open already
 		self.file.close()
@@ -769,6 +780,7 @@ class CommonMonitorTestCase(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(CommonMonitorTestCase, self).setUp()
 		self._failTotal = 0
 
 	def waitFailTotal(self, count, delay=1.):
@@ -819,6 +831,8 @@ def get_monitor_failures_testcase(Filter_):
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
 			self.filter.addLogPath(self.name, autoSeek=False)
+			# speedup search using exact date pattern:
+			self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 			self.filter.active = True
 			self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
 			self.filter.start()
@@ -1223,6 +1237,8 @@ class GetFailures(LogCaptureTestCase):
 		self.jail = DummyJail()
 		self.filter = FileFilter(self.jail)
 		self.filter.active = True
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
 		# TODO Test this
 		#self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 		#self.filter.setTimePattern("%b %d %H:%M:%S")
@@ -1329,6 +1345,11 @@ class GetFailures(LogCaptureTestCase):
 		output = (('212.41.96.186', 4, 1124013600.0),
 				  ('212.41.96.185', 2, 1124013598.0))
 
+		# speedup search using exact date pattern:
+		self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?',
+			'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?',
+			'^EPOCH'
+		))
 		self.filter.setMaxRetry(2)
 		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
@@ -1358,6 +1379,8 @@ class GetFailures(LogCaptureTestCase):
 				if enc is not None:
 					self.tearDown();self.setUp();
 					self.filter.setLogEncoding(enc);
+				# speedup search using exact date pattern:
+				self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS')
 				self.assertNotLogged('Error decoding line');
 				self.filter.addLogPath(fname)
 				self.filter.addFailRegex(failregex)
@@ -1533,6 +1556,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(DNSUtilsNetworkTests, self).setUp()
 		unittest.F2B.SkipIfNoNetwork()
 
 	def test_IPAddr(self):

fail2ban/tests/misctestcase.py

@@ -86,6 +86,7 @@ def _getSysPythonVersion():
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
+		super(SetupTest, self).setUp()
 		unittest.F2B.SkipIfFast()
 		setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py')
 		self.setup = os.path.exists(setup) and setup or None

fail2ban/tests/samplestestcase.py

@@ -43,6 +43,7 @@ class FilterSamplesRegex(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(FilterSamplesRegex, self).setUp()
 		self.filter = Filter(None)
 		self.filter.active = True
 

fail2ban/tests/servertestcase.py

@@ -65,7 +65,7 @@ class TransmitterBase(unittest.TestCase):
 	
 	def setUp(self):
 		"""Call before every test case."""
-		#super(TransmitterBase, self).setUp()
+		super(TransmitterBase, self).setUp()
 		self.transm = self.server._Server__transm
 		# To test thransmitter we don't need to start server...
 		#self.server.start('/dev/null', '/dev/null', force=False)
@@ -301,9 +301,11 @@ class Transmitter(TransmitterBase):
 			("%%%Y%m%d%H%M%S", "{*WD-BEG}%YearMonthDay24hourMinuteSecond{*WD-END}"),
 			jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "Epoch", (None, "Epoch"), jail=self.jailName)
+			"datepattern", "Epoch", (None, "Epoch{*WD-END}"), jail=self.jailName)
 		self.setGetTest(
-			"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
+			"datepattern", "^Epoch", (None, "{^LN-BEG}Epoch{*WD-END}"), jail=self.jailName)
+		self.setGetTest(
+			"datepattern", "TAI64N", (None, "TAI64N{*WD-END}"), jail=self.jailName)
 		self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
 
 	def testJailUseDNS(self):

fail2ban/tests/sockettestcase.py

@@ -41,6 +41,7 @@ class Socket(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		super(Socket, self).setUp()
 		self.server = AsyncServer(self)
 		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'socket')
 		os.close(sock_fd)

fail2ban/tests/utils.py

@@ -526,6 +526,16 @@ if True: ## if not hasattr(unittest.TestCase, 'assertIn'):
 			self.fail(msg)
 	unittest.TestCase.assertNotIn = assertNotIn
 
+_org_setUp = unittest.TestCase.setUp
+def _customSetUp(self):
+	# print('=='*10, self)
+	if unittest.F2B.log_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
+		print("")
+		logSys.debug('='*10 + ' %s ' + '='*20, self.id())
+	_org_setUp(self)
+
+unittest.TestCase.setUp = _customSetUp
+
 
 class LogCaptureTestCase(unittest.TestCase):
 
@@ -601,12 +611,11 @@ class LogCaptureTestCase(unittest.TestCase):
 		# Let's log everything into a string
 		self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy)
 		logSys.handlers = [self._log]
-		if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)!
+		if self._old_level <= logging.DEBUG:
-			print("")
 			logSys.handlers += self._old_handlers
-			logSys.debug('='*10 + ' %s ' + '='*20, self.id())
+		else: # lowest log level to capture messages
-		else:
 			logSys.setLevel(logging.DEBUG)
+		super(LogCaptureTestCase, self).setUp()
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -615,6 +624,7 @@ class LogCaptureTestCase(unittest.TestCase):
 		logSys = getLogger("fail2ban")
 		logSys.handlers = self._old_handlers
 		logSys.level = self._old_level
+		super(LogCaptureTestCase, self).tearDown()
 
 	def _is_logged(self, *s, **kwargs):
 		logged = self._log.getvalue()