MRG: merge date changes to support timezones

ChangeLog

@@ -67,6 +67,8 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
      all platforms to ensure permissions are the same before and after a ban -
      closes gh-266. hostsdeny supports daemon_list now too.
    * filter.d/roundcube-auth - timezone offset can be positive or negative
+   * action.d/bsd-ipfw - action option unsed. Fixed to blocktype for
+     consistency. default to port unreach instead of deny
   Rolf Fokkens
    * action.d/dshield.conf and complain.conf -- reorder mailx arguments.
      https://bugzilla.redhat.com/show_bug.cgi?id=998020
@@ -78,6 +80,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
      avoiding problems with getpid.  Also $network and iptables moved
      to Should- rc init fields
 - New Features:
+  Andy Fragen and Daniel Black
+	 * filter.d/osx-ipfw.conf - ipfw action for OSX based on random rule
+     numbers.
   Daniel Black & ykimon
    * filter.d/3proxy.conf -- filter added
   Daniel Black
@@ -95,6 +100,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
      and extra failure examples in sample logs
 	 * filter.d/apache-auth - added expressions for mod_authz, mod_auth and
      mod_auth_digest failures.
+   * Support %z (Timezone offset) and %f (sub-seconds) support for datedetector
   Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий
    * filter.d/exim.conf -- regex hardening and extra failure examples in
      sample logs

THANKS

@@ -7,6 +7,7 @@ will be added
 Adrien Clerc
 ache
 Andrey G. Grozin
+Andy Fragen
 Arturo 'Buanzo' Busleiman
 Axel Thimm
 Bill Heaton

config/action.d/bsd-ipfw.conf

@@ -14,7 +14,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num deny <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
 
 
 # Option:  actionstop
@@ -68,15 +68,16 @@ port =
 # Values:  STRING
 startstatefile = /var/run/fail2ban/ipfw-started-table_<table>
 
-# Option:  action
-# Notes:   This is the action to take for automaticly created rules. See the 
-#          ACTION defination at the top of man ipfw for allowed values.
-#          "deny" and "unreach port" are probably the useful.
-# Values:  STRING
-action = deny
-
 # Option: block
 # Notes:  This is how much to block.
 #         Can be "ip", "tcp", "udp" or various other options.
 # Values: STRING
 block = ip
+
+# Option:  blocktype
+# Notes.:  How to block the traffic. Use a action from man 5 ipfw
+#          Common values: deny, unreach port, reset
+#          ACTION defination at the top of man ipfw for allowed values.
+# Values:  STRING
+#
+blocktype = unreach port

config/action.d/osx-ipfw.conf

@@ -0,0 +1,87 @@
+# Fail2Ban configuration file
+#
+# Author: Nick Munger
+# Modified by: Andy Fragen and Daniel Black
+#
+# Mod for OS X, using random rulenum as OSX ipfw doesn't include tables
+#
+
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+actionstart = 
+
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+actionstop = 
+
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck = 
+
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+# Values:  CMD
+#
+actionban = ipfw add <rulenum> set <setnum> <blocktype> log <block> from <ip> to <dst> <port>
+
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    <ip>  IP address
+# Values:  CMD
+#
+actionunban = ipfw delete `ipfw -S list | grep -i 'set <setnum> <blocktype> log <block> from <ip> to <dst>' | awk '{print $1;}'`
+
+[Init]
+
+# Option:  port
+# Notes.:  specifies port to block. Can be blank however may require block="ip"
+# Values:  [ NUM | STRING ]
+#
+port = ssh
+
+# Option:  dst
+# Notes.:  the local IP address of the network interface
+# Values:  IP, any, me or anything support by ipfw as a dst
+#
+dst = me
+
+# Option: block
+# Notes:  This is how much to block.
+#         Can be "ip", "tcp", "udp" or various other options.
+# Values: STRING
+block = tcp
+
+# Option:  blocktype
+# Notes.:  How to block the traffic. Use a action from man 8 ipfw
+#          Common values: deny, unreach port, reset
+# Values:  STRING
+#
+blocktype = unreach port
+
+# Option:  set number
+# Notes.:  The ipset number this is added to.
+# Values:  0-31
+setnum = 10
+
+# Option:  number for ipfw rule
+# Notes:   This is meant to be automaticly generated and not overwritten
+# Values:  Random value between 10000 and 12000
+rulenum="`echo $((RANDOM%%2000+10000))`"
+
+# Duplicate prevention mechanism
+#rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`"

config/filter.d/apache-common.conf

@@ -18,4 +18,4 @@ after = apache-common.local
 # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]
 # 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652]
 # Reference: https://github.com/fail2ban/fail2ban/issues/268
-_apache_error_client = \[[^]]*\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]
+_apache_error_client = \[\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]

config/filter.d/named-refused.conf

@@ -22,10 +22,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)
 __line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)?
 
 
-# note - (\.\d+)? is a really ugly catch of the microseconds not captured in
+failregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
-# in the date detector
+            ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
-#
+            ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
-failregex = ^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
-            ^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
-            ^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
 

config/filter.d/roundcube-auth.conf

@@ -17,7 +17,7 @@ before = common.conf
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. AUTHENTICATE .*)?\s*$
+failregex = ^\s*(\[\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. AUTHENTICATE .*)?\s*$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

config/jail.conf

@@ -531,3 +531,47 @@ action   = iptables-allports[name=recidive]
 bantime  = 604800  ; 1 week
 findtime = 86400   ; 1 day
 maxretry = 5
+
+# PF is a BSD based firewall
+[ssh-pf]
+
+enabled=false
+filter = sshd
+action = pf
+logpath  = /var/log/sshd.log
+maxretry=5
+
+[3proxy]
+
+enabled = false
+action  = iptables[name=3proxy, port=3128, protocol=tcp]
+logpath = /var/log/3proxy.log
+
+
+[exim]
+
+enabled = false
+action = iptables-multiport[name=exim,port="25,465,587"]
+logpath = /var/log/exim/mainlog
+
+
+[exim-spam]
+
+enabled = false
+action = iptables-multiport[name=exim-spam,port="25,465,587"]
+logpath = /var/log/exim/mainlog
+
+
+[perdition]
+
+enabled = false
+action = iptables-multiport[name=perdition,port="110,143,993,995"]
+logpath = /var/log/maillog
+
+
+[osx-ssh-ipfw]
+
+enabled = false
+filter = sshd
+action = osx-ipfw
+logpath = /var/log/secure.log

fail2ban/server/datetemplate.py

@@ -24,7 +24,10 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import re, time
+import re, time, calendar
+
+from datetime import datetime
+from datetime import timedelta
 
 from mytime import MyTime
 import iso8601
@@ -82,12 +85,11 @@ class DateEpoch(DateTemplate):
 		self.setRegex("^\d{10}(\.\d{6})?")
 	
 	def getDate(self, line):
-		date = None
 		dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
-			date = list(MyTime.localtime(float(dateMatch.group())))
+			return (float(dateMatch.group()), dateMatch)
-		return date
+		return None
 
 
 ##
@@ -114,9 +116,12 @@ class DateStrptime(DateTemplate):
 	def __init__(self):
 		DateTemplate.__init__(self)
 		self.__pattern = ""
+		self.__unsupportedStrptimeBits = False
 	
 	def setPattern(self, pattern):
-		self.__pattern = pattern.strip()
+		self.__unsupported_f = not DateStrptime._f and re.search('%f', pattern)
+		self.__unsupported_z = not DateStrptime._z and re.search('%z', pattern)
+		self.__pattern = pattern
 		
 	def getPattern(self):
 		return self.__pattern
@@ -133,17 +138,26 @@ class DateStrptime(DateTemplate):
 	convertLocale = staticmethod(convertLocale)
 	
 	def getDate(self, line):
-		date = None
 		dateMatch = self.matchDate(line)
+
 		if dateMatch:
+			datePattern = self.getPattern()
+			if self.__unsupported_f:
+				if dateMatch.group('_f'):
+					datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern)
+					logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern))
+			if self.__unsupported_z:
+				if dateMatch.group('_z'):
+					datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern)
+					logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern))
 			try:
 				# Try first with 'C' locale
-				date = list(time.strptime(dateMatch.group(), self.getPattern()))
+				date = datetime.strptime(dateMatch.group(), datePattern)
 			except ValueError:
 				# Try to convert date string to 'C' locale
 				conv = self.convertLocale(dateMatch.group())
 				try:
-					date = list(time.strptime(conv, self.getPattern()))
+					date = datetime.strptime(conv, self.getPattern())
 				except (ValueError, re.error), e:
 					# Try to add the current year to the pattern. Should fix
 					# the "Feb 29" issue.
@@ -152,7 +166,7 @@ class DateStrptime(DateTemplate):
 					if not '%Y' in opattern:
 						pattern = "%s %%Y" % opattern
 						conv += " %s" % MyTime.gmtime()[0]
-						date = list(time.strptime(conv, pattern))
+						date = datetime.strptime(conv, pattern)
 					else:
 						# we are helpless here
 						raise ValueError(
@@ -160,26 +174,60 @@ class DateStrptime(DateTemplate):
 							"exception was %r and Feb 29 workaround could not "
 							"be tested due to already present year mark in the "
 							"pattern" % (opattern, e))
-			if date[0] < 2000:
+
+			if self.__unsupported_z:
+				z = dateMatch.group('_z')
+				if z:
+					delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:]))
+					direction = z[0]
+					logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta))
+					# here we reverse the effect of the timezone and force it to UTC
+					if direction == '+':
+						date -= delta
+					else:
+						date += delta
+					date = date.replace(tzinfo=iso8601.Utc())
+				else:
+					logSys.warn("No _z group captured and %%z is not supported on current platform"
+								" - timezone ignored and assumed to be localtime. date: %s on line: %s"
+								% (date, line))
+
+			if date.year < 2000:
 				# There is probably no year field in the logs
 				# NOTE: Possibly makes week/year day incorrect
-				date[0] = MyTime.gmtime()[0]
+				date = date.replace(year=MyTime.gmtime()[0])
 				# Bug fix for #1241756
 				# If the date is greater than the current time, we suppose
 				# that the log is not from this year but from the year before
-				if time.mktime(tuple(date)) > MyTime.time():
+				if date > MyTime.now():
 					logSys.debug(
-						u"Correcting deduced year from %d to %d since %f > %f" %
+						u"Correcting deduced year by one since %s > now (%s)" %
-						(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
+						(date, MyTime.time()))
-					# NOTE: Possibly makes week/year day incorrect
+					date = date.replace(year=date.year-1)
-					date[0] -= 1
+				elif date.month == 1 and date.day == 1:
-				elif date[1] == 1 and date[2] == 1:
 					# If it is Jan 1st, it is either really Jan 1st or there
 					# is neither month nor day in the log.
 					# NOTE: Possibly makes week/year day incorrect
-					date[1] = MyTime.gmtime()[1]
+					date = date.replace(month=MyTime.gmtime()[1], day=1)
-					date[2] = MyTime.gmtime()[2]
+
-		return date
+			if date.tzinfo:
+				return ( calendar.timegm(date.utctimetuple()), dateMatch )
+			else:
+				return ( time.mktime(date.utctimetuple()), dateMatch )
+				
+		return None
+
+try:
+	time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f")
+	DateStrptime._f = True
+except (ValueError, KeyError):
+	DateTemplate._f = False
+
+try:
+	time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z")
+	DateStrptime._z = True
+except ValueError:
+	DateStrptime._z = False
 
 class DatePatternRegex(DateStrptime):
 	_reEscape = r"([\\.^$*+?\(\){}\[\]|])"
@@ -227,6 +275,7 @@ class DatePatternRegex(DateStrptime):
 	def setName(self, line):
 		raise NotImplementedError("Name derived from pattern")
 
+
 class DateTai64n(DateTemplate):
 	
 	def __init__(self):
@@ -236,15 +285,14 @@ class DateTai64n(DateTemplate):
 		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
 	
 	def getDate(self, line):
-		date = None
 		dateMatch = self.matchDate(line)
 		if dateMatch:
 			# extract part of format which represents seconds since epoch
 			value = dateMatch.group()
 			seconds_since_epoch = value[2:17]
 			# convert seconds from HEX into local time stamp
-			date = list(MyTime.localtime(int(seconds_since_epoch, 16)))
+			return (int(seconds_since_epoch, 16), dateMatch)
-		return date
+		return None
 
 
 class DateISO8601(DateTemplate):
@@ -257,11 +305,10 @@ class DateISO8601(DateTemplate):
 		self.setRegex(date_re)
 	
 	def getDate(self, line):
-		date = None
 		dateMatch = self.matchDate(line)
 		if dateMatch:
 			# Parses the date.
 			value = dateMatch.group()
-			date = list(iso8601.parse_date(value, None).timetuple())
+			return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch)
-		return date
+		return None
 

fail2ban/server/filter.py

@@ -350,22 +350,9 @@ class Filter(JailThread):
 		line = line.rstrip('\r\n')
 		logSys.log(7, "Working on line %r", line)
 
-		timeMatch = self.dateDetector.matchTime(line)
+		logSys.log(7, "Working on line %r", l)
-		if timeMatch:
+
-			# Lets split into time part and log part of the line
+		return self.findFailure(l, returnRawHost, checkAllRegex)
-			timeLine = timeMatch.group()
-			self.__lastTimeLine = timeLine
-			# Lets leave the beginning in as well, so if there is no
-			# anchore at the beginning of the time regexp, we don't
-			# at least allow injection. Should be harmless otherwise
-			logLine  = line[:timeMatch.start()] + line[timeMatch.end():]
-		else:
-			timeLine = self.__lastTimeLine or line
-			logLine = line
-		self.__lineBuffer = ((self.__lineBuffer +
-				[logLine])[-self.__lineBufferSize:])
-		return self.findFailure(timeLine, "\n".join(self.__lineBuffer) + "\n",
-								returnRawHost, checkAllRegex)
 
 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
@@ -408,11 +395,29 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, timeLine, logLine,
+	def findFailure(self, logLine,
 			returnRawHost=False, checkAllRegex=False):
-		logSys.log(5, "Date: %r, message: %r", timeLine, logLine)
 		failList = list()
-		date = self.dateDetector.getUnixTime(timeLine)
+
+		# Checks if we must ignore this line.
+		if self.ignoreLine(logLine) is not None:
+			# The ignoreregex matched. Return.
+			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine)
+			return failList
+		dd = self.dateDetector.getTime(logLine)
+		
+		if dd is None:
+			return failList
+		date = dd[0]
+		timeMatch = dd[1]
+		if timeMatch:
+			# Lets split into time part and log part of the line
+			timeLine = timeMatch.group()
+			# Lets leave the beginning in as well, so if there is no
+			# anchore at the beginning of the time regexp, we don't
+			# at least allow injection. Should be harmless otherwise
+			logLine  = logLine[:timeMatch.start()] + logLine[timeMatch.end():]
+
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
 			failRegex.search(logLine)

fail2ban/server/mytime.py

@@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import time
+import time, datetime
 
 ##
 # MyTime class.
@@ -74,6 +74,14 @@ class MyTime:
 			return time.gmtime(MyTime.myTime)
 	gmtime = staticmethod(gmtime)
 
+	#@staticmethod
+	def now():
+		if MyTime.myTime is None:
+			return datetime.now()
+		else:
+			return datetime.datetime.fromtimestamp(MyTime.myTime)
+	now = staticmethod(now)
+
 	def localtime(x=None):
 		if MyTime.myTime is None or x is not None:
 			return time.localtime(x)

fail2ban/tests/datedetectortestcase.py

@@ -24,9 +24,10 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import unittest, calendar, datetime, re, pprint
+import unittest, calendar, time, datetime, re, pprint
 from fail2ban.server.datedetector import DateDetector
 from fail2ban.server.datetemplate import DateTemplate
+from fail2ban.server.iso8601 import Utc
 from fail2ban.tests.utils import setUpMyTime, tearDownMyTime
 
 class DateDetectorTest(unittest.TestCase):
@@ -43,11 +44,12 @@ class DateDetectorTest(unittest.TestCase):
 	
 	def testGetEpochTime(self):
 		log = "1138049999 [sshd] error: PAM: Authentication failure"
-		date = [2006, 1, 23, 21, 59, 59, 0, 23, 0]
+		#date = [2006, 1, 23, 21, 59, 59, 0, 23, 0]
 		dateUnix = 1138049999.0
 
-		self.assertEqual(self.__datedetector.getTime(log), date)
+		( datelog, matchlog ) = self.__datedetector.getTime(log)
-		self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
+		self.assertEqual(datelog, dateUnix)
+		self.assertEqual(matchlog.group(), '1138049999')
 	
 	def testGetTime(self):
 		log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure"
@@ -57,8 +59,9 @@ class DateDetectorTest(unittest.TestCase):
 		#      is not correctly determined atm, since year is not present
 		#      in the log entry.  Since this doesn't effect the operation
 		#      of fail2ban -- we just ignore incorrect day of the week
-		self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6])
+		( datelog, matchlog ) = self.__datedetector.getTime(log)
-		self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
+		self.assertEqual(datelog, dateUnix)
+		self.assertEqual(matchlog.group(), 'Jan 23 21:59:59')
 
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
@@ -68,21 +71,23 @@ class DateDetectorTest(unittest.TestCase):
 
 		for sdate in (
 			"Jan 23 21:59:59",
+			"Sun Jan 23 21:59:59.011 2005",
 			"Sun Jan 23 21:59:59 2005",
 			"Sun Jan 23 21:59:59",
 			"2005/01/23 21:59:59",
 			"2005.01.23 21:59:59",
 			"23/01/2005 21:59:59",
 			"23/01/05 21:59:59",
-			"23/Jan/2005:21:59:59",
+			"23/Jan/2005:21:59:59 +0100",
 			"01/23/2005:21:59:59",
 			"2005-01-23 21:59:59",
-			"23-Jan-2005 21:59:59",
+			"23-Jan-2005 21:59:59.02",
+			"23-Jan-2005 21:59:59 +0100",
 			"23-01-2005 21:59:59",
 			"01-23-2005 21:59:59.252", # reported on f2b, causes Feb29 fix to break
 			"@4000000041f4104f00000000", # TAI64N
-			"2005-01-23T21:59:59.252Z", #ISO 8601
+			"2005-01-23T20:59:59.252Z", #ISO 8601
-			"2005-01-23T21:59:59-05:00Z", #ISO 8601 with TZ
+			"2005-01-23T15:59:59-05:00", #ISO 8601 with TZ
 			"<01/23/05@21:59:59>",
 			"050123 21:59:59", # MySQL
 			"Jan 23, 2005 9:59:59 PM", # Apache Tomcat
@@ -94,8 +99,9 @@ class DateDetectorTest(unittest.TestCase):
 			# yoh: on [:6] see in above test
 			logtime = self.__datedetector.getTime(log)
 			self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s" % sdate)
-			self.assertEqual(logtime[:6], date[:6], "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6]))
+			( logUnix, logMatch ) = logtime
-			self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix, "getUnixTime failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6]))
+			self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix))
+			self.assertEqual(logMatch.group(), sdate)
 
 	def testStableSortTemplate(self):
 		old_names = [x.getName() for x in self.__datedetector.getTemplates()]
@@ -112,21 +118,23 @@ class DateDetectorTest(unittest.TestCase):
 		# see https://github.com/fail2ban/fail2ban/pull/130
 		# yoh: unfortunately this test is not really effective to reproduce the
 		#      situation but left in place to assure consistent behavior
-		m1 = [2012, 10, 11, 2, 37, 17]
+		mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).utctimetuple())
-		self.assertEqual(
+		logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
-			self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
+		self.assertNotEqual(logdate, None)
-			m1)
+		( logTime, logMatch ) = logdate
+		self.assertEqual(logTime, mu)
+		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 		self.__datedetector.sortTemplate()
 		# confuse it with year being at the end
 		for i in xrange(10):
-			self.assertEqual(
+			( logTime, logMatch ) =	self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
-				self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')[:6],
+			self.assertEqual(logTime, mu)
-				m1)
+			self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
 		self.__datedetector.sortTemplate()
 		# and now back to the original
-		self.assertEqual(
+		( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
-			self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
+		self.assertEqual(logTime, mu)
-			m1)
+		self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
 
 	def testDateDetectorTemplateOverlap(self):
 		patterns = [template.getPattern()
@@ -141,12 +149,13 @@ class DateDetectorTest(unittest.TestCase):
 						for minute in xrange(0, 60, 15):
 							for second in xrange(0, 60, 15): # Far enough?
 								yield datetime.datetime(
-									year, month, day, hour, minute, second)
+									year, month, day, hour, minute, second, 300, Utc())
 
 		overlapedTemplates = set()
 		for date in iterDates(year):
 			for pattern in patterns:
 				datestr = date.strftime(pattern)
+				datestr = re.sub(r'%f','300', datestr) # for python 2.5 where there is no %f
 				datestrs = set([
 					datestr,
 					re.sub(r"(\s)0", r"\1 ", datestr),
@@ -160,12 +169,12 @@ class DateDetectorTest(unittest.TestCase):
 				matchedTemplates = [template
 					for template in self.__datedetector.getTemplates()
 					if template.getHits() > 0]
-				assert matchedTemplates != [] # Should match at least one
+				self.assertNotEqual(matchedTemplates, [], "Date %r should match at least one template" % pattern)
 				if len(matchedTemplates) > 1:
 					overlapedTemplates.add((pattern, tuple(sorted(template.getName()
 						for template in matchedTemplates))))
 		if overlapedTemplates:
-			print "WARNING: The following date templates overlap:"
+			print("WARNING: The following date templates overlap:")
 			pprint.pprint(overlapedTemplates)
 
 #	def testDefaultTempate(self):

fail2ban/tests/files/logs/dovecot

@@ -1,12 +1,12 @@
-# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" }
+# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "80.187.101.33" }
 @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=<martin@waschbuesch.de>, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS
 
-# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" }
+# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.224" }
 @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224
 # Above example with injected rhost into ruser -- should not match for 1.2.3.4
-# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" }
+# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "192.0.43.10" }
 @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10
-# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" }
+# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.225" }
 @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225  user=root
 
 # failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" }

fail2ban/tests/files/logs/php-url-fopen

@@ -1,2 +1,2 @@
-# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" }
+# failJSON: { "time": "2009-03-26T14:44:20", "match": true , "host": "66.185.212.172" }
 66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"

fail2ban/tests/files/logs/roundcube-auth

@@ -1,4 +1,4 @@
-# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" }
+# failJSON: { "time": "2013-01-22T21:28:21", "match": true , "host": "192.0.43.10" }
 [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10
 # failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" }
 May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47

fail2ban/tests/filtertestcase.py

@@ -48,6 +48,7 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 #
 
 from utils import mtimesleep
+from server.mytime import MyTime
 
 # yoh: per Steven Hiscocks's insight while troubleshooting
 # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836
@@ -89,8 +90,8 @@ def _assert_equal_entries(utest, found, output, count=None):
 	utest.assertEqual(found[0], output[0])            # IP
 	utest.assertEqual(found[1], count or output[1])   # count
 	found_time, output_time = \
-				time.localtime(found[2]),\
+				MyTime.localtime(found[2]),\
-				time.localtime(output[2])
+				MyTime.localtime(output[2])
 	utest.assertEqual(found_time, output_time)
 	if len(output) > 3 and count is None: # match matches
 		# do not check if custom count (e.g. going through them twice)
@@ -737,8 +738,8 @@ class GetFailures(unittest.TestCase):
 	FILENAME_MULTILINE = os.path.join(TEST_FILES_DIR, "testcase-multiline.log")
 
 	# so that they could be reused by other tests
-	FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
+	FAILURES_01 = ('193.168.0.128', 3, 1124017199.0,
-				  [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
+				  ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
 
 	def setUp(self):
 		"""Call before every test case."""
@@ -783,8 +784,8 @@ class GetFailures(unittest.TestCase):
 
 
 	def testGetFailures02(self):
-		output = ('141.3.81.106', 4, 1124013539.0,
+		output = ('141.3.81.106', 4, 1124017139.0,
-				  [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
+				  ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
 				   % m for m in 53, 54, 57, 58])
 
 		self.filter.addLogPath(GetFailures.FILENAME_02)
@@ -793,7 +794,7 @@ class GetFailures(unittest.TestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures03(self):
-		output = ('203.162.223.135', 6, 1124013544.0)
+		output = ('203.162.223.135', 6, 1124017144.0)
 
 		self.filter.addLogPath(GetFailures.FILENAME_03)
 		self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
@@ -801,8 +802,8 @@ class GetFailures(unittest.TestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures04(self):
-		output = [('212.41.96.186', 4, 1124013600.0),
+		output = [('212.41.96.186', 4, 1124017200.0),
-				  ('212.41.96.185', 4, 1124013598.0)]
+				  ('212.41.96.185', 4, 1124017198.0)]
 
 		self.filter.addLogPath(GetFailures.FILENAME_04)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
@@ -816,12 +817,12 @@ class GetFailures(unittest.TestCase):
 
 	def testGetFailuresUseDNS(self):
 		# We should still catch failures with usedns = no ;-)
-		output_yes = ('93.184.216.119', 2, 1124013539.0,
+		output_yes = ('93.184.216.119', 2, 1124017139.0,
-					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
+					  ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
-					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
+					   'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
 
-		output_no = ('93.184.216.119', 1, 1124013539.0,
+		output_no = ('93.184.216.119', 1, 1124017139.0,
-					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
+					  ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
 
 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,
@@ -842,7 +843,7 @@ class GetFailures(unittest.TestCase):
 
 
 	def testGetFailuresMultiRegex(self):
-		output = ('141.3.81.106', 8, 1124013541.0)
+		output = ('141.3.81.106', 8, 1124017141.0)
 
 		self.filter.addLogPath(GetFailures.FILENAME_02)
 		self.filter.addFailRegex("Failed .* from <HOST>")
@@ -851,7 +852,7 @@ class GetFailures(unittest.TestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailuresIgnoreRegex(self):
-		output = ('141.3.81.106', 8, 1124013541.0)
+		output = ('141.3.81.106', 8, 1124017141.0)
 
 		self.filter.addLogPath(GetFailures.FILENAME_02)
 		self.filter.addFailRegex("Failed .* from <HOST>")

fail2ban/tests/samplestestcase.py

@@ -22,7 +22,9 @@
 __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
-import unittest, sys, os, fileinput, re, datetime, inspect
+import unittest, sys, os, fileinput, re, time, datetime, inspect
+from server.mytime import MyTime
+
 
 if sys.version_info >= (2, 6):
 	import json
@@ -119,15 +121,19 @@ def testSampleRegexsFactory(name):
 								 (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
 
 				# Verify timestamp and host as expected
-				failregex, host, time = ret[0]
+				failregex, host, fail2banTime = ret[0]
 				self.assertEqual(host, faildata.get("host", None))
-				fail2banTime = datetime.datetime.fromtimestamp(time)
+
-				jsonTime = datetime.datetime.strptime(
+				t = faildata.get("time", None)
-							faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")
+				jsonTimeLocal =	datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
+
+				jsonTime = time.mktime(jsonTimeLocal.utctimetuple())
 				
 				self.assertEqual(fail2banTime, jsonTime,
-					"Time  mismatch %s != %s on: %s:%i %r:" % 
+					"UTC Time  mismatch fail2ban %s (%s) != failJson %s (%s)  (diff %i seconds) on: %s:%i %r:" % 
-					(fail2banTime, jsonTime, logFile.filename(), logFile.filelineno(), line ) )
+					(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
+					jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
+					fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) )
 
 				regexsUsed.add(failregex)