Merge branch '0.10' into 0.11

config/action.d/cloudflare.conf

@@ -44,7 +44,7 @@ actioncheck =
 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
 actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \
-            -d '{"mode":"block","configuration":{"target":"ip","value":"<ip>"},"notes":"Fail2Ban <name>"}' \
+            -d '{"mode":"block","configuration":{"target":"<cftarget>","value":"<ip>"},"notes":"Fail2Ban <name>"}' \
             <_cf_api_url>
 
 # Option:  actionunban
@@ -59,7 +59,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \
 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>'
 # API v4
 actionunban = id=$(curl -s -X GET <_cf_api_prms> \
-                   "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1&notes=Fail2Ban%%20<name>" \
+                   "<_cf_api_url>?mode=block&configuration_target=<cftarget>&configuration_value=<ip>&page=1&per_page=1&notes=Fail2Ban%%20<name>" \
                    | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; })
               if [ -z "$id" ]; then echo "<name>: id for <ip> cannot be found"; exit 0; fi;
               curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id"
@@ -81,3 +81,8 @@ _cf_api_prms = -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' -H 'Conten
 cftoken =
 
 cfuser =
+
+cftarget = ip
+
+[Init?family=inet6]
+cftarget = ip6

config/filter.d/apache-overflows.conf

@@ -8,7 +8,7 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
+failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
 
 ignoreregex =
 

fail2ban/client/fail2banregex.py

@@ -39,7 +39,6 @@ import os
 import shlex
 import sys
 import time
-import time
 import urllib
 from optparse import OptionParser, Option
 
@@ -52,7 +51,7 @@ except ImportError:
 
 from ..version import version, normVersion
 from .filterreader import FilterReader
-from ..server.filter import Filter, FileContainer
+from ..server.filter import Filter, FileContainer, MyTime
 from ..server.failregex import Regex, RegexException
 
 from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \
@@ -269,15 +268,19 @@ class Fail2banRegex(object):
 			self.setJournalMatch(shlex.split(opts.journalmatch))
 		if opts.timezone:
 			self._filter.setLogTimeZone(opts.timezone)
+		self._filter.checkFindTime = False
+		if True: # not opts.out:
+			MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns)
+			from ..server.strptime import _updateTimeRE
+			_updateTimeRE()
 		if opts.datepattern:
 			self.setDatePattern(opts.datepattern)
 		if opts.usedns:
 			self._filter.setUseDns(opts.usedns)
 		self._filter.returnRawHost = opts.raw
-		self._filter.checkFindTime = False
 		self._filter.checkAllRegex = opts.checkAllRegex and not opts.out
 		# ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved)
-		self._filter.ignorePending = opts.out
+		self._filter.ignorePending = bool(opts.out)
 		# callback to increment ignored RE's by index (during process):
 		self._filter.onIgnoreRegex = self._onIgnoreRegex
 		self._backend = 'auto'

fail2ban/server/datedetector.py

@@ -35,7 +35,7 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-logLevel = 6
+logLevel = 5
 
 RE_DATE_PREMATCH = re.compile(r"(?<!\\)\{DATE\}", re.IGNORECASE)
 DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)

fail2ban/server/datetemplate.py

@@ -136,7 +136,7 @@ class DateTemplate(object):
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
-		logSys.log(7, '  constructed regex %s', regex)
+		logSys.log(4, '  constructed regex %s', regex)
 		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
@@ -159,6 +159,7 @@ class DateTemplate(object):
 		"""
 		if not self._cRegex:
 			self._compileRegex()
+		logSys.log(4, "   search %s", self.regex)
 		dateMatch = self._cRegex.search(line, *args); # pos, endpos
 		if dateMatch:
 			self.hits += 1

fail2ban/server/filtersystemd.py

@@ -95,6 +95,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			if 'files' not in args or not len(args['files']):
 				args['flags'] = 4
 				
+		try:
+			args['namespace'] = kwargs.pop('namespace')
+		except KeyError:
+			pass
+
 		return args
 
 	##

fail2ban/server/strptime.py

@@ -30,6 +30,47 @@ locale_time = LocaleTime()
 TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?"
 FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,))
 
+timeRE = TimeRE()
+
+# %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
+# (corresponds %H, but allows space if not zero-padded).
+# %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
+# (corresponds %I, but allows space if not zero-padded).
+timeRE['k'] = r" ?(?P<H>[0-2]?\d)"
+timeRE['l'] = r" ?(?P<I>1?\d)"
+
+# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
+#       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), 
+#       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' 
+#       with wrong TZ "error", which is at least not backwards compatible.
+#       Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
+timeRE['Z'] = r"(?P<Z>Z|[A-Z]{3,5})"
+timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)"
+
+# Note: this extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase,
+#       see todo above. Don't use them in default date-patterns (if not anchored, few precise resp. optional).
+timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,)
+timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE)
+
+# overwrite default patterns, since they can be non-optimal:
+timeRE['d'] = r"(?P<d>[1-2]\d|[0 ]?[1-9]|3[0-1])"
+timeRE['m'] = r"(?P<m>0?[1-9]|1[0-2])"
+timeRE['Y'] = r"(?P<Y>\d{4})"
+timeRE['H'] = r"(?P<H>[0-1]?\d|2[0-3])"
+timeRE['M'] = r"(?P<M>[0-5]?\d)"
+timeRE['S'] = r"(?P<S>[0-5]?\d|6[0-1])"
+
+# Extend build-in TimeRE with some exact patterns
+# exact two-digit patterns:
+timeRE['Exd'] = r"(?P<d>[1-2]\d|0[1-9]|3[0-1])"
+timeRE['Exm'] = r"(?P<m>0[1-9]|1[0-2])"
+timeRE['ExH'] = r"(?P<H>[0-1]\d|2[0-3])"
+timeRE['Exk'] = r" ?(?P<H>[0-1]?\d|2[0-3])"
+timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)"
+timeRE['ExM'] = r"(?P<M>[0-5]\d)"
+timeRE['ExS'] = r"(?P<S>[0-5]\d|6[0-1])"
+
+def _updateTimeRE():
 	def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
 		""" Build century regex for last year and the next years (distance).
 			
@@ -57,46 +98,22 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 			return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \
 				if len(exprset) > 1 else "".join(exprset)
 		exprset = set( cent(now[0].year + i) for i in (-1, distance) )
-	if len(now) and now[1]:
+		if len(now) > 1 and now[1]:
 			exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) )
 		return grp(sorted(list(exprset)))
 
-timeRE = TimeRE()
-
-# %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
-# (corresponds %H, but allows space if not zero-padded).
-# %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
-# (corresponds %I, but allows space if not zero-padded).
-timeRE['k'] = r" ?(?P<H>[0-2]?\d)"
-timeRE['l'] = r" ?(?P<I>1?\d)"
-
-# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
-#       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), 
-#       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' 
-#       with wrong TZ "error", which is at least not backwards compatible.
-#       Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
-timeRE['Z'] = r"(?P<Z>Z|[A-Z]{3,5})"
-timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)"
-
-# Note: this extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase,
-#       see todo above. Don't use them in default date-patterns (if not anchored, few precise resp. optional).
-timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,)
-timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE)
-
-# Extend build-in TimeRE with some exact patterns
-# exact two-digit patterns:
-timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
-timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])"
-timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)"
-timeRE['Exk'] = r" ?(?P<H>2[0-3]|[0-1]\d|\d)"
-timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)"
-timeRE['ExM'] = r"(?P<M>[0-5]\d)"
-timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
 	# more precise year patterns, within same century of last year and
 	# the next 3 years (for possible long uptime of fail2ban); thereby
 	# respect possible run in the test-cases (alternate date used there):
+	if MyTime.alternateNowTime != 0:
 		timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3)
 		timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3)
+	else: # accept years: 19xx|2xxx up to current century
+		timeRE['ExY'] = r"(?P<Y>(?:19\d{2}|%s\d))" % _getYearCentRE(cent=(0,3), distance=3, 
+			now=(MyTime.now(), datetime.datetime.fromtimestamp(978393600)))
+		timeRE['Exy'] = r"(?P<y>\d{2})"
+
+_updateTimeRE()
 
 def getTimePatternRE():
 	keys = timeRE.keys()
@@ -188,9 +205,9 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
 	"""
 
 	now = \
-	year = month = day = hour = minute = tzoffset = \
+	year = month = day = tzoffset = \
 	weekday = julian = week_of_year = None
-	second = fraction = 0
+	hour = minute = second = fraction = 0
 	for key, val in found_dict.iteritems():
 		if val is None: continue
 		# Directives not explicitly handled below:

fail2ban/tests/datedetectortestcase.py

@@ -551,6 +551,9 @@ class CustomDateFormatsTest(unittest.TestCase):
 			(1123970401.0,  "^%ExH:%ExM:%ExS**", '00:00:01'),
 			# cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004):
 			(1094068799.0,  "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'),
+			# no time (only date) in pattern, assume local 00:00:00 for H:M:S :
+			(1093989600.0,  "^%Y-%m-%d**",   '2004-09-01'),
+			(1093996800.0,  "^%Y-%m-%d%z**", '2004-09-01Z'),
 		):
 			logSys.debug('== test: %r', (matched, dp, line))
 			dd = DateDetector()

fail2ban/tests/files/logs/apache-overflows

@@ -3,6 +3,8 @@
 [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8
 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" }
 [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9
+# failJSON: { "time": "2010-03-15T16:04:06", "match": true , "host": "192.0.2.1", "desc": "AH00126 failure, gh-2908" }
+[Sat Mar 15 16:04:06.105212 2010] [core:error] [pid 17408] [client 192.0.2.1:55280] AH00126: Invalid URI in request GET /static/../../../a/../../../../etc/passwd HTTP/1.1
 
 # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488
 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" }