Merge branch '0.10' into 0.11

ChangeLog

@@ -41,6 +41,7 @@ ver. 0.11.0-dev-0 (2017/??/??) - development nightly edition
 ### Fixes
 * Fixed logging to systemd-journal: new logtarget value SYSOUT can be used instead of STDOUT, to avoid 
   write of the time-stamp, if logging to systemd-journal from foreground mode (gh-1876)
+* Fixed recognition of the new date-format on mysqld-auth filter (gh-1639)
 * jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely 
   (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942.
 * config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf)
@@ -53,6 +54,12 @@ ver. 0.11.0-dev-0 (2017/??/??) - development nightly edition
   see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
 
 ### New Features
+* datedetector: extended default date-patterns (allows extra space between the date and time stamps);
+  introduces 2 new format directives (with corresponding %Ex prefix for more precise parsing):
+  - %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
+    (corresponds %H, but allows space if not zero-padded).
+  - %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
+    (corresponds %I, but allows space if not zero-padded).
 * New Actions:
   - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in
     nginx-location with map-file);
@@ -456,7 +463,10 @@ releases.
 * filter.d/dovecot.conf:
     - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880)
     - extended to match pam_authenticate failures with "Permission denied" (gh-1897)
-* filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
+* filter.d/exim.conf
+    - fixed failregex for case of flood attempts with `D=0s` (gh-1887)
+    - fixed failregex of "AUTH command used when not advertised" to better handle the foreign
+      input SMTP command (lower/mixed case auth command, prevent injection) (gh-1979)
 * filter.d/postfix-*.conf - added optional port regex (gh-1902)
 
 ### New Features

config/filter.d/apache-shellshock.conf

@@ -25,4 +25,4 @@ ignoreregex =
 # [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt
 # [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST'
 #
-# Author: Eugene Hopkinson (riot@riot.so)
+# Author: Eugene Hopkinson (e.hopkinson@gmail.com)

config/filter.d/exim.conf

@@ -21,7 +21,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
             ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
-            ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$
+            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
 

fail2ban/server/datedetector.py

@@ -124,22 +124,22 @@ class DateDetectorCache(object):
 		"""
 		self.__tmpcache = [], []
 		# ISO 8601, simple date, optional subsecond and timezone:
-		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59
+		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23  8:59:59
 		# simple date: 2005/01/23 21:59:59 
 		# custom for syslog-ng 2006.12.21 06:43:20
-		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T|  ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
-		self._cacheTemplate("(?:%a )?%b %d %ExY %H:%M:%S(?:\.%f)?")
+		self._cacheTemplate("(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?")
 		# simple date too (from x11vnc): 23/01/2005 21:59:59 
 		# and with optional year given by 2 digits: 23/01/05 21:59:59 
 		# (See http://bugs.debian.org/537610)
 		# 17-07-2008 17:23:25
-		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
+		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
 		# Apache format optional time zone:
 		# [31/Oct/2006:09:22:55 -0000]
 		# 26-Jul-2007 15:20:52
@@ -150,8 +150,8 @@ class DateDetectorCache(object):
 		self._cacheTemplate("%m/%d/%ExY:%H:%M:%S")
 		# 01-27-2012 16:22:44.252
 		# subseconds explicit to avoid possible %m<->%d confusion
-		# with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S")
-		self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?")
+		# with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
+		self._cacheTemplate("%m-%d-%ExY %k:%M:%S(?:\.%f)?")
 		# Epoch
 		self._cacheTemplate('EPOCH')
 		# Only time information in the log
@@ -163,14 +163,14 @@ class DateDetectorCache(object):
 		# Apache Tomcat
 		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
-		# 20050123T215959, 20050123 215959
-		self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
+		self._cacheTemplate("^%b-%d-%Exy %k:%M:%S")
+		# 20050123T215959, 20050123 215959, 20050123  85959
+		self._cacheTemplate("%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
-		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# +00:00 Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
+		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
 		# TAI64N
 		self._cacheTemplate("TAI64N")
 		#

fail2ban/server/strptime.py

@@ -43,6 +43,13 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
 
 timeRE = TimeRE()
 
+# %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock,
+# (corresponds %H, but allows space if not zero-padded).
+# %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock,
+# (corresponds %I, but allows space if not zero-padded).
+timeRE['k'] = r" ?(?P<H>[0-2]?\d)"
+timeRE['l'] = r" ?(?P<I>1?\d)"
+
 # TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
 #       check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), 
 #       to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' 
@@ -61,6 +68,8 @@ timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABB
 timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])"
 timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])"
 timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)"
+timeRE['Exk'] = r" ?(?P<H>2[0-3]|[0-1]\d|\d)"
+timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)"
 timeRE['ExM'] = r"(?P<M>[0-5]\d)"
 timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)"
 # more precise year patterns, within same century of last year and

fail2ban/tests/files/logs/exim

@@ -76,3 +76,10 @@
 2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com)
 # failJSON: { "time": "2017-05-01T07:42:42", "match": true , "host": "192.0.2.3", "desc": "rejected RCPT - Unrouteable address" }
 2017-05-01 07:42:42 H=some.rev.dns.if.found (the.connector.reports.this.name) [192.0.2.3] F=<some.name@some.domain> rejected RCPT <some.invalid.name@a.domain>: Unrouteable address
+
+# failJSON: { "time": "2017-11-28T14:14:30", "match": true , "host": "192.0.2.4", "desc": "lower case AUTH command" }
+2017-11-28 14:14:30 SMTP protocol error in "auth login" H=(roxzgj) [192.0.2.4] AUTH command used when not advertised
+# failJSON: { "time": "2017-11-28T14:14:31", "match": true , "host": "192.0.2.5", "desc": "mixed case AUTH command" }
+2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised
+# failJSON: { "time": "2017-11-28T14:14:32", "match": true , "host": "192.0.2.6", "desc": "quoted injecting on AUTH command" }
+2017-11-28 14:14:32 SMTP protocol error in "aUtH lOgIn" H=(test) [8.8.8.8]" H=(roxzgj) [192.0.2.6] AUTH command used when not advertised

fail2ban/tests/files/logs/mysqld-auth

@@ -17,6 +17,9 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for
 
 # failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" }
 2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES)
+# failJSON: { "time": "2015-10-07T08:25:38", "match": true , "host": "192.0.2.1", "desc": "log format with extra space between the date and time stamps, gh-1639" }
+2015-10-07  8:25:38 139845235329792 [Warning] Access denied for user 'user'@'192.0.2.1' (using password: YES)
+
 # failJSON: { "time": "2016-02-24T15:26:18", "match": true , "host": "localhost", "desc": "mysql 5.6 log format, Note instead of Warning" }
 2016-02-24T15:26:18.237955 6 [Note] Access denied for user 'root'@'localhost' (using password: YES)