From 660d57e6ba7bfa2dd6a16fdb6f5f9c27eecb335b Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Wed, 29 Nov 2017 04:58:00 +0000 Subject: [PATCH 1/8] updating my email address --- config/filter.d/apache-shellshock.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/apache-shellshock.conf b/config/filter.d/apache-shellshock.conf index 39df1704..725859a2 100644 --- a/config/filter.d/apache-shellshock.conf +++ b/config/filter.d/apache-shellshock.conf @@ -23,4 +23,4 @@ ignoreregex = # [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt # [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST' # -# Author: Eugene Hopkinson (riot@riot.so) +# Author: Eugene Hopkinson (e.hopkinson@gmail.com) From aa158ac05fc8bfcd21c5ba8dac2f77934dc8fbcb Mon Sep 17 00:00:00 2001 From: Peter Nowee Date: Wed, 29 Nov 2017 14:07:00 +0100 Subject: [PATCH 2/8] Exim failregex: Include lower/mixed case AUTH When reporting the error `AUTH command used when not advertised`, Exim starts with `SMTP protocol error in "........."`. Here, Exim logs the SMTP command as it was provided by the connecting client. https://github.com/Exim/exim/blob/exim-4_89+fixes/src/src/smtp_in.c#L2850 According to RFC 5321 (SMTP) "[..] a command verb [..] MAY be encoded in upper case, lower case, or any mixture of upper and lower case with no impact on its meaning." https://tools.ietf.org/html/rfc5321#section-2.4 Lower case `auth login` brute-force attempts were seen in the wild and were not caught by the current failregex. This commit makes the failregex case-insensitive for the `AUTH` command, so that lower case (`auth`) or mixed case (`aUtH`) now also match. The failregex was already case-insensitive for the command arguments (e.g. `AUTH login` already matched). --- config/filter.d/exim.conf | 2 +- fail2ban/tests/files/logs/exim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index eb56feff..c195882a 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user| ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ - ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ + ^%(pid)s SMTP protocol error in "[Aa][Uu][Tt][Hh] \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$ ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index f87d393b..e97fd7aa 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -76,3 +76,8 @@ 2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com) # failJSON: { "time": "2017-05-01T07:42:42", "match": true , "host": "192.0.2.3", "desc": "rejected RCPT - Unrouteable address" } 2017-05-01 07:42:42 H=some.rev.dns.if.found (the.connector.reports.this.name) [192.0.2.3] F= rejected RCPT : Unrouteable address + +# failJSON: { "time": "2017-11-28T14:14:30", "match": true , "host": "192.0.2.4", "desc": "lower case AUTH command" } +2017-11-28 14:14:30 SMTP protocol error in "auth login" H=(roxzgj) [192.0.2.4] AUTH command used when not advertised +# failJSON: { "time": "2017-11-28T14:14:31", "match": true , "host": "192.0.2.5", "desc": "mixed case AUTH command" } +2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised From f59df2e156a140dce34a94fb146f4b4655e936b6 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 29 Nov 2017 20:55:48 +0100 Subject: [PATCH 3/8] Avoid any injecting on protocol (e. g. tries using camel-case) The phrase "AUTH command used when not advertised" is precise enough as anchor here, so prevent by any foreign-input (any auth protocol error). --- config/filter.d/exim.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index c195882a..5e269f65 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user| ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ - ^%(pid)s SMTP protocol error in "[Aa][Uu][Tt][Hh] \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ + ^%(pid)s SMTP protocol error in "[^"]+" %(host_info)sAUTH command used when not advertised\s*$ ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$ ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ From 4f631806110ee8bdb760b78da5e52632cff3aed8 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 30 Nov 2017 12:32:24 +0100 Subject: [PATCH 4/8] Avoid injection using quotes after `auth` command; Added non-greedy fallback for quoted something (with lookahead simulated possessive greedy catch of non-quoted parts `[^"]*(?=")`). Note that because host-info's are hereafter (with foreign input in-between), we would not use greedy or non-greedy catch-alls (`.*` or `.*?`) here (preventing performance losses). --- config/filter.d/exim.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 5e269f65..21fcaf73 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -18,7 +18,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user| ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ - ^%(pid)s SMTP protocol error in "[^"]+" %(host_info)sAUTH command used when not advertised\s*$ + ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$ ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$ ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ From cbd63d9cd545bf3e389d7f9510b83ad177410a8d Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 30 Nov 2017 12:45:11 +0100 Subject: [PATCH 5/8] added test to cover quoted injecting on AUTH command --- fail2ban/tests/files/logs/exim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index e97fd7aa..97786f4e 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -81,3 +81,5 @@ 2017-11-28 14:14:30 SMTP protocol error in "auth login" H=(roxzgj) [192.0.2.4] AUTH command used when not advertised # failJSON: { "time": "2017-11-28T14:14:31", "match": true , "host": "192.0.2.5", "desc": "mixed case AUTH command" } 2017-11-28 14:14:31 SMTP protocol error in "aUtH lOgIn" H=(roxzgj) [192.0.2.5] AUTH command used when not advertised +# failJSON: { "time": "2017-11-28T14:14:32", "match": true , "host": "192.0.2.6", "desc": "quoted injecting on AUTH command" } +2017-11-28 14:14:32 SMTP protocol error in "aUtH lOgIn" H=(test) [8.8.8.8]" H=(roxzgj) [192.0.2.6] AUTH command used when not advertised From 2e437937c36d7ca5a7d2a4740c2270d4e89d50ef Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 30 Nov 2017 17:06:37 +0100 Subject: [PATCH 6/8] datedetector: extended default date-patterns (allows extra space between the date and time stamps); * introduces 2 new format directives (with corresponding `%Ex` prefix for more precise parsing): - %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, (corresponds %H, but allows space if not zero-padded). - %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock, (corresponds %I, but allows space if not zero-padded). * mysqld-auth test extended to cover new date-format in log. Closes gh-1639 --- fail2ban/server/datedetector.py | 24 ++++++++++++------------ fail2ban/server/strptime.py | 9 +++++++++ fail2ban/tests/files/logs/mysqld-auth | 3 +++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 90cfe1fd..278b2f20 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -124,22 +124,22 @@ class DateDetectorCache(object): """ self.__tmpcache = [], [] # ISO 8601, simple date, optional subsecond and timezone: - # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59 + # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23 8:59:59 # simple date: 2005/01/23 21:59:59 # custom for syslog-ng 2006.12.21 06:43:20 - self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") + self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T| ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?") # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - self._cacheTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?") # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 - self._cacheTemplate("(?:%a )?%b %d %ExY %H:%M:%S(?:\.%f)?") + self._cacheTemplate("(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?") # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) # 17-07-2008 17:23:25 - self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") + self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S") # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] # 26-Jul-2007 15:20:52 @@ -150,8 +150,8 @@ class DateDetectorCache(object): self._cacheTemplate("%m/%d/%ExY:%H:%M:%S") # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion - # with previous ("%d-%m-%ExY %H:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %H:%M:%S") - self._cacheTemplate("%m-%d-%ExY %H:%M:%S(?:\.%f)?") + # with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S") + self._cacheTemplate("%m-%d-%ExY %k:%M:%S(?:\.%f)?") # Epoch self._cacheTemplate('EPOCH') # Only time information in the log @@ -163,14 +163,14 @@ class DateDetectorCache(object): # Apache Tomcat self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 - self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") - # 20050123T215959, 20050123 215959 - self._cacheTemplate("%ExY%Exm%Exd[T ]%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") + self._cacheTemplate("^%b-%d-%Exy %k:%M:%S") + # 20050123T215959, 20050123 215959, , 20050123 85959 + self._cacheTemplate("%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 - self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?") # +00:00 Jan 23 21:59:59.011 2005 - self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") + self._cacheTemplate("(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?") # TAI64N self._cacheTemplate("TAI64N") # diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index dbf75d21..498d284b 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -43,6 +43,13 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo timeRE = TimeRE() +# %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, +# (corresponds %H, but allows space if not zero-padded). +# %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock, +# (corresponds %I, but allows space if not zero-padded). +timeRE['k'] = r" ?(?P[0-2]?\d)" +timeRE['l'] = r" ?(?P1?\d)" + # TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching, # check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), # to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' @@ -61,6 +68,8 @@ timeRE['Exz'] = r"(?P(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABB timeRE['Exd'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" timeRE['Exm'] = r"(?P1[0-2]|0[1-9])" timeRE['ExH'] = r"(?P2[0-3]|[0-1]\d)" +timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" +timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" # more precise year patterns, within same century of last year and diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index c77d214d..ebb8c0c4 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -17,6 +17,9 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for # failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" } 2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES) +# failJSON: { "time": "2015-10-07T08:25:38", "match": true , "host": "192.0.2.1", "desc": "log format with extra space between the date and time stamps, gh-1639" } +2015-10-07 8:25:38 139845235329792 [Warning] Access denied for user 'user'@'192.0.2.1' (using password: YES) + # failJSON: { "time": "2016-02-24T15:26:18", "match": true , "host": "localhost", "desc": "mysql 5.6 log format, Note instead of Warning" } 2016-02-24T15:26:18.237955 6 [Note] Access denied for user 'root'@'localhost' (using password: YES) From 55476974016103e92c23071fee9fa1098acbf318 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 1 Dec 2017 10:16:14 +0100 Subject: [PATCH 7/8] ChangeLog and typo --- ChangeLog | 7 +++++++ fail2ban/server/datedetector.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 4a4eee9c..d3ee6d02 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,7 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition ### Fixes * Fixed logging to systemd-journal: new logtarget value SYSOUT can be used instead of STDOUT, to avoid write of the time-stamp, if logging to systemd-journal from foreground mode (gh-1876) +* Fixed recognition of the new date-format on mysqld-auth filter (gh-1639) * jail.conf: port `imap3` replaced with `imap` everywhere, since imap3 is not a standard port and old rarely (if ever) used and can missing on some systems (e. g. debian stretch), see gh-1942. * config/paths-common.conf: added missing initial values (and small normalization in config/paths-*.conf) @@ -53,6 +54,12 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944); ### New Features +* datedetector: extended default date-patterns (allows extra space between the date and time stamps); + introduces 2 new format directives (with corresponding %Ex prefix for more precise parsing): + - %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, + (corresponds %H, but allows space if not zero-padded). + - %l - one- or two-digit number giving the hour of the day (12-11) on a 12-hour clock, + (corresponds %I, but allows space if not zero-padded). * New Actions: - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in nginx-location with map-file); diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 278b2f20..42df308e 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -164,7 +164,7 @@ class DateDetectorCache(object): self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") # ASSP: Apr-27-13 02:33:06 self._cacheTemplate("^%b-%d-%Exy %k:%M:%S") - # 20050123T215959, 20050123 215959, , 20050123 85959 + # 20050123T215959, 20050123 215959, 20050123 85959 self._cacheTemplate("%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?") # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 From e4bbaf3d58e78cb30b8a990129a905920d6b47b8 Mon Sep 17 00:00:00 2001 From: Peter Nowee Date: Fri, 1 Dec 2017 15:01:48 +0100 Subject: [PATCH 8/8] Update ChangeLog --- ChangeLog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f3868e4b..1010d6d7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,7 +33,10 @@ releases. * filter.d/dovecot.conf: - fixed failregex, see gh-1879 (partially cherry-picked from gh-1880) - extended to match pam_authenticate failures with "Permission denied" (gh-1897) -* filter.d/exim.conf - fixed failregex for case of flood attempts with `D=0s` (gh-1887) +* filter.d/exim.conf + - fixed failregex for case of flood attempts with `D=0s` (gh-1887) + - fixed failregex of "AUTH command used when not advertised" to better handle the foreign + input SMTP command (lower/mixed case auth command, prevent injection) (gh-1979) * filter.d/postfix-*.conf - added optional port regex (gh-1902) ### New Features