MRG/BF: merge from master. Fix bugs in iso8601

ChangeLog

@@ -50,6 +50,22 @@ code-review and minor additions from Yaroslav Halchenko.
 	 Some filters have been change as required to capture these elements in the
 	 right timezone correctly.
 
+ver. 0.8.12 (2013/12/XX) - things-can-only-get-better
+-----------
+
+- IMPORTANT incompatible changes:
+
+- Fixes:
+  - allow for ",milliseconds" in the custom date format of proftpd.log
+  - allow for ", referer ..." in apache-* filter for apache error logs.
+
+- New Features:
+
+  Daniel Black
+   * filter.d/solid-pop3d -- added thanks to Jacques Lav!gnotte on mailinglist.
+
+- Enhancements:
+
 ver. 0.8.11 (2013/11/13) - loves-unittests-and-tight-DoS-free-filter-regexes
 
 In light of CVE-2013-2178 that triggered our last release we have put

DEVELOP

@@ -805,6 +805,8 @@ Which indicates that testcases/files/logs/mysqld.log has been moved or is a dire
 
 # Provide a release sample to distributors
 
+  * Arch Linux:
+            https://www.archlinux.org/packages/community/any/fail2ban/
   * Debian: Yaroslav Halchenko <debian@onerussian.com>
             http://packages.qa.debian.org/f/fail2ban.html
   * FreeBSD: Christoph Theis theis@gmx.at>, Nick Hilliard <nick@foobar.org>
@@ -839,10 +841,15 @@ Which indicates that testcases/files/logs/mysqld.log has been moved or is a dire
   page: http://www.fail2ban.org/wiki/index.php/Commands
 
 * Update:
-  http://www.fail2ban.org/wiki/index.php/Downloads
+  http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_Versions&action=edit
+
+  http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_News&action=edit
+  move old bits to:
+  http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_OldNews&action=edit
+
+  http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_Versions&action=edit
   http://www.fail2ban.org/wiki/index.php/ChangeLog
   http://www.fail2ban.org/wiki/index.php/Requirements (Check requirement)
-  http://www.fail2ban.org/wiki/index.php/Main_Page (Add to News)
   http://www.fail2ban.org/wiki/index.php/Features
 
 * See if any filters are upgraded:

MANIFEST

@@ -99,6 +99,7 @@ fail2ban/tests/files/logs/proftpd
 fail2ban/tests/files/logs/pure-ftpd
 fail2ban/tests/files/logs/roundcube-auth
 fail2ban/tests/files/logs/sogo-auth
+fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/sshd
 fail2ban/tests/files/logs/sshd-ddos
 fail2ban/tests/files/logs/vsftpd
@@ -165,6 +166,7 @@ config/filter.d/pam-generic.conf
 config/filter.d/php-url-fopen.conf
 config/filter.d/postfix-sasl.conf
 config/filter.d/sieve.conf
+config/filter.d/solid-pop3d.conf
 config/filter.d/sshd.conf
 config/filter.d/sshd-ddos.conf
 config/filter.d/vsftpd.conf

THANKS

@@ -33,6 +33,7 @@ Georgiy Mernov
 Guillaume Delvit
 Hanno 'Rince' Wagner
 Iain Lea
+Jacques Lav!gnotte
 Jonathan Kamens
 Jonathan Lanning
 Jonathan Underwood
@@ -71,6 +72,7 @@ Tyler
 Vaclav Misek
 Vincent Deffontaines
 Yaroslav Halchenko
+Winston Smith
 ykimon
 Yehuda Katz
 zugeschmiert

config/filter.d/apache-auth.conf

@@ -10,19 +10,19 @@ before = apache-common.conf
 [Definition]
 
 
-failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*\s*$
+failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01617: )?user .* authentication failure for "\S*": Password Mismatch$
+            ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$
-            ^%(_apache_error_client)s (AH01618: )?user .* not found(: )?\S*\s*$
+            ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*\s*$
+            ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$
             ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$
-            ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*: password mismatch: \S*\s*$
+            ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*?: password mismatch: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*' in realm `.+' (not found|denied by provider): \S*\s*$
+            ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01631: )?user .*: authorization failure for "\S*":\s*$
+            ^%(_apache_error_client)s (AH01631: )?user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+\s*$
+            ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*' but expected `.+'\s*$
+            ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*' received: \S*\s*$
+            ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01793: )?invalid qop `.*' received: \S*\s*$
+            ^%(_apache_error_client)s (AH01793: )?invalid qop `.*?' received: \S*(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .* received - user attempted time travel\s*$
+            ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$
 
 ignoreregex = 
 
@@ -50,5 +50,7 @@ ignoreregex =
 #     ^%(_apache_error_client)s (AH01779: )?user .*: one-time-nonce mismatch - sending new nonce\s*$
 #     ^%(_apache_error_client)s (AH02486: )?realm mismatch - got `.*' but no realm specified\s*$
 #
+# referer is always in error log messages if it exists added as per the log_error_core function in server/log.c
+# 
 # Author: Cyril Jaquier
 # Major edits by Daniel Black

config/filter.d/apache-noscript.conf

@@ -9,8 +9,8 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)\s*$
+failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat\s*$
+            ^%(_apache_error_client)s script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat(, referer: \S+)?\s*$
 
 ignoreregex = 
 

config/filter.d/apache-overflows.conf

@@ -8,7 +8,7 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)$
+failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)(, referer: \S+)?$
 
 ignoreregex =
 

config/filter.d/solid-pop3d.conf

@@ -0,0 +1,32 @@
+# Fail2Ban filter for unsuccesful solid-pop3 authentication attempts
+#
+# Doesn't currently provide PAM support as PAM log messages don't include rhost as
+# remote IP.
+#
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+_daemon = solid-pop3d
+
+failregex = ^%(__prefix_line)sauthentication failed: (no such user|can't map user name): .*? - <HOST>$
+            ^%(__prefix_line)s(APOP )?authentication failed for (mapped )?user .*? - <HOST>$
+            ^%(__prefix_line)sroot login not allowed - <HOST>$
+            ^%(__prefix_line)scan't find APOP secret for user .*? - <HOST>$
+
+ignoreregex = 
+
+# DEV Notes:
+#
+# solid-pop3d needs to be compiled with --enable-logextend to support
+# IP addresses in log messages.
+#
+# solid-pop3d-0.15/src/main.c contains all authentication errors
+# except for PAM authentication messages ( src/authenticate.c )
+#
+# A pam authentication failure message (note no IP for rhost).
+# Nov 17 23:17:50 emf1pt2-2-35-70 solid-pop3d[17176]: pam_unix(solid-pop3d:auth): authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=  user=jacques
+# 
+# Authors: Daniel Black

config/jail.conf

@@ -493,6 +493,12 @@ port    = pop3,pop3s,imap,imaps,submission,smtps,sieve
 logpath = /var/log/secure
 
 
+[solid-pop3d]
+
+posrt  = pop3,pop3s
+logpath = /var/log/mail.log
+
+
 [exim]
 
 port   = smtp,ssmtp,submission

fail2ban/server/datedetector.py

@@ -61,6 +61,8 @@ class DateDetector:
 			self.appendTemplate("%a %b %d %H:%M:%S")
 			# standard: Jan 23 21:59:59 
 			self.appendTemplate("%b %d %H:%M:%S")
+			# proftpd date: 2005-01-23 21:59:59,333
+			self.appendTemplate("%Y-%m-%d %H:%M:%S,%f")
 			# simple date: 2005-01-23 21:59:59 
 			self.appendTemplate("%Y-%m-%d %H:%M:%S")
 			# simple date: 2005/01/23 21:59:59 

fail2ban/server/iso8601.py

@@ -32,7 +32,8 @@ datetime.datetime(2007, 1, 25, 12, 0, tzinfo=<iso8601.iso8601.Utc ...>)
 
 """
 
-from datetime import datetime, timedelta, tzinfo, time
+from datetime import datetime, timedelta, tzinfo
+import time
 import re
 
 __all__ = ["parse_date", "ParseError"]
@@ -92,7 +93,7 @@ def parse_timezone(tzstring):
 
     if tzstring is None:
         zone_sec = -time.timezone 
-        return FixedOffset(name=time.tzname[0],hours=(zone_sec / 3600),minutes=(zone_sec % 3600)/60,seconds=zone_sec % 60)
+        return FixedOffset(name=time.tzname[0],offset_hours=(zone_sec / 3600), offset_minutes=(zone_sec % 3600)/60, offset_seconds=zone_sec % 60)
 
     m = TIMEZONE_REGEX.match(tzstring)
     prefix, hours, minutes = m.groups()

fail2ban/tests/datedetectortestcase.py

@@ -81,6 +81,7 @@ class DateDetectorTest(unittest.TestCase):
 			(False, "23/Jan/2005:21:59:59 +0100"),
 			(False, "01/23/2005:21:59:59"),
 			(False, "2005-01-23 21:59:59"),
+		    (False, "2005-01-23 21:59:59,000"),	  # proftpd
 			(False, "23-Jan-2005 21:59:59"),
 			(False, "23-Jan-2005 21:59:59.02"),
 			(False, "23-Jan-2005 21:59:59 +0100"),

fail2ban/tests/files/logs/apache-auth

@@ -114,3 +114,6 @@
 
 # failJSON: { "time": "2013-06-01T02:17:42", "match": true , "host": "192.168.0.2" }
 [Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found
+
+# failJSON: { "time": "2013-11-18T22:39:33", "match": true , "host": "91.49.82.139" }
+[Mon Nov 18 22:39:33 2013] [error] [client 91.49.82.139] user gg not found: /, referer: http://sj.hopto.org/management.html

fail2ban/tests/files/logs/proftpd

@@ -14,3 +14,5 @@ Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.
 May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded 
 # failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" }
 Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 
+# failJSON: { "time": "2013-11-16T21:59:30", "match": true , "host": "1.2.3.4", "desc": "proftpd-basic 1.3.5~rc3-2.1 on Debian uses date format with milliseconds if logging under /var/log/proftpd/proftpd.log" }
+2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21

fail2ban/tests/files/logs/solid-pop3d

@@ -0,0 +1,25 @@
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: authentication failed: no such user: adrian - 123.33.44.45
+
+# All below are manufactured from looking at log
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: authentication failed: can't map user name: adrian - 123.33.44.45
+
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: authentication failed for user adrain - 123.33.44.45
+
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: authentication failed for mapped user adrain - 123.33.44.45
+
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: root login not allowed - 123.33.44.45
+
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: can't find APOP secret for user adrian - 123.33.44.45
+
+# failJSON: { "time": "2004-11-15T00:34:53", "match": true , "host": "123.33.44.45" }
+Nov 15 00:34:53 rmc1pt2-2-35-70 solid-pop3d[3822]: APOP authentication failed for user adrian - 123.33.44.45
+
+# Real log messages again:
+# failJSON: { "time": "2004-11-17T23:10:03", "match": true , "host": "190.16.165.230" }
+Nov 17 23:10:03 emf1pt2-2-35-70 solid-pop3d[16993]: authentication failed for user jacques - 190.16.165.230

fail2ban/tests/files/logs/suhosin

@@ -2,3 +2,9 @@
 Mar 11 22:52:12   lighttpd[53690]: (mod_fastcgi.c.2676) FastCGI-stderr: ALERT - configured request variable name length limit exceeded - dropped variable 'upqchi07vFfAFuBjnIKGIwiLrHo3Vt68T3yqvhQu2TqetQ78roy7Q6bpTfDUtYFR593/MA' (attacker '198.51.100.167', file '/usr/local/captiveportal/index.php')
 # failJSON: { "time": "2005-02-26T22:52:29", "match": true , "host": "198.51.100.77" }
 Feb 26 22:52:29 host suhosin[9636]: ALERT - script tried to increase memory_limit to 268435456 bytes which is above the allowed value (attacker '198.51.100.77', file '/var/www/wordpress/wp-admin/includes/image.php', line 161)
+
+# failJSON: { "time": "2004-11-18T20:18:31", "match": true , "host": "188.132.244.3" }
+Nov 18 20:18:31 platypus suhosin[28433]: ALERT - ASCII-NUL chars not allowed within request variables - dropped variable 'templatefile' (attacker '188.132.244.3', file '/home/ace-hosting/public_html/cart.php')
+
+# failJSON: { "time": "2004-10-25T10:59:49", "match": true , "host": "38.111.147.83" }
+Oct 25 10:59:49 platypus suhosin[13953]: ALERT - configured GET variable value length limit exceeded - dropped variable '_route_' (attacker '38.111.147.83', file '/home/thegoblin/public_html/index.php')