Merge branch 'master' into 0.10 (resolve conflicts and cleaning tree points after back-porting gh-1508 0.10 -> 0.9)

.travis.yml

@@ -41,6 +41,8 @@ script:
   - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi
   # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
   - sudo $VENV_BIN/pip install .
+  # Doc files should get installed on Travis under Linux
+  - test -e /usr/share/doc/fail2ban/FILTERS
 after_success:
   - coveralls
   - codecov

ChangeLog

@@ -64,13 +64,54 @@ TODO: implementing of options resp. other tasks from PR #1346
   syntax `[Section?family=inet6]` (currently use for IPv6-support only).
 
 
-ver. 0.9.5 (2016/07/15) - old-not-obsolete
+ver. 0.9.6 (2016/XX/XX) - wanna-be-released
 -----------
 
 0.9.x line is no longer heavily developed.  If you are interested in
 new features (e.g. IPv6 support), please consider 0.10 branch and its
 releases.
 
+### Fixes
+* Misleading add resp. enable of (already available) jail in database, that
+  induced a subsequent error: last position of log file will be never retrieved (gh-795)
+* Fixed a distribution related bug within testReadStockJailConfForceEnabled
+  (e.g. test-cases faults on Fedora, see gh-1353)
+* Fixed pythonic filters and test scripts (running via wrong python version, 
+  uses "fail2ban-python" now);
+* Fixed test case "testSetupInstallRoot" for not default python version (also
+  using direct call, out of virtualenv);
+* `filter.d/ignorecommands/apache-fakegooglebot`
+    - Fixed error within apache-fakegooglebot, that will be called 
+      with wrong python version (gh-1506)
+* `filter.d/assp.conf`
+    - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494)
+* `filter.d/postfix-sasl.conf`
+    - Allow for having no trailing space after 'failed:' (gh-1497)
+
+### New Features
+
+### Enhancements
+* Introduces new command "fail2ban-python", as automatically created symlink to 
+  python executable, where fail2ban currently installed (resp. its modules are located):
+    - allows to use the same version, fail2ban currently running, e.g. in 
+      external scripts just via replace python with fail2ban-python:
+      ```diff
+      -#!/usr/bin/env python
+      +#!/usr/bin/env fail2ban-python
+      ```
+    - always the same pickle protocol
+    - the same (and also guaranteed available) fail2ban modules
+    - simplified stand-alone install, resp. stand-alone installation possibility
+      via setup (like gh-1487) is getting closer
+* Several test cases rewritten using new methods assertIn, assertNotIn
+* New forward compatibility method assertRaisesRegexp (normally python >= 2.7).
+  Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged
+  are test covered now
+
+
+ver. 0.9.5 (2016/07/15) - old-not-obsolete
+-----------
+
 ### Fixes
 * `filter.d/monit.conf`
     - Extended failregex with new monit "access denied" version (gh-1355)

config/action.d/badips.conf

@@ -1,6 +1,6 @@
 # Fail2ban reporting to badips.com
 #
-# Note: This reports and IP only and does not actually ban traffic. Use 
+# Note: This reports an IP only and does not actually ban traffic. Use
 # another action in the same jail if you want bans to occur.
 #
 # Set the category to the appropriate value before use.

config/filter.d/assp.conf

@@ -1,24 +1,43 @@
-# Fail2Ban filter for Anti-Spam SMTP Proxy Server also known as ASSP
+# Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP)
+#    Filter works in theory for both ASSP V1 and V2. Recommended ASSP is V2.5.1 or later.
+#    Support for ASSP V1 ended in 2014 so if you are still running ASSP V1 an immediate upgrade is recommended.
 #
-#    Honmepage:   http://www.magicvillage.de/~Fritz_Borgstedt/assp/0003D91C-8000001C/
+#    Homepage:    http://sourceforge.net/projects/assp/
-#    ProjektSite: http://sourceforge.net/projects/assp/?source=directory
+#    ProjectSite: http://sourceforge.net/projects/assp/?source=directory
 #
 #
 
 [Definition] 
+# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibilty reasons.
 
 __assp_actions = (?:dropping|refusing)
 
 failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$
 			^(?: \[SSL-out\])? <HOST> SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$
 			^ Blocking <HOST> - too much AUTH errors \(\d{,3}\);$
+			^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)*<HOST> (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))?: \S+$
+			^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)*<HOST> \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)?$
 
 ignoreregex = 
 
 # DEV Notes:
+# V1 Examples matches:
+#   Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);
+#   Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+#   Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded 
 #
-# Examples: Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);
+# V2 Examples matches:
-#           Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+#   Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user@example.org relay attempt blocked for: someone@example.org
-#           Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded 
+#   Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+#   Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+#   Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed:
+#   Jan-05-16 08:38:49 m1-01129-09140 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> relay attempt blocked for (parsing): <user2@example>
+#   Jun-12-16 16:43:37 m1-64217-12013 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user2@example.com relay attempt blocked for (parsing): <a.notheruser69@example.c>
+#   Jan-22-16 22:25:51 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid authentication mechanism
+#   Mar-19-16 13:42:20 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid base64 data in continued response
+#   Jul-18-16 16:54:21 [Worker_2] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server
+#   Jul-18-16 17:14:23 m1-76453-02949 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server
+
 #
 # Author: Enrico Labedzki (enrico.labedzki@deiwos.de)
+# V2 Filters: Robert Hardy (rhardy@webcon.ca)

fail2ban/tests/files/logs/assp

@@ -22,4 +22,23 @@ Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5)
 Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;
 # failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;
-
+# failJSON: { "time": "2016-07-29T16:49:52", "match": true , "host": "0.0.0.0" }
+Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user@example.org relay attempt blocked for: someone@example.org
+# failJSON: { "time": "2016-07-30T17:07:25", "match": true , "host": "0.0.0.0" }
+Jul-30-16 17:07:25 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+# failJSON: { "time": "2016-07-30T17:11:05", "match": true , "host": "0.0.0.0" }
+Jul-30-16 17:11:05 m1-13060-05386 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6
+# failJSON: { "time": "2016-07-31T06:45:59", "match": true , "host": "0.0.0.0" }
+Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: 
+# failJSON: { "time": "2016-01-05T08:38:49", "match": true , "host": "0.0.0.0" }
+Jan-05-16 08:38:49 m1-01129-09140 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> relay attempt blocked for (parsing): <user2@example>
+# failJSON: { "time": "2016-06-12T16:43:37", "match": true , "host": "0.0.0.0" }
+Jun-12-16 16:43:37 m1-64217-12013 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 <user@example.com> to: user2@example.com relay attempt blocked for (parsing): <a.notheruser69@example.c>
+# failJSON: { "time": "2016-01-22T22:25:51", "match": true , "host": "0.0.0.0" }
+Jan-22-16 22:25:51 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid authentication mechanism
+# failJSON: { "time": "2016-03-19T13:42:20", "match": true , "host": "0.0.0.0" }
+Mar-19-16 13:42:20 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid base64 data in continued response
+# failJSON: { "time": "2016-07-18T16:54:21", "match": true , "host": "0.0.0.0" }
+Jul-18-16 16:54:21 [Worker_2] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server
+# failJSON: { "time": "2016-07-18T17:14:23", "match": true , "host": "0.0.0.0" }
+Jul-18-16 17:14:23 m1-76453-02949 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server

setup.py

@@ -19,9 +19,11 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 __author__ = "Cyril Jaquier, Steven Hiscocks, Yaroslav Halchenko"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2013 Fail2Ban Contributors"
+__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2016 Fail2Ban Contributors"
 __license__ = "GPL"
 
+import platform
+
 try:
 	import setuptools
 	from setuptools import setup
@@ -113,6 +115,18 @@ if os.path.exists('/var/run'):
 	# realpath is used to possibly resolve /var/run -> /run symlink
 	data_files_extra += [(realpath('/var/run/fail2ban'), '')]
 
+# Installing documentation files only under Linux or other GNU/ systems
+# (e.g. GNU/kFreeBSD), since others might have protective mechanisms forbidding
+# installation there (see e.g. #1233)
+platform_system = platform.system().lower()
+doc_files = ['README.md', 'DEVELOP', 'FILTERS', 'doc/run-rootless.txt']
+if platform_system in ('solaris', 'sunos'):
+	doc_files.append('README.Solaris')
+if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswith('gnu'):
+	data_files_extra.append(
+		('/usr/share/doc/fail2ban', doc_files)
+	)
+
 # Get version number, avoiding importing fail2ban.
 # This is due to tests not functioning for python3 as 2to3 takes place later
 exec(open(join("fail2ban", "version.py")).read())
@@ -180,10 +194,6 @@ setup(
 		('/var/lib/fail2ban',
 			''
 		),
-		('/usr/share/doc/fail2ban',
-			['README.md', 'README.Solaris', 'DEVELOP', 'FILTERS',
-			 'doc/run-rootless.txt']
-		)
 	] + data_files_extra,
 	**setup_extra
 )