Back-port pull request #1508 from sebres/_fix/python-executable-gh-1506 to master (0.10 -> 0.9)

ChangeLog

@@ -18,6 +18,13 @@ releases.
   induced a subsequent error: last position of log file will be never retrieved (gh-795)
 * Fixed a distribution related bug within testReadStockJailConfForceEnabled
   (e.g. test-cases faults on Fedora, see gh-1353)
+* Fixed pythonic filters and test scripts (running via wrong python version, 
+  uses "fail2ban-python" now);
+* Fixed test case "testSetupInstallRoot" for not default python version (also
+  using direct call, out of virtualenv);
+* `filter.d/ignorecommands/apache-fakegooglebot`
+    - Fixed error within apache-fakegooglebot, that will be called 
+      with wrong python version (gh-1506)
 * `filter.d/assp.conf`
     - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494)
 * `filter.d/postfix-sasl.conf`
@@ -26,6 +33,18 @@ releases.
 ### New Features
 
 ### Enhancements
+* Introduces new command "fail2ban-python", as automatically created symlink to 
+  python executable, where fail2ban currently installed (resp. its modules are located):
+    - allows to use the same version, fail2ban currently running, e.g. in 
+      external scripts just via replace python with fail2ban-python:
+      ```diff
+      -#!/usr/bin/env python
+      +#!/usr/bin/env fail2ban-python
+      ```
+    - always the same pickle protocol
+    - the same (and also guaranteed available) fail2ban modules
+    - simplified stand-alone install, resp. stand-alone installation possibility
+      via setup (like gh-1487) is getting closer
 * Several test cases rewritten using new methods assertIn, assertNotIn
 * New forward compatibility method assertRaisesRegexp (normally python >= 2.7).
   Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged

MANIFEST

@@ -194,6 +194,7 @@ fail2ban/server/server.py
 fail2ban/server/strptime.py
 fail2ban/server/ticket.py
 fail2ban/server/transmitter.py
+fail2ban/setup.py
 fail2ban-testcases-all
 fail2ban-testcases-all-python3
 fail2ban/tests/action_d/__init__.py

bin/fail2ban-testcases

@@ -39,10 +39,18 @@ from fail2ban.version import version
 
 from fail2ban.tests.utils import gatherTests
 from fail2ban.helpers import FormatterWithTraceBack, getLogger
+from fail2ban.setup import updatePyExec
 from fail2ban.server.mytime import MyTime
 
 from optparse import OptionParser, Option
 
+# Update fail2ban-python env to current python version (where f2b-modules located/installed)
+bindir = os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'fail2ban-testcases' else __file__
+)
+updatePyExec(bindir)
+
 def get_opt_parser():
 	# use module docstring for help output
 	p = OptionParser(

config/filter.d/ignorecommands/apache-fakegooglebot

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/
 #
 # Written in Python to reuse built-in Python batteries and not depend on

fail2ban/setup.py

@@ -0,0 +1,42 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Serg G. Brester"
+__license__ = "GPL"
+
+import os
+import sys
+
+
+def updatePyExec(bindir, executable=None):
+	"""Update fail2ban-python link to current python version (where f2b-modules located/installed)
+	"""
+	bindir = os.path.realpath(bindir)
+	if executable is None:
+		executable = sys.executable
+	pypath = os.path.join(bindir, 'fail2ban-python')
+	# if not exists or point to another version - update link:
+	isfile = os.path.isfile(pypath)
+	if not isfile or os.path.realpath(pypath) != os.path.realpath(executable):
+		if isfile:
+			os.unlink(pypath)
+		os.symlink(executable, pypath)
+	# extend current environment path (e.g. if fail2ban not yet installed):
+	if bindir not in os.environ["PATH"].split(os.pathsep):
+		os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir;

fail2ban/tests/files/config/apache-auth/digest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 import requests
 
 try:

fail2ban/tests/files/ignorecommand.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env fail2ban-python
 import sys
 if sys.argv[1] == "10.0.0.1":
 	exit(0)

fail2ban/tests/misctestcase.py

@@ -70,6 +70,22 @@ class HelpersTest(unittest.TestCase):
 		self.assertEqual(splitwords(' 1\n  2, 3'), ['1', '2', '3'])
 
 
+if sys.version_info >= (2,7):
+	def _sh_call(cmd):
+		import subprocess, locale
+		ret = subprocess.check_output(cmd, shell=True)
+		if sys.version_info >= (3,):
+			ret = ret.decode(locale.getpreferredencoding(), 'replace')
+		return str(ret).rstrip()
+else:
+	def _sh_call(cmd):
+		import subprocess
+		ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
+		return str(ret).rstrip()
+
+def _getSysPythonVersion():
+	return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'")
+
 class SetupTest(unittest.TestCase):
 
 	def setUp(self):
@@ -79,6 +95,12 @@ class SetupTest(unittest.TestCase):
 			raise unittest.SkipTest(
 				"Seems to be running not out of source distribution"
 				" -- cannot locate setup.py")
+		# compare current version of python installed resp. active one:
+		sysVer = _getSysPythonVersion()
+		if sysVer != str(tuple(sys.version_info)):
+			raise unittest.SkipTest(
+				"Seems to be running with python distribution %s"
+				" -- install can be tested only with system distribution %s" % (str(tuple(sys.version_info)), sysVer))
 
 	def testSetupInstallRoot(self):
 		if not self.setup:
@@ -125,6 +147,14 @@ class SetupTest(unittest.TestCase):
 					  'etc/fail2ban/jail.conf'):
 				self.assertTrue(os.path.exists(os.path.join(tmp, f)),
 								msg="Can't find %s" % f)
+			# Because the install (test) path in virtual-env differs from some development-env,
+			# it is not a `tmp + '/usr/local/bin/'`, so search for it:
+			installedPath = _sh_call('find ' + tmp+ ' -name fail2ban-python').split('\n')
+			self.assertTrue(len(installedPath) > 0)
+			for installedPath in installedPath:
+				self.assertEqual(
+					os.path.realpath(installedPath), os.path.realpath(sys.executable))
+
 		finally:
 			# clean up
 			shutil.rmtree(tmp)

setup.py

@@ -40,12 +40,40 @@ except ImportError:
 	# python 2.x
 	from distutils.command.build_py import build_py
 	from distutils.command.build_scripts import build_scripts
+# all versions
+from distutils.command.install_scripts import install_scripts
+
 import os
 from os.path import isfile, join, isdir, realpath
 import sys
 import warnings
 from glob import glob
 
+from fail2ban.setup import updatePyExec
+
+
+# Wrapper to install python binding (to current python version):
+class install_scripts_f2b(install_scripts):
+
+	def get_outputs(self):
+		outputs = install_scripts.get_outputs(self)
+		fn = None
+		for fn in outputs:
+			if os.path.basename(fn) == 'fail2ban-server':
+				break
+		bindir = os.path.dirname(fn)
+		print('creating fail2ban-python binding -> %s' % (bindir,))
+		updatePyExec(bindir)
+		return outputs
+
+
+# Update fail2ban-python env to current python version (where f2b-modules located/installed)
+rootdir = os.path.realpath(os.path.dirname(
+	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
+	sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__
+))
+updatePyExec(os.path.join(rootdir, 'bin'))
+
 if setuptools and "test" in sys.argv:
 	import logging
 	logSys = logging.getLogger("fail2ban")
@@ -113,12 +141,16 @@ setup(
 	url = "http://www.fail2ban.org",
 	license = "GPL",
 	platforms = "Posix",
-	cmdclass = {'build_py': build_py, 'build_scripts': build_scripts},
+	cmdclass = {
+		'build_py': build_py, 'build_scripts': build_scripts, 
+		'install_scripts': install_scripts_f2b
+	},
 	scripts = [
 		'bin/fail2ban-client',
 		'bin/fail2ban-server',
 		'bin/fail2ban-regex',
 		'bin/fail2ban-testcases',
+		# 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper
 	],
 	packages = [
 		'fail2ban',