Merge branch 'gh-actions--test-systemd' into 0.10 + several revisions cherry-picked from 0.11/master

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-rc.2', pypy2, pypy3]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3]
       fail-fast: false
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -40,33 +40,56 @@ jobs:
       - name: Python version
         run: |
           F2B_PY=$(python -c "import sys; print(sys.version)")
-          echo "Python: ${{ matrix.python-version }} -- $F2B_PY"
+          echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }"
+          F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)')
           F2B_PY=${F2B_PY:0:1}
-          echo "Set F2B_PY=$F2B_PY"
+          echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV"
           echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV
+          echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV
+          # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4):
+          echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV
 
       - name: Install dependencies
         run: |
-          #python -m pip install --upgrade pip
+          if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi
           if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
             #pip install 2to3
             sudo apt-get -y install 2to3
           fi
-          #pip install systemd-python || echo 'systemd not available'
+          #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available'
-          #pip install pyinotify || echo 'inotify not available'
+          python -m pip install pyinotify || echo 'inotify not available'
-          sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
+          #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
-          sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available'
+          sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable'
+          python -m pip install systemd-python || echo 'systemd not available'
 
       - name: Before scripts
         run: |
           cd "$GITHUB_WORKSPACE"
           # Manually execute 2to3 for now
           if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi
+          _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; }
           # (debug) output current preferred encoding:
-          python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
+          _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
+          # (debug) backend availabilities:
+          echo 'Backends:'
+          _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd'
+          _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify'
 
       - name: Test suite
-        run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi
+        run: |
+          if [[ "$F2B_PY" = 2 ]]; then
+            python setup.py test
+          elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then 
+            python bin/fail2ban-testcases --verbosity=2
+          else
+            echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..."
+            python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"
+          fi
+
+      #- name: Test suite (debug some systemd tests only)
+         #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal"
+         #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar"
 
       - name: Build
         run: python setup.py build

fail2ban/server/filtersystemd.py

@@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
 import datetime
+import os
 import time
 from distutils.version import LooseVersion
 
@@ -93,7 +94,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		except KeyError:
 			# be sure all journal types will be opened if files/path specified (don't set flags):
 			if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']):
-				args['flags'] = 4
+				args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4))
 				
 		try:
 			args['namespace'] = kwargs.pop('namespace')

fail2ban/tests/misctestcase.py

@@ -111,7 +111,7 @@ class SetupTest(unittest.TestCase):
 		supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
 			# try dry-run:
-			os.system("%s %s --dry-run install --disable-2to3 --root=%s%s"
+			os.system("%s %s --dry-run install --root=%s%s"
 					  % (sys.executable, self.setup , tmp, supdbgout))
 			# check nothing was created:
 			self.assertTrue(not os.listdir(tmp))
@@ -127,7 +127,7 @@ class SetupTest(unittest.TestCase):
 		# suppress stdout (and stderr) if not heavydebug
 		supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG
 		try:
-			self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s"
+			self.assertEqual(os.system("%s %s install --root=%s%s"
 					  % (sys.executable, self.setup, tmp, supdbgout)), 0)
 
 			def strippath(l):

setup.py

@@ -48,7 +48,7 @@ import warnings
 from glob import glob
 
 from fail2ban.setup import updatePyExec
-
+from fail2ban.version import version
 
 source_dir = os.path.realpath(os.path.dirname(
 	# __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.):
@@ -112,22 +112,12 @@ class install_scripts_f2b(install_scripts):
 # Wrapper to specify fail2ban own options:
 class install_command_f2b(install):
 	user_options = install.user_options + [
-		('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'),
 		('without-tests', None, 'without tests files installation'),
 	]
 	def initialize_options(self):
-		self.disable_2to3 = None
 		self.without_tests = not with_tests
 		install.initialize_options(self)
 	def finalize_options(self):
-		global _2to3
-		## in the test cases 2to3 should be already done (fail2ban-2to3):
-		if self.disable_2to3:
-			_2to3 = False
-		if _2to3:
-			cmdclass = self.distribution.cmdclass
-			cmdclass['build_py'] = build_py_2to3
-			cmdclass['build_scripts'] = build_scripts_2to3
 		if self.without_tests:
 			self.distribution.scripts.remove('bin/fail2ban-testcases')
 
@@ -178,7 +168,6 @@ commands.'''
 if setuptools:
 	setup_extra = {
 		'test_suite': "fail2ban.tests.utils.gatherTests",
-		'use_2to3': True,
 	}
 else:
 	setup_extra = {}
@@ -202,9 +191,6 @@ if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswit
 		('/usr/share/doc/fail2ban', doc_files)
 	)
 
-# Get version number, avoiding importing fail2ban.
-# This is due to tests not functioning for python3 as 2to3 takes place later
-exec(open(join("fail2ban", "version.py")).read())
 
 setup(
 	name = "fail2ban",