From 1bcb62e31ca0372abc4fefb81724f30fb6ff8837 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Nov 2021 19:15:59 +0100 Subject: [PATCH 1/5] gh-actions: python releases upgrade + debug/test systemd backend availability in GHA --- .github/workflows/main.yml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e346635d..c0dc726b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10.0-rc.2', pypy2, pypy3] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3] fail-fast: false # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -40,33 +40,44 @@ jobs: - name: Python version run: | F2B_PY=$(python -c "import sys; print(sys.version)") - echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }" F2B_PY=${F2B_PY:0:1} echo "Set F2B_PY=$F2B_PY" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV - name: Install dependencies run: | - #python -m pip install --upgrade pip + if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then #pip install 2to3 sudo apt-get -y install 2to3 fi - #pip install systemd-python || echo 'systemd not available' - #pip install pyinotify || echo 'inotify not available' - sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' - sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' + #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' + python -m pip install pyinotify || echo 'inotify not available' + #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' + sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable' + python -m pip install systemd-python || echo 'systemd not available' - name: Before scripts run: | cd "$GITHUB_WORKSPACE" # Manually execute 2to3 for now if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; } # (debug) output current preferred encoding: - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + # (debug) backend availabilities: + echo 'Backends:' + _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - - name: Test suite - run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + # - name: Test suite + # run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + - name: Test suite (debug some systemd tests only) + # run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" + run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" - name: Build run: python setup.py build From a147a8b0e1b2f32b6f191932afd3c2db9765e2e3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 1 Dec 2021 20:04:46 +0100 Subject: [PATCH 2/5] gh-actions: coverage for systemd backend (to monitor journals in test-suite in GHA-env we need to use 0 as default flags, because otherwise it cannot be found using SYSTEM_ONLY(4)) --- .github/workflows/main.yml | 14 ++++++++------ fail2ban/server/filtersystemd.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0dc726b..788b17e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,8 @@ jobs: F2B_PY=${F2B_PY:0:1} echo "Set F2B_PY=$F2B_PY" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4): + echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV - name: Install dependencies run: | @@ -71,13 +73,13 @@ jobs: _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd' _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - - # - name: Test suite - # run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi - - name: Test suite (debug some systemd tests only) - # run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" - run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test suite (debug some systemd tests only) + #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" + #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" - name: Build run: python setup.py build diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index e2220105..6301b93a 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import datetime +import os import time from distutils.version import LooseVersion @@ -93,7 +94,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover except KeyError: # be sure all journal types will be opened if files/path specified (don't set flags): if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']): - args['flags'] = 4 + args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4)) try: args['namespace'] = kwargs.pop('namespace') From 0fa76ef75a7c6102f383c703fdcf6d53e45149b0 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 2 Dec 2021 18:27:56 +0100 Subject: [PATCH 3/5] gh-actions: temporary ignore tests of systemd backend for python >= v.3.10 (otherwise it fails with "PY_SSIZE_T_CLEAN macro must be defined for '#' formats") --- .github/workflows/main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 788b17e3..ff31db19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,9 +41,11 @@ jobs: run: | F2B_PY=$(python -c "import sys; print(sys.version)") echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }" + F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)') F2B_PY=${F2B_PY:0:1} - echo "Set F2B_PY=$F2B_PY" + echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV" echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4): echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV @@ -75,7 +77,15 @@ jobs: _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' - name: Test suite - run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + run: | + if [[ "$F2B_PY" = 2 ]]; then + python setup.py test + elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then + python bin/fail2ban-testcases --verbosity=2 + else + echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." + python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" + fi #- name: Test suite (debug some systemd tests only) #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" From 196c55e93103d1e56d1336f27a5f2591b4e54fef Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:49:18 +0200 Subject: [PATCH 4/5] fix gh-3098: build fails with error in fail2ban setup command: use_2to3 is invalid (setuptools 58+) --- setup.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/setup.py b/setup.py index f4c2550f..98413273 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ import warnings from glob import glob from fail2ban.setup import updatePyExec - +from fail2ban.version import version source_dir = os.path.realpath(os.path.dirname( # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): @@ -112,22 +112,12 @@ class install_scripts_f2b(install_scripts): # Wrapper to specify fail2ban own options: class install_command_f2b(install): user_options = install.user_options + [ - ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): - self.disable_2to3 = None self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): - global _2to3 - ## in the test cases 2to3 should be already done (fail2ban-2to3): - if self.disable_2to3: - _2to3 = False - if _2to3: - cmdclass = self.distribution.cmdclass - cmdclass['build_py'] = build_py_2to3 - cmdclass['build_scripts'] = build_scripts_2to3 if self.without_tests: self.distribution.scripts.remove('bin/fail2ban-testcases') @@ -178,7 +168,6 @@ commands.''' if setuptools: setup_extra = { 'test_suite': "fail2ban.tests.utils.gatherTests", - 'use_2to3': True, } else: setup_extra = {} @@ -202,9 +191,6 @@ if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswit ('/usr/share/doc/fail2ban', doc_files) ) -# Get version number, avoiding importing fail2ban. -# This is due to tests not functioning for python3 as 2to3 takes place later -exec(open(join("fail2ban", "version.py")).read()) setup( name = "fail2ban", From 21d94ff178192116eb5288190d78dd4630534a7b Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 19 Sep 2021 18:52:34 +0200 Subject: [PATCH 5/5] amend to fix gh-3098: no option `--disable-2to3` anymore --- fail2ban/tests/misctestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 97669be0..e2faa6fd 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -111,7 +111,7 @@ class SetupTest(unittest.TestCase): supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: # try dry-run: - os.system("%s %s --dry-run install --disable-2to3 --root=%s%s" + os.system("%s %s --dry-run install --root=%s%s" % (sys.executable, self.setup , tmp, supdbgout)) # check nothing was created: self.assertTrue(not os.listdir(tmp)) @@ -127,7 +127,7 @@ class SetupTest(unittest.TestCase): # suppress stdout (and stderr) if not heavydebug supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: - self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s" + self.assertEqual(os.system("%s %s install --root=%s%s" % (sys.executable, self.setup, tmp, supdbgout)), 0) def strippath(l):