From 24b1dea197ec6bd8f1d39c37d02fe31a996b4466 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 16 Aug 2022 12:44:57 +0200 Subject: [PATCH 1/4] check large nofile limit issue (#3334) --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 39c85231..e805672a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,12 +81,12 @@ jobs: - name: Test suite run: | if [[ "$F2B_PY" = 2 ]]; then - python setup.py test + sudo sh -c 'ulimit -n 1048576 && python setup.py test' elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then - python bin/fail2ban-testcases --verbosity=2 + sudo sh -c 'ulimit -n 1048576 && python bin/fail2ban-testcases --verbosity=2' else echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." - python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" + sudo sh -c 'ulimit -n 1048576 && python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"' fi #- name: Test suite (debug some systemd tests only) From 535a982dccfb5e05f8fa0f3965e5bf454716179d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 Aug 2022 15:07:30 +0200 Subject: [PATCH 2/4] fixes #3334: speedup daemonization process by huge open files limit (try to close open file descriptors obtained from `/proc/self/fd` or `/proc/fd`) --- fail2ban/server/server.py | 42 ++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 627ffe8d..02937a5a 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -809,6 +809,26 @@ class Server: def getDatabase(self): return self.__db + @staticmethod + def __get_fdlist(): + """Generate a list of open file descriptors. + + This wouldn't work on some platforms, or if proc/fdescfs not mounted, or a chroot environment, + then it'd raise a FileExistsError. + """ + for path in ( + '/proc/self/fd', # Linux, Cygwin and NetBSD + '/proc/fd', # MacOS and FreeBSD + ): + if os.path.exists(path): + def fdlist(): + for name in os.listdir(path): + if name.isdigit(): + yield int(name) + return fdlist() + # other platform or unmounted, chroot etc: + raise FileExistsError("fd-list not found") + def __createDaemon(self): # pragma: no cover """ Detach a process from the controlling terminal and run it in the background as a daemon. @@ -866,25 +886,37 @@ class Server: # Signal to exit, parent of the first child. return None - # Close all open files. Try the system configuration variable, SC_OPEN_MAX, + # Close all open files. Try to obtain the range of open descriptors directly. + # As a fallback try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use # the default value (configurable). try: - maxfd = os.sysconf("SC_OPEN_MAX") - except (AttributeError, ValueError): - maxfd = 256 # default maximum + fdlist = self.__get_fdlist() + maxfd = -1 + except: + try: + maxfd = os.sysconf("SC_OPEN_MAX") + except (AttributeError, ValueError): + maxfd = 256 # default maximum + fdlist = xrange(maxfd+1) # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 # http://bugs.python.org/issue21207 if sys.version_info[0:3] == (3, 4, 0): # pragma: no cover urandom_fd = os.open("/dev/urandom", os.O_RDONLY) - for fd in range(0, maxfd): + for fd in fdlist: try: if not os.path.sameopenfile(urandom_fd, fd): os.close(fd) except OSError: # ERROR (ignore) pass os.close(urandom_fd) + elif maxfd == -1: + for fd in fdlist: + try: + os.close(fd) + except OSError: # ERROR (ignore) + pass else: os.closerange(0, maxfd) From 38026e59631fc1e1941a471941134e34ff253a81 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 Aug 2022 16:01:04 +0200 Subject: [PATCH 3/4] code review (replace deprecated setter, since python 3.10) --- fail2ban/client/fail2banclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 6ea18fda..c72208cd 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -196,7 +196,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): th.daemon = True th.start() # Mark current (main) thread as daemon: - self.setDaemon(True) + self.daemon = True # Start server direct here in main thread (not fork): self._server = Fail2banServer.startServerDirect(self._conf, False) From 476136281c7db70a734b87e60f9aa0911be65dee Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 Aug 2022 16:04:10 +0200 Subject: [PATCH 4/4] Revert "check large nofile limit issue (#3334)" (back to original open files limit) This reverts commit 24b1dea197ec6bd8f1d39c37d02fe31a996b4466. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e805672a..39c85231 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,12 +81,12 @@ jobs: - name: Test suite run: | if [[ "$F2B_PY" = 2 ]]; then - sudo sh -c 'ulimit -n 1048576 && python setup.py test' + python setup.py test elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then - sudo sh -c 'ulimit -n 1048576 && python bin/fail2ban-testcases --verbosity=2' + python bin/fail2ban-testcases --verbosity=2 else echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." - sudo sh -c 'ulimit -n 1048576 && python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal"' + python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" fi #- name: Test suite (debug some systemd tests only)