From f555ff45e99c804a3fa48a2db1af56fa9200da0a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 19:08:52 +0200 Subject: [PATCH 1/7] attempt to speedup ban- and fail-manager (e. g. fail2ban-client status, see gh-2819), remove unneeded lock (GIL is enough here) --- fail2ban/server/banmanager.py | 20 +++++++------------- fail2ban/server/failmanager.py | 9 +++------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 479ba26f..fe38dec6 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -66,8 +66,7 @@ class BanManager: # @param value the time def setBanTime(self, value): - with self.__lock: - self.__banTime = int(value) + self.__banTime = int(value) ## # Get the ban time. @@ -76,8 +75,7 @@ class BanManager: # @return the time def getBanTime(self): - with self.__lock: - return self.__banTime + return self.__banTime ## # Set the total number of banned address. @@ -85,8 +83,7 @@ class BanManager: # @param value total number def setBanTotal(self, value): - with self.__lock: - self.__banTotal = value + self.__banTotal = value ## # Get the total number of banned address. @@ -94,8 +91,7 @@ class BanManager: # @return the total number def getBanTotal(self): - with self.__lock: - return self.__banTotal + return self.__banTotal ## # Returns a copy of the IP list. @@ -103,8 +99,7 @@ class BanManager: # @return IP list def getBanList(self): - with self.__lock: - return list(self.__banList.keys()) + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,9 +107,8 @@ class BanManager: # @return ban list iterator def __iter__(self): - # ensure iterator is safe (traverse over the list in snapshot created within lock): - with self.__lock: - return iter(list(self.__banList.values())) + # ensure iterator is safe - traverse over the list in snapshot created within lock (GIL): + return iter(list(self.__banList.values())) ## # Returns normalized value diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3458aed5..3e81e8b5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -47,12 +47,10 @@ class FailManager: self.__bgSvc = BgService() def setFailTotal(self, value): - with self.__lock: - self.__failTotal = value + self.__failTotal = value def getFailTotal(self): - with self.__lock: - return self.__failTotal + return self.__failTotal def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... @@ -123,8 +121,7 @@ class FailManager: return attempts def size(self): - with self.__lock: - return len(self.__failList) + return len(self.__failList) def cleanup(self, time): with self.__lock: From 5abc4ba4ae280b6ec89c6bffc9dd2140d0d56dc4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 22:11:51 +0200 Subject: [PATCH 2/7] amend to 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c (#2758): better reaction on broken pipe (on long output), don't close stdout explicitly (allows usage of modules like cProfile, which outputs result on exit), just flush it before exit. --- fail2ban/client/fail2bancmdline.py | 24 ++++++++++++++++-------- fail2ban/helpers.py | 5 +++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 8936e03f..03683cad 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,13 +27,17 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise # Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") @@ -310,12 +314,16 @@ class Fail2banCmdLine(): def _exit(code=0): # implicit flush without to produce broken pipe error (32): sys.stderr.close() - sys.stdout.close() - # exit: - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f381576e..c45be849 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -224,9 +224,10 @@ def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover sys.exit(0) try: - BrokenPipeError + BrokenPipeError = BrokenPipeError except NameError: # pragma: 3.x no cover - BrokenPipeError = IOError + BrokenPipeError = IOError + __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, From e8ee3ba544c7eefd7b41e34ec3a44550c1280fbb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:36:54 +0200 Subject: [PATCH 3/7] resolves a bottleneck within transmitting of large data between server and client: speedup search of communications end-marker and increase max buffer size (up to 32KB) --- fail2ban/client/csocket.py | 14 +++++++++----- fail2ban/tests/sockettestcase.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 8cd22a41..2d414e5c 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -152,7 +152,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(RuntimeError, r"socket connection broken", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler @@ -168,7 +168,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: TestMsgError() - #self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage) + #self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage) self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error') finally: RequestHandler.found_terminator = org_handler From f381b982465e3e9bf58f565cfea006fedc1989f1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:44:55 +0200 Subject: [PATCH 4/7] introduces new flavor `short` for `fail2ban-client status $jail short`: output total and current counts only, without banned IPs list in order to speedup it and to provide more clear output (gh-2819), flavor `basic` (still default) is unmodified for backwards compatibility; it can be changed later to `short`, so for full list of IPs in newer version one should better use: - `fail2ban-client status $jail basic` - `fail2ban-client get $jail banned` or `fail2ban-client banned` --- fail2ban/server/actions.py | 14 ++++++++++---- fail2ban/tests/actionstestcase.py | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3308d4b2..f14d8d7b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -661,13 +661,19 @@ class Actions(JailThread, Mapping): """Status of current and total ban counts and current banned IP list. """ # TODO: Allow this list to be printed as 'status' output - supported_flavors = ["basic", "cymru"] + supported_flavors = ["short", "basic", "cymru"] if flavor is None or flavor not in supported_flavors: logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) - ret = [("Currently banned", self.__banManager.size()), - ("Total banned", self.__banManager.getBanTotal()), - ("Banned IP list", self.__banManager.getBanList())] + if flavor != "short": + banned = self.__banManager.getBanList() + cnt = len(banned) + else: + cnt = self.__banManager.size() + ret = [("Currently banned", cnt), + ("Total banned", self.__banManager.getBanTotal())] + if flavor != "short": + ret += [("Banned IP list", banned)] if flavor == "cymru": cymru_info = self.__banManager.getBanListExtendedCymruInfo() ret += \ diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index d97d9921..532fe6ed 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase): self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop') self.assertEqual(self.__actions.status(),[("Currently banned", 0 ), ("Total banned", 0 ), ("Banned IP list", [] )]) + self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ), + ("Total banned", 0 )]) def testAddActionPython(self): self.__actions.add( From d977d81ef75f96d41e8aa994aa8efd81f13bf795 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 17 Sep 2020 12:39:08 +0200 Subject: [PATCH 5/7] action.d/abuseipdb.conf: removed broken link, simplified usage example, fixed typos --- config/action.d/abuseipdb.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index d36cb3a9..ed958c86 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -21,14 +21,13 @@ # # Example, for ssh bruteforce (in section [sshd] of `jail.local`): # action = %(known/action)s -# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] # -# See below for catagories. +# See below for categories. # -# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) -## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call. # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] # ID Title Description # 3 Fraud Orders @@ -101,5 +100,5 @@ actionunban = # Notes Your API key from abuseipdb.com # Values: STRING Default: None # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. -# You will need to set the catagory in the action call. +# You will need to set the category in the action call. abuseipdb_apikey = From f518d42c590f4ea3d8632937dc6ad7e875419b00 Mon Sep 17 00:00:00 2001 From: Nathan Henrie Date: Wed, 12 Aug 2020 14:11:33 -0600 Subject: [PATCH 6/7] Add a note about `journalflags` options to `systemd-journal` backend Also adds systemd backend configuration examples to jail.conf(5) Closes #2696 --- fail2ban/client/fail2banregex.py | 4 +++- man/jail.conf.5 | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index d0fe55dc..56da17dd 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -113,7 +113,9 @@ class _f2bOptParser(OptionParser): LOG: string a string representing a log line filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + "systemd-journal" search systemd journal. Optionally specify + `systemd-journal[journalflags=X]` to determine + which journals are used (systemd-python required) REGEX: string a string representing a 'failregex' diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 830c8aed..d7722124 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst uses a polling algorithm which does not require external libraries. .TP .B systemd -uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples: +.PP +.RS +.nf +backend = systemd[journalpath=/run/log/journal/machine-1] +backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"] +backend = systemd[journalflags=1] +.fi .SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: From 24093de32daa05af5f178639095ac7106d55b544 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 23 Sep 2020 19:35:17 +0200 Subject: [PATCH 7/7] small amend (simplifying formatted help and man) --- fail2ban/client/fail2banregex.py | 22 +++++++++++----------- man/fail2ban-regex.1 | 9 +++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 56da17dd..e7a4e214 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,7 +21,6 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" @@ -109,21 +108,22 @@ class _f2bOptParser(OptionParser): def format_help(self, *args, **kwargs): """ Overwritten format helper with full ussage.""" self.usage = '' - return "Usage: " + usage() + __doc__ + """ + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal. Optionally specify - `systemd-journal[journalflags=X]` to determine - which journals are used (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n Report bugs to https://github.com/fail2ban/fail2ban/issues\n """ + __copyright__ + "\n" diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index bb89ef8c..3964126f 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -18,13 +18,18 @@ a string representing a log line filename path to a log file (\fI\,/var/log/auth.log\/\fP) .TP -"systemd\-journal" -search systemd journal (systemd\-python required) +systemd\-journal +search systemd journal (systemd\-python required), +optionally with backend parameters, see `man jail.conf` +for usage and examples (systemd\-journal[journalflags=1]). .SS "REGEX:" .TP string a string representing a 'failregex' .TP +filter +name of filter, optionally with options (sshd[mode=aggressive]) +.TP filename path to a filter file (filter.d/sshd.conf) .SS "IGNOREREGEX:"