From 689938ee9926320b6d24df7b352fe15c785c511e Mon Sep 17 00:00:00 2001 From: benrubson Date: Tue, 9 Oct 2018 19:27:39 +0200 Subject: [PATCH 01/10] Add a shortloglevel badips.py option --- config/action.d/badips.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 1ad711f4..c982bf33 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -75,6 +75,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable loglevel : int/str, optional Log level of the message when an IP is (un)banned. Default `DEBUG`. + shortloglevel : int/str, optional + Log level of the summary message when a group of IPs is (un)banned. + Default `DEBUG`. agent : str, optional User agent transmitted to server. Default `Fail2Ban/ver.` @@ -91,8 +94,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable return Request(url, headers={'User-Agent': self.agent}, **argv) def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900, loglevel='DEBUG', agent="Fail2Ban", - timeout=TIMEOUT): + banaction=None, bancategory=None, bankey=None, updateperiod=900, + loglevel='DEBUG', shortloglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): super(BadIPsAction, self).__init__(jail, name) self.timeout = timeout @@ -105,6 +108,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable self.bancategory = bancategory or category self.bankey = bankey self.loglevel = str2LogLevel(loglevel) + self.shortloglevel = str2LogLevel(shortloglevel) self.updateperiod = updateperiod self._bannedips = set() @@ -350,9 +354,13 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable s = ips - self._bannedips p = len(s) self._banIPs(s) - self._logSys.log(self.loglevel, - "Updated IPs for jail '%s' (-%d/+%d). Update again in %i seconds", - self._jail.name, m, p, self.updateperiod) + if m != 0 or p != 0: + self._logSys.log(self.shortloglevel, + "Updated IPs for jail '%s' (-%d/+%d)", + self._jail.name, m, p) + self._logSys.debug( + "Next update for jail '%' in %i seconds", + self._jail.name, self.updateperiod) finally: self._timer = threading.Timer(self.updateperiod, self.update) self._timer.start() From 34edec297bb2b10c01e1182e1d0d809623e0093d Mon Sep 17 00:00:00 2001 From: Ben RUBSON Date: Tue, 4 Dec 2018 21:26:24 +0100 Subject: [PATCH 02/10] Add changelog entry --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index d389ea0c..089fae08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -105,6 +105,7 @@ ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four * `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168) * since v.0.10.4, fail2ban-client, fail2ban-server and fail2ban-regex will return version without logo info, additionally option `-V` can be used to get version in normalized machine-readable short format. +* `action.d/badips.py`: add a `shortloglevel` option for the summary message, in addition to the existing `loglevel` one ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right From 51260680997b8aa645835ca29e42094996ef1335 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 22 Feb 2019 14:03:21 +0100 Subject: [PATCH 03/10] loglevel and shortloglevel combined to single parameter loglevel, below an example logging summary with NOTICE and rest with DEBUG log-levels: action = badips.py[... , loglevel="debug, notice"] --- config/action.d/badips.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index c982bf33..e5de7555 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -32,7 +32,7 @@ else: # pragma: 3.x no cover from urllib import urlencode from fail2ban.server.actions import ActionBase -from fail2ban.helpers import str2LogLevel +from fail2ban.helpers import splitwords, str2LogLevel @@ -75,9 +75,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable loglevel : int/str, optional Log level of the message when an IP is (un)banned. Default `DEBUG`. - shortloglevel : int/str, optional - Log level of the summary message when a group of IPs is (un)banned. - Default `DEBUG`. + Can be also supplied as two-value list (comma- or space separated) to + provide level of the summary message when a group of IPs is (un)banned. + Example `DEBUG,INFO`. agent : str, optional User agent transmitted to server. Default `Fail2Ban/ver.` @@ -95,7 +95,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable def __init__(self, jail, name, category, score=3, age="24h", key=None, banaction=None, bancategory=None, bankey=None, updateperiod=900, - loglevel='DEBUG', shortloglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): + loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): super(BadIPsAction, self).__init__(jail, name) self.timeout = timeout @@ -107,8 +107,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable self.banaction = banaction self.bancategory = bancategory or category self.bankey = bankey - self.loglevel = str2LogLevel(loglevel) - self.shortloglevel = str2LogLevel(shortloglevel) + loglevel = splitwords(loglevel) + self.sumloglevel = str2LogLevel(loglevel[-1]) + self.loglevel = str2LogLevel(loglevel[0]) self.updateperiod = updateperiod self._bannedips = set() @@ -355,7 +356,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable p = len(s) self._banIPs(s) if m != 0 or p != 0: - self._logSys.log(self.shortloglevel, + self._logSys.log(self.sumloglevel, "Updated IPs for jail '%s' (-%d/+%d)", self._jail.name, m, p) self._logSys.debug( From 3fcb0a868d1a53ca42cd78fd477ce9f8c75427cc Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 22 Feb 2019 14:07:11 +0100 Subject: [PATCH 04/10] test-cases: availability of badips-service - avoid sporadic errors (like "The handshake operation timed out") during setup of tests --- fail2ban/tests/action_d/test_badips.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index 3ea1fc76..7f2222e1 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -55,6 +55,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable pythonModule = None modAction = None + @skip_if_not_available def setUp(self): """Call before every test case.""" super(BadIPsActionTest, self).setUp() From 410a9804b1b7a96a9be382fe9172af365b4c9497 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 22 Feb 2019 14:23:05 +0100 Subject: [PATCH 05/10] Update ChangeLog --- ChangeLog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 089fae08..425979f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,6 +62,9 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition attempts (failure) for IP (resp. failure-ID), see gh-2351; Syntax: - `fail2ban-client set attempt [ ... ]` +* `action.d/badips.py`: option `loglevel` extended with level of summary message, + following example configuration logging summary with NOTICE and rest with DEBUG log-levels: + `action = badips.py[loglevel="debug, notice"]` ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four @@ -105,7 +108,6 @@ ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four * `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168) * since v.0.10.4, fail2ban-client, fail2ban-server and fail2ban-regex will return version without logo info, additionally option `-V` can be used to get version in normalized machine-readable short format. -* `action.d/badips.py`: add a `shortloglevel` option for the summary message, in addition to the existing `loglevel` one ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right From 3c70fe298a55f587b8a5633ad775ebfc19b83c95 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 16:45:14 +0100 Subject: [PATCH 06/10] closes gh-969: introduces new section `[Thread]` and option `stacksize` to configure default stack-size of the threads running in fail2ban. Example: ```ini [Thread] stacksize = 32 ``` --- config/fail2ban.conf | 8 ++++++++ fail2ban/client/configreader.py | 2 ++ fail2ban/client/fail2banreader.py | 13 ++++++++++--- fail2ban/server/server.py | 10 ++++++++++ fail2ban/server/transmitter.py | 7 +++++++ fail2ban/tests/fail2banclienttestcase.py | 14 +++++++++++++- man/jail.conf.5 | 13 ++++++++++--- 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index 52e47187..08e8fcf3 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -67,3 +67,11 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) dbpurgeage = 1d + +[Thread] + +# Options: stacksize +# Notes.: Specifies the stack size (in KiB) to be used for subsequently created threads, +# and must be 0 or a positive integer value of at least 32. +# Values: [ SIZE ] Default: 0 (use platform or configured default) +#stacksize = 0 diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b03daca9..2506d98f 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -239,8 +239,10 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): try: if opttype == "bool": v = self.getboolean(sec, optname) + if v is None: continue elif opttype == "int": v = self.getint(sec, optname) + if v is None: continue else: v = self.get(sec, optname, vars=pOptions) values[optname] = v diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c81d585e..f0788ce0 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigReader, NoSectionError from ..helpers import getLogger, str2LogLevel # Gets the instance of the logger. @@ -60,12 +60,19 @@ class Fail2banReader(ConfigReader): self.__opts.update(updateMainOpt) # check given log-level: str2LogLevel(self.__opts.get('loglevel', 0)) - + # thread options: + opts = [["int", "stacksize", ], + ] + if ConfigReader.has_section(self, "Thread"): + thopt = ConfigReader.getOptions(self, "Thread", opts) + if thopt: + self.__opts['thread'] = thopt + def convert(self): # Ensure logtarget/level set first so any db errors are captured # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return - order = {"syslogsocket":0, "loglevel":1, "logtarget":2, + order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, "dbfile":50, "dbpurgeage":51} stream = list() for opt in self.__opts: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 14013290..b7ee264a 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -709,6 +709,16 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + def setThreadOptions(self, value): + for o, v in value.iteritems(): + if o == 'stacksize': + threading.stack_size(int(v)*1024) + else: # pragma: no cover + raise KeyError("unknown option %r" % o) + + def getThreadOptions(self): + return {'stacksize': threading.stack_size() / 1024} + def setDatabase(self, filename): # if not changed - nothing to do if self.__db and self.__db.filename == filename: diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index a267c4b0..9d1ddbc9 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -170,6 +170,10 @@ class Transmitter: return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + #Thread + elif name == "thread": + value = command[1] + return self.__server.setThreadOptions(value) #Database elif name == "dbfile": self.__server.setDatabase(command[1]) @@ -384,6 +388,9 @@ class Transmitter: return self.__server.getLogTarget() elif name == "syslogsocket": return self.__server.getSyslogSocket() + #Thread + elif name == "thread": + return self.__server.getThreadOptions() #Database elif name == "dbfile": db = self.__server.getDatabase() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 3972f80a..67481211 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -478,6 +478,12 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundInside(self, tmp): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) + # test additional options: + _write_file(pjoin(tmp, "config", "fail2ban.conf"), "a", + "[Thread]", + "stacksize = 32" + "", + ) # start: self.execCmd(SUCCESS, ("-b",) + startparams, "start") # wait for server (socket and ready): @@ -485,10 +491,16 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: + # check thread options were set: + self.pruneLog() + self.execCmd(SUCCESS, startparams, "get", "thread") + self.assertLogged("{'stacksize': 32}") + # several: + self.pruneLog() self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") - self.pruneLog() # start again (should fail): + self.pruneLog() self.execCmd(FAILED, ("-b",) + startparams, "start") self.assertLogged("Server already running") finally: diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 25b3dd70..0f3a4aa7 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -127,9 +127,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" -These files have one section, [Definition]. - -The items that can be set are: +The items that can be set in section [Definition] are: .TP .B loglevel verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) @@ -163,6 +161,15 @@ Database purge age in seconds. Default: 86400 (24hours) .br This sets the age at which bans should be purged from the database. +.RE +The config parameters of section [Thread] are: + +.TP +.B stacksize +Stack size of each thread in fail2ban. Default: 0 (platform or configured default) +.br +This specifies the stack size (in KiB) to be used for subsequently created threads, and must be 0 or a positive integer value of at least 32. + .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP From fffeb7785c972961f9e0601bde9df793ffa1b24b Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 16:56:13 +0100 Subject: [PATCH 07/10] code review --- fail2ban/client/fail2banreader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index f0788ce0..5760e0ba 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader, NoSectionError +from .configreader import ConfigReader from ..helpers import getLogger, str2LogLevel # Gets the instance of the logger. @@ -63,7 +63,7 @@ class Fail2banReader(ConfigReader): # thread options: opts = [["int", "stacksize", ], ] - if ConfigReader.has_section(self, "Thread"): + if self.has_section("Thread"): thopt = ConfigReader.getOptions(self, "Thread", opts) if thopt: self.__opts['thread'] = thopt From f6468e753bc2c1ef37775bfa88fd224174d9ac9e Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 17:14:53 +0100 Subject: [PATCH 08/10] resolves py3.x compat issues in tests --- fail2ban/server/server.py | 2 +- fail2ban/tests/fail2banclienttestcase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index b7ee264a..33edf315 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -717,7 +717,7 @@ class Server: raise KeyError("unknown option %r" % o) def getThreadOptions(self): - return {'stacksize': threading.stack_size() / 1024} + return {'stacksize': threading.stack_size() // 1024} def setDatabase(self, filename): # if not changed - nothing to do diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 67481211..1b072db7 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -479,7 +479,7 @@ class Fail2banClientTest(Fail2banClientServerBase): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) # test additional options: - _write_file(pjoin(tmp, "config", "fail2ban.conf"), "a", + _write_file(pjoin(tmp, "config", "fail2ban.local"), "w", "[Thread]", "stacksize = 32" "", From 32ba74463f886e08d79502013f5da36e85c04a50 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 18:43:23 +0100 Subject: [PATCH 09/10] coverage - move to another tests (directly covering server) --- fail2ban/tests/fail2banclienttestcase.py | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 1b072db7..c532143e 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -140,7 +140,8 @@ def _read_file(fn): def _start_params(tmp, use_stock=False, use_stock_cfg=None, - logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None + logtarget="/dev/null", db=":memory:", f2b_local=(), jails=("",), + create_before_start=None, ): cfg = pjoin(tmp, "config") if db == 'auto': @@ -190,6 +191,9 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(pjoin(cfg, "fail2ban.conf")) _out_file(pjoin(cfg, "jail.conf")) + if f2b_local: + _write_file(pjoin(cfg, "fail2ban.local"), "w", *f2b_local) + # link stock actions and filters: if use_stock_cfg and STOCK: for n in use_stock_cfg: @@ -431,8 +435,16 @@ class Fail2banClientServerBase(LogCaptureTestCase): phase['end'] = True logSys.debug("end of test worker") - @with_foreground_server_thread() + @with_foreground_server_thread(startextra={'f2b_local':( + "[Thread]", + "stacksize = 32" + "", + )}) def testStartForeground(self, tmp, startparams): + # check thread options were set: + self.pruneLog() + self.execCmd(SUCCESS, startparams, "get", "thread") + self.assertLogged("{'stacksize': 32}") # several commands to server: self.execCmd(SUCCESS, startparams, "ping") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") @@ -478,12 +490,6 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundInside(self, tmp): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) - # test additional options: - _write_file(pjoin(tmp, "config", "fail2ban.local"), "w", - "[Thread]", - "stacksize = 32" - "", - ) # start: self.execCmd(SUCCESS, ("-b",) + startparams, "start") # wait for server (socket and ready): @@ -491,16 +497,10 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: - # check thread options were set: - self.pruneLog() - self.execCmd(SUCCESS, startparams, "get", "thread") - self.assertLogged("{'stacksize': 32}") - # several: - self.pruneLog() self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") - # start again (should fail): self.pruneLog() + # start again (should fail): self.execCmd(FAILED, ("-b",) + startparams, "start") self.assertLogged("Server already running") finally: From 6c14f1987f2c136dac416cb616265fac645296b0 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 1 Mar 2019 12:31:17 +0100 Subject: [PATCH 10/10] Update ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 425979f7..e20f6ccc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,9 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition * `filter.d/traefik-auth.conf`: used to ban hosts, that were failed through traefik ### Enhancements +* fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size + of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to + avoid runtime error "can't start new thread" (see gh-969); * jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations containing new-line); * fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349);