From 3c70fe298a55f587b8a5633ad775ebfc19b83c95 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 16:45:14 +0100 Subject: [PATCH 1/5] closes gh-969: introduces new section `[Thread]` and option `stacksize` to configure default stack-size of the threads running in fail2ban. Example: ```ini [Thread] stacksize = 32 ``` --- config/fail2ban.conf | 8 ++++++++ fail2ban/client/configreader.py | 2 ++ fail2ban/client/fail2banreader.py | 13 ++++++++++--- fail2ban/server/server.py | 10 ++++++++++ fail2ban/server/transmitter.py | 7 +++++++ fail2ban/tests/fail2banclienttestcase.py | 14 +++++++++++++- man/jail.conf.5 | 13 ++++++++++--- 7 files changed, 60 insertions(+), 7 deletions(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index 52e47187..08e8fcf3 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -67,3 +67,11 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) dbpurgeage = 1d + +[Thread] + +# Options: stacksize +# Notes.: Specifies the stack size (in KiB) to be used for subsequently created threads, +# and must be 0 or a positive integer value of at least 32. +# Values: [ SIZE ] Default: 0 (use platform or configured default) +#stacksize = 0 diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b03daca9..2506d98f 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -239,8 +239,10 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): try: if opttype == "bool": v = self.getboolean(sec, optname) + if v is None: continue elif opttype == "int": v = self.getint(sec, optname) + if v is None: continue else: v = self.get(sec, optname, vars=pOptions) values[optname] = v diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c81d585e..f0788ce0 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigReader, NoSectionError from ..helpers import getLogger, str2LogLevel # Gets the instance of the logger. @@ -60,12 +60,19 @@ class Fail2banReader(ConfigReader): self.__opts.update(updateMainOpt) # check given log-level: str2LogLevel(self.__opts.get('loglevel', 0)) - + # thread options: + opts = [["int", "stacksize", ], + ] + if ConfigReader.has_section(self, "Thread"): + thopt = ConfigReader.getOptions(self, "Thread", opts) + if thopt: + self.__opts['thread'] = thopt + def convert(self): # Ensure logtarget/level set first so any db errors are captured # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return - order = {"syslogsocket":0, "loglevel":1, "logtarget":2, + order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, "dbfile":50, "dbpurgeage":51} stream = list() for opt in self.__opts: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 14013290..b7ee264a 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -709,6 +709,16 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + def setThreadOptions(self, value): + for o, v in value.iteritems(): + if o == 'stacksize': + threading.stack_size(int(v)*1024) + else: # pragma: no cover + raise KeyError("unknown option %r" % o) + + def getThreadOptions(self): + return {'stacksize': threading.stack_size() / 1024} + def setDatabase(self, filename): # if not changed - nothing to do if self.__db and self.__db.filename == filename: diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index a267c4b0..9d1ddbc9 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -170,6 +170,10 @@ class Transmitter: return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + #Thread + elif name == "thread": + value = command[1] + return self.__server.setThreadOptions(value) #Database elif name == "dbfile": self.__server.setDatabase(command[1]) @@ -384,6 +388,9 @@ class Transmitter: return self.__server.getLogTarget() elif name == "syslogsocket": return self.__server.getSyslogSocket() + #Thread + elif name == "thread": + return self.__server.getThreadOptions() #Database elif name == "dbfile": db = self.__server.getDatabase() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 3972f80a..67481211 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -478,6 +478,12 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundInside(self, tmp): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) + # test additional options: + _write_file(pjoin(tmp, "config", "fail2ban.conf"), "a", + "[Thread]", + "stacksize = 32" + "", + ) # start: self.execCmd(SUCCESS, ("-b",) + startparams, "start") # wait for server (socket and ready): @@ -485,10 +491,16 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: + # check thread options were set: + self.pruneLog() + self.execCmd(SUCCESS, startparams, "get", "thread") + self.assertLogged("{'stacksize': 32}") + # several: + self.pruneLog() self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") - self.pruneLog() # start again (should fail): + self.pruneLog() self.execCmd(FAILED, ("-b",) + startparams, "start") self.assertLogged("Server already running") finally: diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 25b3dd70..0f3a4aa7 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -127,9 +127,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" -These files have one section, [Definition]. - -The items that can be set are: +The items that can be set in section [Definition] are: .TP .B loglevel verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) @@ -163,6 +161,15 @@ Database purge age in seconds. Default: 86400 (24hours) .br This sets the age at which bans should be purged from the database. +.RE +The config parameters of section [Thread] are: + +.TP +.B stacksize +Stack size of each thread in fail2ban. Default: 0 (platform or configured default) +.br +This specifies the stack size (in KiB) to be used for subsequently created threads, and must be 0 or a positive integer value of at least 32. + .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP From fffeb7785c972961f9e0601bde9df793ffa1b24b Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 16:56:13 +0100 Subject: [PATCH 2/5] code review --- fail2ban/client/fail2banreader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index f0788ce0..5760e0ba 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader, NoSectionError +from .configreader import ConfigReader from ..helpers import getLogger, str2LogLevel # Gets the instance of the logger. @@ -63,7 +63,7 @@ class Fail2banReader(ConfigReader): # thread options: opts = [["int", "stacksize", ], ] - if ConfigReader.has_section(self, "Thread"): + if self.has_section("Thread"): thopt = ConfigReader.getOptions(self, "Thread", opts) if thopt: self.__opts['thread'] = thopt From f6468e753bc2c1ef37775bfa88fd224174d9ac9e Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 17:14:53 +0100 Subject: [PATCH 3/5] resolves py3.x compat issues in tests --- fail2ban/server/server.py | 2 +- fail2ban/tests/fail2banclienttestcase.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index b7ee264a..33edf315 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -717,7 +717,7 @@ class Server: raise KeyError("unknown option %r" % o) def getThreadOptions(self): - return {'stacksize': threading.stack_size() / 1024} + return {'stacksize': threading.stack_size() // 1024} def setDatabase(self, filename): # if not changed - nothing to do diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 67481211..1b072db7 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -479,7 +479,7 @@ class Fail2banClientTest(Fail2banClientServerBase): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) # test additional options: - _write_file(pjoin(tmp, "config", "fail2ban.conf"), "a", + _write_file(pjoin(tmp, "config", "fail2ban.local"), "w", "[Thread]", "stacksize = 32" "", From 32ba74463f886e08d79502013f5da36e85c04a50 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 24 Feb 2019 18:43:23 +0100 Subject: [PATCH 4/5] coverage - move to another tests (directly covering server) --- fail2ban/tests/fail2banclienttestcase.py | 30 ++++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 1b072db7..c532143e 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -140,7 +140,8 @@ def _read_file(fn): def _start_params(tmp, use_stock=False, use_stock_cfg=None, - logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None + logtarget="/dev/null", db=":memory:", f2b_local=(), jails=("",), + create_before_start=None, ): cfg = pjoin(tmp, "config") if db == 'auto': @@ -190,6 +191,9 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(pjoin(cfg, "fail2ban.conf")) _out_file(pjoin(cfg, "jail.conf")) + if f2b_local: + _write_file(pjoin(cfg, "fail2ban.local"), "w", *f2b_local) + # link stock actions and filters: if use_stock_cfg and STOCK: for n in use_stock_cfg: @@ -431,8 +435,16 @@ class Fail2banClientServerBase(LogCaptureTestCase): phase['end'] = True logSys.debug("end of test worker") - @with_foreground_server_thread() + @with_foreground_server_thread(startextra={'f2b_local':( + "[Thread]", + "stacksize = 32" + "", + )}) def testStartForeground(self, tmp, startparams): + # check thread options were set: + self.pruneLog() + self.execCmd(SUCCESS, startparams, "get", "thread") + self.assertLogged("{'stacksize': 32}") # several commands to server: self.execCmd(SUCCESS, startparams, "ping") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") @@ -478,12 +490,6 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundInside(self, tmp): # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) - # test additional options: - _write_file(pjoin(tmp, "config", "fail2ban.local"), "w", - "[Thread]", - "stacksize = 32" - "", - ) # start: self.execCmd(SUCCESS, ("-b",) + startparams, "start") # wait for server (socket and ready): @@ -491,16 +497,10 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: - # check thread options were set: - self.pruneLog() - self.execCmd(SUCCESS, startparams, "get", "thread") - self.assertLogged("{'stacksize': 32}") - # several: - self.pruneLog() self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") - # start again (should fail): self.pruneLog() + # start again (should fail): self.execCmd(FAILED, ("-b",) + startparams, "start") self.assertLogged("Server already running") finally: From 6c14f1987f2c136dac416cb616265fac645296b0 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 1 Mar 2019 12:31:17 +0100 Subject: [PATCH 5/5] Update ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 425979f7..e20f6ccc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,9 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition * `filter.d/traefik-auth.conf`: used to ban hosts, that were failed through traefik ### Enhancements +* fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size + of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to + avoid runtime error "can't start new thread" (see gh-969); * jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations containing new-line); * fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349);