Merge branch '0.10' into 0.11

ChangeLog

@@ -81,6 +81,9 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition
 * `filter.d/traefik-auth.conf`: used to ban hosts, that were failed through traefik
 
 ### Enhancements
+* fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size
+  of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to
+  avoid runtime error "can't start new thread" (see gh-969);
 * jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations
   containing new-line);
 * fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349);
@@ -91,6 +94,9 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition
   attempts (failure) for IP (resp. failure-ID), see gh-2351;
   Syntax:
   - `fail2ban-client set <jail> attempt <ip> [<failure-message1> ... <failure-messageN>]`
+* `action.d/badips.py`: option `loglevel` extended with level of summary message,
+  following example configuration logging summary with NOTICE and rest with DEBUG log-levels:
+  `action = badips.py[loglevel="debug, notice"]`
 
 
 ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four

config/action.d/badips.py

@@ -32,7 +32,7 @@ else: # pragma: 3.x no cover
 	from urllib import urlencode
 
 from fail2ban.server.actions import ActionBase
-from fail2ban.helpers import str2LogLevel
+from fail2ban.helpers import splitwords, str2LogLevel
 
 
 
@@ -75,6 +75,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 	loglevel : int/str, optional
 		Log level of the message when an IP is (un)banned.
 		Default `DEBUG`.
+		Can be also supplied as two-value list (comma- or space separated) to
+		provide level of the summary message when a group of IPs is (un)banned.
+		Example `DEBUG,INFO`.
 	agent : str, optional
 		User agent transmitted to server.
 		Default `Fail2Ban/ver.`
@@ -91,8 +94,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, loglevel='DEBUG', agent="Fail2Ban", 
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, 
-		timeout=TIMEOUT):
+		loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
 		self.timeout = timeout
@@ -104,7 +107,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		self.banaction = banaction
 		self.bancategory = bancategory or category
 		self.bankey = bankey
-		self.loglevel = str2LogLevel(loglevel)
+		loglevel = splitwords(loglevel)
+		self.sumloglevel = str2LogLevel(loglevel[-1])
+		self.loglevel = str2LogLevel(loglevel[0])
 		self.updateperiod = updateperiod
 
 		self._bannedips = set()
@@ -350,9 +355,13 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				s = ips - self._bannedips
 				p = len(s)
 				self._banIPs(s)
-				self._logSys.log(self.loglevel,
+				if m != 0 or p != 0:
-					"Updated IPs for jail '%s' (-%d/+%d). Update again in %i seconds",
+					self._logSys.log(self.sumloglevel,
-					self._jail.name, m, p, self.updateperiod)
+						"Updated IPs for jail '%s' (-%d/+%d)",
+						self._jail.name, m, p)
+				self._logSys.debug(
+					"Next update for jail '%' in %i seconds",
+					self._jail.name, self.updateperiod)
 			finally:
 				self._timer = threading.Timer(self.updateperiod, self.update)
 				self._timer.start()

config/fail2ban.conf

@@ -67,3 +67,11 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3
 # Notes.: Sets age at which bans should be purged from the database
 # Values: [ SECONDS ] Default: 86400 (24hours)
 dbpurgeage = 1d
+
+[Thread]
+
+# Options: stacksize
+# Notes.: Specifies the stack size (in KiB) to be used for subsequently created threads,
+#         and must be 0 or a positive integer value of at least 32.
+# Values: [ SIZE ] Default: 0 (use platform or configured default)
+#stacksize = 0

fail2ban/client/configreader.py

@@ -239,8 +239,10 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 			try:
 				if opttype == "bool":
 					v = self.getboolean(sec, optname)
+					if v is None: continue
 				elif opttype == "int":
 					v = self.getint(sec, optname)
+					if v is None: continue
 				else:
 					v = self.get(sec, optname, vars=pOptions)
 				values[optname] = v

fail2ban/client/fail2banreader.py

@@ -60,12 +60,19 @@ class Fail2banReader(ConfigReader):
 			self.__opts.update(updateMainOpt)
 		# check given log-level:
 		str2LogLevel(self.__opts.get('loglevel', 0))
-	
+		# thread options:
+		opts = [["int", "stacksize", ],
+		]
+		if self.has_section("Thread"):
+			thopt = ConfigReader.getOptions(self, "Thread", opts)
+			if thopt:
+				self.__opts['thread'] = thopt
+
 	def convert(self):
 		# Ensure logtarget/level set first so any db errors are captured
 		# Also dbfile should be set before all other database options.
 		# So adding order indices into items, to be stripped after sorting, upon return
-		order = {"syslogsocket":0, "loglevel":1, "logtarget":2,
+		order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13,
 			"dbfile":50, "dbpurgeage":51}
 		stream = list()
 		for opt in self.__opts:

fail2ban/server/server.py

@@ -746,6 +746,16 @@ class Server:
 				logSys.info("flush performed on %s" % self.__logTarget)
 			return "flushed"
 			
+	def setThreadOptions(self, value):
+		for o, v in value.iteritems():
+			if o == 'stacksize':
+				threading.stack_size(int(v)*1024)
+			else: # pragma: no cover
+				raise KeyError("unknown option %r" % o)
+
+	def getThreadOptions(self):
+		return {'stacksize': threading.stack_size() // 1024}
+
 	def setDatabase(self, filename):
 		# if not changed - nothing to do
 		if self.__db and self.__db.filename == filename:

fail2ban/server/transmitter.py

@@ -170,6 +170,10 @@ class Transmitter:
 				return self.__server.getSyslogSocket()
 			else:
 				raise Exception("Failed to change syslog socket")
+		#Thread
+		elif name == "thread":
+			value = command[1]
+			return self.__server.setThreadOptions(value)
 		#Database
 		elif name == "dbfile":
 			self.__server.setDatabase(command[1])
@@ -390,6 +394,9 @@ class Transmitter:
 			return self.__server.getLogTarget()
 		elif name == "syslogsocket":
 			return self.__server.getSyslogSocket()
+		#Thread
+		elif name == "thread":
+			return self.__server.getThreadOptions()
 		#Database
 		elif name == "dbfile":
 			db = self.__server.getDatabase()

fail2ban/tests/action_d/test_badips.py

@@ -55,6 +55,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 		pythonModule = None
 		modAction = None
 
+		@skip_if_not_available
 		def setUp(self):
 			"""Call before every test case."""
 			super(BadIPsActionTest, self).setUp()

fail2ban/tests/fail2banclienttestcase.py

@@ -169,7 +169,8 @@ def _read_file(fn):
 
 
 def _start_params(tmp, use_stock=False, use_stock_cfg=None, 
-	logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None
+	logtarget="/dev/null", db=":memory:", f2b_local=(), jails=("",), 
+	create_before_start=None,
 ):
 	cfg = pjoin(tmp, "config")
 	if db == 'auto':
@@ -219,6 +220,9 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None,
 		if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
 			_out_file(pjoin(cfg, "fail2ban.conf"))
 			_out_file(pjoin(cfg, "jail.conf"))
+	if f2b_local:
+		_write_file(pjoin(cfg, "fail2ban.local"), "w", *f2b_local)
+
 	# link stock actions and filters:
 	if use_stock_cfg and STOCK:
 		for n in use_stock_cfg:
@@ -462,8 +466,16 @@ class Fail2banClientServerBase(LogCaptureTestCase):
 			phase['end'] = True
 			logSys.debug("end of test worker")
 
-	@with_foreground_server_thread()
+	@with_foreground_server_thread(startextra={'f2b_local':(
+			"[Thread]",
+			"stacksize = 32"
+			"",
+		)})
 	def testStartForeground(self, tmp, startparams):
+		# check thread options were set:
+		self.pruneLog()
+		self.execCmd(SUCCESS, startparams, "get", "thread")
+		self.assertLogged("{'stacksize': 32}")
 		# several commands to server:
 		self.execCmd(SUCCESS, startparams, "ping")
 		self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~")

man/jail.conf.5

@@ -127,9 +127,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com
 
 .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)"
 
-These files have one section, [Definition].
+The items that can be set in section [Definition] are:
-
-The items that can be set are:
 .TP
 .B loglevel
 verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40)
@@ -163,6 +161,15 @@ Database purge age in seconds. Default: 86400 (24hours)
 .br
 This sets the age at which bans should be purged from the database.
 
+.RE
+The config parameters of section [Thread] are:
+
+.TP
+.B stacksize
+Stack size of each thread in fail2ban. Default: 0 (platform or configured default)
+.br
+This specifies the stack size (in KiB) to be used for subsequently created threads, and must be 0 or a positive integer value of at least 32.
+
 .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)"
 The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section.
 .TP