Merge branch 'gh-2655--f2b-regex-4-jail': implemented loading of jail settings in fail2ban-regex;

closes gh-2655

ChangeLog

@@ -16,6 +16,14 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition
 * `action.d/*ipset*`: make `maxelem` ipset option configurable through banaction arguments (gh-3564)
 
 ### New Features and Enhancements
+* `fail2ban-regex` extended to load settings from jail (by simple name it'd prefer jail to the filter now, gh-2655);
+  to load the settings from filter one could use:
+```diff
+- fail2ban-regex ... sshd           ; # jail
++ fail2ban-regex ... sshd.conf      ; # filter
+# or:
++ fail2ban-regex ... filter.d/sshd  ; # filter
+```
 * better auto-detection for IPv6 support (`allowipv6 = auto` by default), trying to check sysctl net.ipv6.conf.all.disable_ipv6
   (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces
   if available for platform and uses DNS to find local IPv6 as a fallback only

fail2ban/client/fail2banregex.py

@@ -51,7 +51,7 @@ except ImportError:
 	FilterSystemd = None
 
 from ..version import version, normVersion
-from .filterreader import FilterReader
+from .jailreader import FilterReader, JailReader, NoJailError
 from ..server.filter import Filter, FileContainer, MyTime
 from ..server.failregex import Regex, RegexException
 
@@ -312,12 +312,18 @@ class Fail2banRegex(object):
 	def _dumpRealOptions(self, reader, fltOpt):
 		realopts = {}
 		combopts = reader.getCombined()
+		if isinstance(reader, FilterReader):
+			_get_opt = lambda k: reader.get('Definition', k)
+		elif reader.filter: # JailReader for jail with filter:
+			_get_opt = lambda k: reader.filter.get('Definition', k)
+		else: # JailReader for jail without filter:
+			_get_opt = lambda k: None
 		# output all options that are specified in filter-argument as well as some special (mostly interested):
 		for k in ['logtype', 'datepattern'] + list(fltOpt.keys()):
 			# combined options win, but they contain only a sub-set in filter expected keys,
 			# so get the rest from definition section:
 			try:
-				realopts[k] = combopts[k] if k in combopts else reader.get('Definition', k)
+				realopts[k] = combopts[k] if k in combopts else _get_opt(k)
 			except NoOptionError: # pragma: no cover
 				pass
 		self.output("Real  filter options : %r" % realopts)
@@ -330,16 +336,26 @@ class Fail2banRegex(object):
 		fltName = value
 		fltFile = None
 		fltOpt = {}
+		jail = None
 		if regextype == 'fail':
 			if re.search(r'(?ms)^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value):
 				try:
 					fltName, fltOpt = extractOptions(value)
+					if re.search(r'(?ms)^[\w/_\-]+$', fltName): # name of jail?
+						try:
+							jail = JailReader(fltName, force_enable=True, 
+								share_config=self.share_config, basedir=basedir)
+							jail.read()
+						except NoJailError:
+							jail = None
 					if "." in fltName[~5:]:
 						tryNames = (fltName,)
 					else:
 						tryNames = (fltName, fltName + '.conf', fltName + '.local')
 					for fltFile in tryNames:
-						if not "/" in fltFile:
+						if os.path.dirname(fltFile) == 'filter.d':
+							fltFile = os.path.join(basedir, fltFile)
+						elif not "/" in fltFile:
 							if os.path.basename(basedir) == 'filter.d':
 								fltFile = os.path.join(basedir, fltFile)
 							else:
@@ -354,8 +370,25 @@ class Fail2banRegex(object):
 					output("       while parsing: %s" % (value,))
 					if self._verbose: raise(e)
 					return False
+		
+		readercommands = None
+		# if it is jail:
+		if jail:
+			self.output( "Use %11s jail : %s" % ('', fltName) )
+			if fltOpt:
+				self.output( "Use jail/flt options : %r" % fltOpt )
+			if not fltOpt: fltOpt = {}
+			fltOpt['backend'] = self._backend
+			ret = jail.getOptions(addOpts=fltOpt)
+			if not ret:
+				output('ERROR: Failed to get jail for %r' % (value,))
+				return False
+			# show real options if expected:
+			if self._verbose > 1 or logSys.getEffectiveLevel()<=logging.DEBUG:
+				self._dumpRealOptions(jail, fltOpt)
+			readercommands = jail.convert(allow_no_files=True)
 		# if it is filter file:
-		if fltFile is not None:
+		elif fltFile is not None:
 			if (basedir == self._opts.config
 				or os.path.basename(basedir) == 'filter.d'
 				or ("." not in fltName[~5:] and "/" not in fltName)
@@ -364,16 +397,17 @@ class Fail2banRegex(object):
 				if os.path.basename(basedir) == 'filter.d':
 					basedir = os.path.dirname(basedir)
 				fltName = os.path.splitext(os.path.basename(fltName))[0]
-				self.output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) )
+				self.output( "Use %11s file : %s, basedir: %s" % ('filter', fltName, basedir) )
 			else:
 				## foreign file - readexplicit this file and includes if possible:
-				self.output( "Use %11s file : %s" % (regex, fltName) )
+				self.output( "Use %11s file : %s" % ('filter', fltName) )
 				basedir = None
 				if not os.path.isabs(fltName): # avoid join with "filter.d" inside FilterReader
 					fltName = os.path.abspath(fltName)
 			if fltOpt:
 				self.output( "Use   filter options : %r" % fltOpt )
-			reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir)
+			reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt,
+				share_config=self.share_config, basedir=basedir)
 			ret = None
 			try:
 				if basedir is not None:
@@ -398,6 +432,7 @@ class Fail2banRegex(object):
 			# to stream:
 			readercommands = reader.convert()
 
+		if readercommands:
 			regex_values = {}
 			for opt in readercommands:
 				if opt[0] == 'multi-set':

fail2ban/client/jailreader.py

@@ -29,16 +29,19 @@ import json
 import os.path
 import re
 
-from .configreader import ConfigReaderUnshared, ConfigReader
+from .configreader import ConfigReaderUnshared, ConfigReader, NoSectionError
 from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..version import version
-from ..helpers import getLogger, extractOptions, splitWithOptions, splitwords
+from ..helpers import _merge_dicts, getLogger, extractOptions, splitWithOptions, splitwords
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
+class NoJailError(ValueError):
+	pass
+
 class JailReader(ConfigReader):
 	
 	def __init__(self, name, force_enable=False, **kwargs):
@@ -64,7 +67,7 @@ class JailReader(ConfigReader):
 		# Before returning -- verify that requested section
 		# exists at all
 		if not (self.__name in self.sections()):
-			raise ValueError("Jail %r was not found among available"
+			raise NoJailError("Jail %r was not found among available"
 							 % self.__name)
 		return out
 	
@@ -117,9 +120,9 @@ class JailReader(ConfigReader):
 	}
 	_configOpts.update(FilterReader._configOpts)
 
-	_ignoreOpts = set(['action', 'filter', 'enabled'] + list(FilterReader._configOpts.keys()))
+	_ignoreOpts = set(['action', 'filter', 'enabled', 'backend'] + list(FilterReader._configOpts.keys()))
 
-	def getOptions(self):
+	def getOptions(self, addOpts=None):
 
 		basedir = self.getBaseDir()
 
@@ -136,6 +139,8 @@ class JailReader(ConfigReader):
 				shouldExist=True)
 			if not self.__opts: # pragma: no cover
 				raise JailDefError("Init jail options failed")
+			if addOpts:
+				self.__opts = _merge_dicts(self.__opts, addOpts)
 		
 			if not self.isEnabled():
 				return True
@@ -147,6 +152,8 @@ class JailReader(ConfigReader):
 					filterName, filterOpt = extractOptions(flt)
 				except ValueError as e:
 					raise JailDefError("Invalid filter definition %r: %s" % (flt, e))
+				if addOpts:
+					filterOpt = _merge_dicts(filterOpt, addOpts)
 				self.__filter = FilterReader(
 					filterName, self.__name, filterOpt, 
 					share_config=self.share_config, basedir=basedir)
@@ -219,6 +226,15 @@ class JailReader(ConfigReader):
 			return False
 		return True
 	
+	@property
+	def filter(self):
+		return self.__filter
+
+	def getCombined(self):
+		if not self.__filter:
+			return self.__opts
+		return _merge_dicts(self.__opts, self.__filter.getCombined())
+
 	def convert(self, allow_no_files=False):
 		"""Convert read before __opts to the commands stream
 
@@ -240,9 +256,10 @@ class JailReader(ConfigReader):
 			stream.extend(self.__filter.convert())
 		# and using options from jail:
 		FilterReader._fillStream(stream, self.__opts, self.__name)
+		backend = self.__opts.get('backend', 'auto')
 		for opt, value in self.__opts.items():
 			if opt == "logpath":
-				if self.__opts.get('backend', '').startswith("systemd"): continue
+				if backend.startswith("systemd"): continue
 				found_files = 0
 				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
@@ -260,8 +277,6 @@ class JailReader(ConfigReader):
 					if not allow_no_files:
 						raise ValueError(msg)
 					logSys.warning(msg)
-			elif opt == "backend":
-				backend = value
 			elif opt == "ignoreip":
 				stream.append(["set", self.__name, "addignoreip"] + splitwords(value))
 			elif opt not in JailReader._ignoreOpts:

fail2ban/tests/fail2banregextestcase.py

@@ -258,7 +258,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		"-l", "notice", # put down log-level, because of too many debug-messages
 			"-v", "--verbose-date", "--print-all-matched", "--print-all-ignored",
 			"-c", CONFIG_DIR,
-			FILENAME_SSHD, "sshd"
+			FILENAME_SSHD, "sshd.conf"
 		))
 		# test failure line and not-failure lines both presents:
 		self.assertLogged("[29116]: User root not allowed because account is locked",
@@ -269,7 +269,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		"-l", "notice", # put down log-level, because of too many debug-messages
 			"-vv", "-c", CONFIG_DIR,
 			"Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.1",
-			"sshd[logtype=short]"
+			"filter.d/sshd[logtype=short]"
 		))
 		# tet logtype is specified and set in real options:
 		self.assertLogged("Real  filter options :", "'logtype': 'short'", all=True)
@@ -288,6 +288,16 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			"[29116]: User root not allowed because account is locked",
 			"[29116]: Received disconnect from 192.0.2.4", all=True)
 
+	def testLoadFromJail(self):
+		self.assertTrue(_test_exec(
+		"-l", "notice", # put down log-level, because of too many debug-messages
+			"-c", CONFIG_DIR, '-vv',
+			FILENAME_ZZZ_SSHD, "sshd[logtype=short]"
+		))
+		# test it was jail not filter:
+		self.assertLogged(
+			"Use %11s jail : %s" % ('','sshd'))
+
 	def testMultilineSshd(self):
 		# by the way test of missing lines by multiline in `for bufLine in orgLineBuffer[int(fullBuffer):]`
 		self.assertTrue(_test_exec(
@@ -431,11 +441,11 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		)
 		_test = lambda *args: _test_exec(*(opts + args))
 		# with MLFID from prefregex and IP after failure obtained from F-NOFAIL RE:
-		self.assertTrue(_test('-o', 'IP:<ip>', log, 'sshd'))
+		self.assertTrue(_test('-o', 'IP:<ip>', log, 'sshd.conf'))
 		self.assertLogged('IP:192.0.2.76')
 		self.pruneLog()
 		# test diverse ID/IP constellations:
-		def _test_variants(flt="sshd", prefix=""):
+		def _test_variants(flt="sshd.conf", prefix=""):
 			# with different ID/IP from failregex (ID/User from first, IP from second message):
 			self.assertTrue(_test('-o', 'ID:"<fid>" | IP:<ip> | U:<F-USER>', log, 
 				flt+'[failregex="'
@@ -455,7 +465,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		# first with sshd and prefregex:
 		_test_variants()
 		# the same without prefregex and MLFID directly in failregex (no merge with prefregex groups):
-		_test_variants('common', prefix=r"\s*\S+ sshd\[<F-MLFID>\d+</F-MLFID>\]:\s+")
+		_test_variants('common.conf', prefix=r"\s*\S+ sshd\[<F-MLFID>\d+</F-MLFID>\]:\s+")
 
 	def testNoDateTime(self):
 		# datepattern doesn't match:
@@ -490,7 +500,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		# complex substitution using tags and message (ip, user, msg):
 		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
 			'-c', CONFIG_DIR, '--usedns', 'no',
-			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]'))
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd.conf[logtype=short, publickey=invalid]'))
 		# be sure we don't have IP in one line and have it in another:
 		lines = STR_ML_SSHD.split("\n")
 		self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1])
@@ -506,7 +516,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)")
 		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
 			'-c', CONFIG_DIR, '--usedns', 'no',
-			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]'))
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd.conf[logtype=short, mode=aggressive]'))
 		self.assertLogged(
 			'192.0.2.2, git, '+lines[-4],
 			'192.0.2.2, git, '+lines[-3],
@@ -520,7 +530,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)")
 		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
 			'-c', CONFIG_DIR, '--usedns', 'no',
-			STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]'))
+			STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd.conf[logtype=short, mode=aggressive]'))
 		# 192.0.2.1 should be found for every failure (2x failed key + 1x closed):
 		lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:]
 		self.assertLogged(
@@ -541,7 +551,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			'svc[2] connect started 192.0.2.4\n'
 			'svc[2] connect authorized 192.0.2.4\n'
 			'svc[2] connect finished 192.0.2.4\n',
-			r'common[prefregex="^svc\[<F-MLFID>\d+</F-MLFID>\] connect <F-CONTENT>.+</F-CONTENT>$"'
+			r'common.conf[prefregex="^svc\[<F-MLFID>\d+</F-MLFID>\] connect <F-CONTENT>.+</F-CONTENT>$"'
 			', failregex="'
 			'^started\n'
 			'^<F-NOFAIL><F-MLFFORGET>finished</F-MLFFORGET></F-NOFAIL> <ADDR>\n'