From 781321d6092d415e079439389e6e6588b8feaaa7 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 31 Dec 2023 16:34:22 +0100 Subject: [PATCH 1/4] fail2ban-regex: loading parsing settings from jail now (by simple name it'd prefer jail to the filter now), fallback: - fail2ban-regex ... sshd + fail2ban-regex ... filter.d/sshd closes gh-2655 --- fail2ban/client/fail2banregex.py | 46 +++++++++++++++++++++++++++----- fail2ban/client/jailreader.py | 31 +++++++++++++++------ 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 5275a683..8e78a489 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -51,7 +51,7 @@ except ImportError: FilterSystemd = None from ..version import version, normVersion -from .filterreader import FilterReader +from .jailreader import FilterReader, JailReader, NoJailError from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException @@ -312,12 +312,18 @@ class Fail2banRegex(object): def _dumpRealOptions(self, reader, fltOpt): realopts = {} combopts = reader.getCombined() + if isinstance(reader, FilterReader): + _get_opt = lambda k: reader.get('Definition', k) + elif reader.filter: # JailReader for jail with filter: + _get_opt = lambda k: reader.filter.get('Definition', k) + else: # JailReader for jail without filter: + _get_opt = lambda k: None # output all options that are specified in filter-argument as well as some special (mostly interested): for k in ['logtype', 'datepattern'] + list(fltOpt.keys()): # combined options win, but they contain only a sub-set in filter expected keys, # so get the rest from definition section: try: - realopts[k] = combopts[k] if k in combopts else reader.get('Definition', k) + realopts[k] = combopts[k] if k in combopts else _get_opt(k) except NoOptionError: # pragma: no cover pass self.output("Real filter options : %r" % realopts) @@ -330,16 +336,26 @@ class Fail2banRegex(object): fltName = value fltFile = None fltOpt = {} + jail = None if regextype == 'fail': if re.search(r'(?ms)^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): try: fltName, fltOpt = extractOptions(value) + if re.search(r'(?ms)^[\w/_\-]+$', fltName): # name of jail? + try: + jail = JailReader(fltName, force_enable=True, + share_config=self.share_config) + jail.read() + except NoJailError: + jail = None if "." in fltName[~5:]: tryNames = (fltName,) else: tryNames = (fltName, fltName + '.conf', fltName + '.local') for fltFile in tryNames: - if not "/" in fltFile: + if os.path.dirname(fltFile) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + elif not "/" in fltFile: if os.path.basename(basedir) == 'filter.d': fltFile = os.path.join(basedir, fltFile) else: @@ -354,8 +370,25 @@ class Fail2banRegex(object): output(" while parsing: %s" % (value,)) if self._verbose: raise(e) return False + + readercommands = None + # if it is jail: + if jail: + self.output( "Use %11s jail : %s" % ('', fltName) ) + if fltOpt: + self.output( "Use jail/flt options : %r" % fltOpt ) + if not fltOpt: fltOpt = {} + fltOpt['backend'] = self._backend + ret = jail.getOptions(addOpts=fltOpt) + if not ret: + output('ERROR: Failed to get jail for %r' % (value,)) + return False + # show real options if expected: + if self._verbose > 1 or logSys.getEffectiveLevel()<=logging.DEBUG: + self._dumpRealOptions(jail, fltOpt) + readercommands = jail.convert(allow_no_files=True) # if it is filter file: - if fltFile is not None: + elif fltFile is not None: if (basedir == self._opts.config or os.path.basename(basedir) == 'filter.d' or ("." not in fltName[~5:] and "/" not in fltName) @@ -364,10 +397,10 @@ class Fail2banRegex(object): if os.path.basename(basedir) == 'filter.d': basedir = os.path.dirname(basedir) fltName = os.path.splitext(os.path.basename(fltName))[0] - self.output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) + self.output( "Use %11s file : %s, basedir: %s" % ('filter', fltName, basedir) ) else: ## foreign file - readexplicit this file and includes if possible: - self.output( "Use %11s file : %s" % (regex, fltName) ) + self.output( "Use %11s file : %s" % ('filter', fltName) ) basedir = None if not os.path.isabs(fltName): # avoid join with "filter.d" inside FilterReader fltName = os.path.abspath(fltName) @@ -398,6 +431,7 @@ class Fail2banRegex(object): # to stream: readercommands = reader.convert() + if readercommands: regex_values = {} for opt in readercommands: if opt[0] == 'multi-set': diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index d4ee3e8a..e7242bfd 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -29,16 +29,19 @@ import json import os.path import re -from .configreader import ConfigReaderUnshared, ConfigReader +from .configreader import ConfigReaderUnshared, ConfigReader, NoSectionError from .filterreader import FilterReader from .actionreader import ActionReader from ..version import version -from ..helpers import getLogger, extractOptions, splitWithOptions, splitwords +from ..helpers import _merge_dicts, getLogger, extractOptions, splitWithOptions, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) +class NoJailError(ValueError): + pass + class JailReader(ConfigReader): def __init__(self, name, force_enable=False, **kwargs): @@ -64,7 +67,7 @@ class JailReader(ConfigReader): # Before returning -- verify that requested section # exists at all if not (self.__name in self.sections()): - raise ValueError("Jail %r was not found among available" + raise NoJailError("Jail %r was not found among available" % self.__name) return out @@ -117,9 +120,9 @@ class JailReader(ConfigReader): } _configOpts.update(FilterReader._configOpts) - _ignoreOpts = set(['action', 'filter', 'enabled'] + list(FilterReader._configOpts.keys())) + _ignoreOpts = set(['action', 'filter', 'enabled', 'backend'] + list(FilterReader._configOpts.keys())) - def getOptions(self): + def getOptions(self, addOpts=None): basedir = self.getBaseDir() @@ -136,6 +139,8 @@ class JailReader(ConfigReader): shouldExist=True) if not self.__opts: # pragma: no cover raise JailDefError("Init jail options failed") + if addOpts: + self.__opts = _merge_dicts(self.__opts, addOpts) if not self.isEnabled(): return True @@ -147,6 +152,8 @@ class JailReader(ConfigReader): filterName, filterOpt = extractOptions(flt) except ValueError as e: raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) + if addOpts: + filterOpt = _merge_dicts(filterOpt, addOpts) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=basedir) @@ -219,6 +226,15 @@ class JailReader(ConfigReader): return False return True + @property + def filter(self): + return self.__filter + + def getCombined(self): + if not self.__filter: + return self.__opts + return _merge_dicts(self.__opts, self.__filter.getCombined()) + def convert(self, allow_no_files=False): """Convert read before __opts to the commands stream @@ -240,9 +256,10 @@ class JailReader(ConfigReader): stream.extend(self.__filter.convert()) # and using options from jail: FilterReader._fillStream(stream, self.__opts, self.__name) + backend = self.__opts.get('backend', 'auto') for opt, value in self.__opts.items(): if opt == "logpath": - if self.__opts.get('backend', '').startswith("systemd"): continue + if backend.startswith("systemd"): continue found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) @@ -260,8 +277,6 @@ class JailReader(ConfigReader): if not allow_no_files: raise ValueError(msg) logSys.warning(msg) - elif opt == "backend": - backend = value elif opt == "ignoreip": stream.append(["set", self.__name, "addignoreip"] + splitwords(value)) elif opt not in JailReader._ignoreOpts: From b3178851fe35711ec375b4a4e4db1ef2eb5dea1d Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 31 Dec 2023 17:03:38 +0100 Subject: [PATCH 2/4] test coverage (restore usage with filter and load setting from jail) --- fail2ban/tests/fail2banregextestcase.py | 28 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 9a9c7d50..c70e5fd6 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -258,7 +258,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "-l", "notice", # put down log-level, because of too many debug-messages "-v", "--verbose-date", "--print-all-matched", "--print-all-ignored", "-c", CONFIG_DIR, - FILENAME_SSHD, "sshd" + FILENAME_SSHD, "sshd.conf" )) # test failure line and not-failure lines both presents: self.assertLogged("[29116]: User root not allowed because account is locked", @@ -269,7 +269,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "-l", "notice", # put down log-level, because of too many debug-messages "-vv", "-c", CONFIG_DIR, "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.1", - "sshd[logtype=short]" + "filter.d/sshd[logtype=short]" )) # tet logtype is specified and set in real options: self.assertLogged("Real filter options :", "'logtype': 'short'", all=True) @@ -288,6 +288,16 @@ class Fail2banRegexTest(LogCaptureTestCase): "[29116]: User root not allowed because account is locked", "[29116]: Received disconnect from 192.0.2.4", all=True) + def testLoadFromJail(self): + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "-c", CONFIG_DIR, '-vv', + FILENAME_ZZZ_SSHD, "sshd[logtype=short]" + )) + # test it was jail not filter: + self.assertLogged( + "Use %11s jail : %s" % ('','sshd')) + def testMultilineSshd(self): # by the way test of missing lines by multiline in `for bufLine in orgLineBuffer[int(fullBuffer):]` self.assertTrue(_test_exec( @@ -431,11 +441,11 @@ class Fail2banRegexTest(LogCaptureTestCase): ) _test = lambda *args: _test_exec(*(opts + args)) # with MLFID from prefregex and IP after failure obtained from F-NOFAIL RE: - self.assertTrue(_test('-o', 'IP:', log, 'sshd')) + self.assertTrue(_test('-o', 'IP:', log, 'sshd.conf')) self.assertLogged('IP:192.0.2.76') self.pruneLog() # test diverse ID/IP constellations: - def _test_variants(flt="sshd", prefix=""): + def _test_variants(flt="sshd.conf", prefix=""): # with different ID/IP from failregex (ID/User from first, IP from second message): self.assertTrue(_test('-o', 'ID:"" | IP: | U:', log, flt+'[failregex="' @@ -455,7 +465,7 @@ class Fail2banRegexTest(LogCaptureTestCase): # first with sshd and prefregex: _test_variants() # the same without prefregex and MLFID directly in failregex (no merge with prefregex groups): - _test_variants('common', prefix=r"\s*\S+ sshd\[\d+\]:\s+") + _test_variants('common.conf', prefix=r"\s*\S+ sshd\[\d+\]:\s+") def testNoDateTime(self): # datepattern doesn't match: @@ -490,7 +500,7 @@ class Fail2banRegexTest(LogCaptureTestCase): # complex substitution using tags and message (ip, user, msg): self.assertTrue(_test_exec('-o', ', , ', '-c', CONFIG_DIR, '--usedns', 'no', - STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]')) + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd.conf[logtype=short, publickey=invalid]')) # be sure we don't have IP in one line and have it in another: lines = STR_ML_SSHD.split("\n") self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1]) @@ -506,7 +516,7 @@ class Fail2banRegexTest(LogCaptureTestCase): self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)") self.assertTrue(_test_exec('-o', ', , ', '-c', CONFIG_DIR, '--usedns', 'no', - STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]')) + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd.conf[logtype=short, mode=aggressive]')) self.assertLogged( '192.0.2.2, git, '+lines[-4], '192.0.2.2, git, '+lines[-3], @@ -520,7 +530,7 @@ class Fail2banRegexTest(LogCaptureTestCase): self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)") self.assertTrue(_test_exec('-o', ', , ', '-c', CONFIG_DIR, '--usedns', 'no', - STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]')) + STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd.conf[logtype=short, mode=aggressive]')) # 192.0.2.1 should be found for every failure (2x failed key + 1x closed): lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:] self.assertLogged( @@ -541,7 +551,7 @@ class Fail2banRegexTest(LogCaptureTestCase): 'svc[2] connect started 192.0.2.4\n' 'svc[2] connect authorized 192.0.2.4\n' 'svc[2] connect finished 192.0.2.4\n', - r'common[prefregex="^svc\[\d+\] connect .+$"' + r'common.conf[prefregex="^svc\[\d+\] connect .+$"' ', failregex="' '^started\n' '^finished \n' From cab6f93364c7853ddc6ebeeb2de972a80b07add4 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Jan 2024 13:17:37 +0100 Subject: [PATCH 3/4] fail2ban-regex: fixes forgotten basedir (-c "$basedir") of jailreader --- fail2ban/client/fail2banregex.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 8e78a489..160bde77 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -344,7 +344,7 @@ class Fail2banRegex(object): if re.search(r'(?ms)^[\w/_\-]+$', fltName): # name of jail? try: jail = JailReader(fltName, force_enable=True, - share_config=self.share_config) + share_config=self.share_config, basedir=basedir) jail.read() except NoJailError: jail = None @@ -406,7 +406,8 @@ class Fail2banRegex(object): fltName = os.path.abspath(fltName) if fltOpt: self.output( "Use filter options : %r" % fltOpt ) - reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) + reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, + share_config=self.share_config, basedir=basedir) ret = None try: if basedir is not None: From 302252b25c5e5f515b3552e31da33ca5fc8fb563 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 3 Jan 2024 13:37:45 +0100 Subject: [PATCH 4/4] ChangeLog, gh-2655 --- ChangeLog | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index 89a7cfed..9e99a4f6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,14 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition * `action.d/*ipset*`: make `maxelem` ipset option configurable through banaction arguments (gh-3564) ### New Features and Enhancements +* `fail2ban-regex` extended to load settings from jail (by simple name it'd prefer jail to the filter now, gh-2655); + to load the settings from filter one could use: +```diff +- fail2ban-regex ... sshd ; # jail ++ fail2ban-regex ... sshd.conf ; # filter +# or: ++ fail2ban-regex ... filter.d/sshd ; # filter +``` * better auto-detection for IPv6 support (`allowipv6 = auto` by default), trying to check sysctl net.ipv6.conf.all.disable_ipv6 (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces if available for platform and uses DNS to find local IPv6 as a fallback only