From 8614ca8c41a0d868f0d54d219666374216fb70d4 Mon Sep 17 00:00:00 2001 From: Shane Forsythe <2287983+shaneforsythe@users.noreply.github.com> Date: Tue, 2 Oct 2018 17:24:33 -0400 Subject: [PATCH 001/136] Update proftpd.conf proftpd 1.3.5e can leave inconsistent error message if ftp or mod_sftp is used Oct 2 15:45:31 ftp01 proftpd[5516]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted Oct 2 15:45:44 ftp01 proftpd[5517]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted. Fix regex to make trailing period optional, otherwise brute force attacks against root account using ftp are not blocked correctly. --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 303be5e5..feb59f11 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -18,7 +18,7 @@ __suffix_failed_login = (User not authorized for login|No such user found|Incorr failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\.? *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ ignoreregex = From b158f83aa3795f387c8475ceb48df197a94a37e8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 13 Jan 2020 12:37:19 +0100 Subject: [PATCH 002/136] testIPAddr_CompareDNS: add missing network constraint (gh-2596) --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index d6ad8235..6ca8162b 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -2064,6 +2064,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): + unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) From 31a6c8cf5d8897b957fa47edb8c4dcdd5ef57836 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 13 Jan 2020 20:12:16 +0100 Subject: [PATCH 003/136] closes gh-2599: fixes `splitwords` for unicode string --- fail2ban/helpers.py | 2 +- fail2ban/tests/misctestcase.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 213a405f..241543c1 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -291,7 +291,7 @@ def splitwords(s): """ if not s: return [] - return filter(bool, map(str.strip, re.split('[ ,\n]+', s))) + return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) if sys.version_info >= (3,5): eval(compile(r'''if 1: diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index cd27ad92..9b986f53 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -66,6 +66,8 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1, 2 , '), ['1', '2']) self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) + # string as unicode: + self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) if sys.version_info >= (2,7): From ec37b1942c4da76f7a0f71efe81bea6835466648 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Jan 2020 11:39:13 +0100 Subject: [PATCH 004/136] action.d/nginx-block-map.conf: fixed backslash substitution (different echo behavior in some shells, gh-2596) --- config/action.d/nginx-block-map.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index 0b6aa0ad..ee702907 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -103,6 +103,8 @@ actionstop = %(actionflush)s actioncheck = -actionban = echo "\\\\ 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s +_echo_blck_row = printf '\%%s 1;\n' "" -actionunban = id=$(echo "" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^\\\\$id 1;$/d" %(blck_lst_file)s; %(blck_lst_reload)s +actionban = %(_echo_blck_row)s >> '%(blck_lst_file)s'; %(blck_lst_reload)s + +actionunban = id=$(%(_echo_blck_row)s | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^$id$/d" %(blck_lst_file)s; %(blck_lst_reload)s From 8694c547285c4030d4bf7661981673038e6e9829 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Jan 2020 11:51:27 +0100 Subject: [PATCH 005/136] increase test stack size to 128K (on some platforms min size is greater then 32K), closes gh-2597 --- fail2ban/tests/fail2banclienttestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 29adb122..5caa4dd9 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -469,14 +469,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): @with_foreground_server_thread(startextra={'f2b_local':( "[Thread]", - "stacksize = 32" + "stacksize = 128" "", )}) def testStartForeground(self, tmp, startparams): # check thread options were set: self.pruneLog() self.execCmd(SUCCESS, startparams, "get", "thread") - self.assertLogged("{'stacksize': 32}") + self.assertLogged("{'stacksize': 128}") # several commands to server: self.execCmd(SUCCESS, startparams, "ping") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") From d4c921c22abd9cb80fb7351a0e1d292286705171 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Jan 2020 13:22:55 +0100 Subject: [PATCH 006/136] amend to 31b8d91ba2211595182d8d3fe6d89034b562aef0: tag `` is normally dynamic tag (ticket related), so better to replace it this way (may avoid confusing if tag is used directly during restore sane env process for both families); conditional replacement is not affected here --- fail2ban/server/action.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index e748fa77..2a5bb704 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -392,10 +392,13 @@ class CommandAction(ActionBase): def _getOperation(self, tag, family): # replace operation tag (interpolate all values), be sure family is enclosed as conditional value # (as lambda in addrepl so only if not overwritten in action): - return self.replaceTag(tag, self._properties, + cmd = self.replaceTag(tag, self._properties, conditional=('family='+family if family else ''), - addrepl=(lambda tag:family if tag == 'family' else None), cache=self.__substCache) + if '<' not in cmd or not family: return cmd + # replace family as dynamic tags, important - don't cache, no recursion and auto-escape here: + cmd = self.replaceDynamicTags(cmd, {'family':family}) + return cmd def _operationExecuted(self, tag, family, *args): """ Get, set or delete command of operation considering family. From 8dc6f30cdd855c41b80ebdde3fe2bc91cc94e594 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Jan 2020 19:22:53 +0100 Subject: [PATCH 007/136] closes #2596: fixed supplying of backend-related `logtype` to the jail filter - don't merge it (provide as init parameter if not set in definition section), init parameters don't affect config-cache (better implementation as in #2387 and it covered now with new test) --- MANIFEST | 2 ++ fail2ban/client/configreader.py | 8 +++-- fail2ban/client/fail2banregex.py | 7 ++--- fail2ban/client/filterreader.py | 8 +++++ fail2ban/client/jailreader.py | 7 ++--- fail2ban/tests/clientreadertestcase.py | 17 +++++++++- .../tests/config/filter.d/checklogtype.conf | 31 +++++++++++++++++++ .../config/filter.d/checklogtype_test.conf | 12 +++++++ fail2ban/tests/config/jail.conf | 25 +++++++++++++++ 9 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 fail2ban/tests/config/filter.d/checklogtype.conf create mode 100644 fail2ban/tests/config/filter.d/checklogtype_test.conf diff --git a/MANIFEST b/MANIFEST index dbcc2f60..5680492a 100644 --- a/MANIFEST +++ b/MANIFEST @@ -226,6 +226,8 @@ fail2ban/tests/clientreadertestcase.py fail2ban/tests/config/action.d/action.conf fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/checklogtype.conf +fail2ban/tests/config/filter.d/checklogtype_test.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 66b987b2..20709b72 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -120,6 +120,10 @@ class ConfigReader(): except AttributeError: return False + def has_option(self, sec, opt, withDefault=True): + return self._cfg.has_option(sec, opt) if withDefault \ + else opt in self._cfg._sections.get(sec, {}) + def merge_defaults(self, d): self._cfg.get_defaults().update(d) @@ -261,8 +265,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (optname, sec, optvalue)) values[optname] = optvalue - elif logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) + # elif logSys.getEffectiveLevel() <= logLevel: + # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index f6a4b141..334c031f 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -372,11 +372,8 @@ class Fail2banRegex(object): if not ret: output( "ERROR: failed to load filter %s" % value ) return False - # overwrite default logtype (considering that the filter could specify this too in Definition/Init sections): - if not fltOpt.get('logtype'): - reader.merge_defaults({ - 'logtype': ['file','journal'][int(self._backend.startswith("systemd"))] - }) + # set backend-related options (logtype): + reader.applyAutoOptions(self._backend) # get, interpolate and convert options: reader.getOptions(None) # show real options if expected: diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index ede18dca..413f125e 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -53,6 +53,14 @@ class FilterReader(DefinitionInitConfigReader): def getFile(self): return self.__file + def applyAutoOptions(self, backend): + # set init option to backend-related logtype, considering + # that the filter settings may be overwritten in its local: + if (not self._initOpts.get('logtype') and + not self.has_option('Definition', 'logtype', False) + ): + self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))] + def convert(self): stream = list() opts = self.getCombined() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 917a562c..1d7db0dc 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -142,11 +142,8 @@ class JailReader(ConfigReader): ret = self.__filter.read() if not ret: raise JailDefError("Unable to read the filter %r" % filterName) - if not filterOpt.get('logtype'): - # overwrite default logtype backend-related (considering that the filter settings may be overwritten): - self.__filter.merge_defaults({ - 'logtype': ['file','journal'][int(self.__opts.get('backend', '').startswith("systemd"))] - }) + # set backend-related options (logtype): + self.__filter.applyAutoOptions(self.__opts.get('backend', '')) # merge options from filter as 'known/...' (all options unfiltered): self.__filter.getOptions(self.__opts, all=True) ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d39860f4..2c1d0a0e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -328,7 +328,22 @@ class JailReaderTest(LogCaptureTestCase): self.assertFalse(len(o) > 2 and o[2].endswith('regex')) i += 1 if i > usednsidx: break - + + def testLogTypeOfBackendInJail(self): + unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf + # test twice to check cache works peoperly: + for i in (1, 2): + # backend-related, overwritten in definition, specified in init parameters: + for prefline in ('JRNL', 'FILE', 'TEST', 'INIT'): + jail = JailReader('checklogtype_'+prefline.lower(), basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + stream = jail.convert() + # 'JRNL' for systemd, 'FILE' for file backend, 'TEST' for custom logtype (overwrite it): + self.assertEqual([['set', jail.getName(), 'addfailregex', '^%s failure from $' % prefline]], + [o for o in stream if len(o) > 2 and o[2] == 'addfailregex']) + def testSplitOption(self): # Simple example option = "mail-whois[name=SSH]" diff --git a/fail2ban/tests/config/filter.d/checklogtype.conf b/fail2ban/tests/config/filter.d/checklogtype.conf new file mode 100644 index 00000000..4d700fff --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype.conf @@ -0,0 +1,31 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = ../../../../config/filter.d/common.conf + +[Definition] + +_daemon = test + +failregex = ^/__prefix_line> failure from $ +ignoreregex = + +# following sections define prefix line considering logtype: + +# backend-related (retrieved from backend, overwrite default): +[lt_file] +__prefix_line = FILE + +[lt_journal] +__prefix_line = JRNL + +# specified in definition section of filter (see filter checklogtype_test.conf): +[lt_test] +__prefix_line = TEST + +# specified in init parameter of jail (see ../jail.conf, jail checklogtype_init): +[lt_init] +__prefix_line = INIT diff --git a/fail2ban/tests/config/filter.d/checklogtype_test.conf b/fail2ban/tests/config/filter.d/checklogtype_test.conf new file mode 100644 index 00000000..a76f5fcf --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype_test.conf @@ -0,0 +1,12 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = checklogtype.conf + +[Definition] + +# overwrite logtype in definition (no backend anymore): +logtype = test \ No newline at end of file diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index de5bbbdc..b1a1707b 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -74,3 +74,28 @@ journalmatch = _COMM=test maxlines = 2 usedns = no enabled = false + +[checklogtype_jrnl] +filter = checklogtype +backend = systemd +action = action +enabled = false + +[checklogtype_file] +filter = checklogtype +backend = polling +logpath = README.md +action = action +enabled = false + +[checklogtype_test] +filter = checklogtype_test +backend = systemd +action = action +enabled = false + +[checklogtype_init] +filter = checklogtype_test[logtype=init] +backend = systemd +action = action +enabled = false From 3965d690b137152b2a0a6a46989178b5566cfd8e Mon Sep 17 00:00:00 2001 From: Angelo Compagnucci Date: Thu, 16 Jan 2020 12:05:13 +0100 Subject: [PATCH 008/136] Revert "setup.py: adding option to install without tests" Test should actually removed from the stup data in finalize_options instead of being added back. This reverts commit 9b918bba2f672780fb4469294d80ba7deb6b8cab. Signed-off-by: Angelo Compagnucci --- setup.py | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/setup.py b/setup.py index e476c5dd..8da29268 100755 --- a/setup.py +++ b/setup.py @@ -119,11 +119,9 @@ class install_scripts_f2b(install_scripts): class install_command_f2b(install): user_options = install.user_options + [ ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), - ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): self.disable_2to3 = None - self.without_tests = None install.initialize_options(self) def finalize_options(self): global _2to3 @@ -134,28 +132,6 @@ class install_command_f2b(install): cmdclass = self.distribution.cmdclass cmdclass['build_py'] = build_py_2to3 cmdclass['build_scripts'] = build_scripts_2to3 - if not self.without_tests: - self.distribution.scripts += [ - 'bin/fail2ban-testcases', - ] - - self.distribution.packages += [ - 'fail2ban.tests', - 'fail2ban.tests.action_d', - ] - - self.distribution.package_data = { - 'fail2ban.tests': - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/files') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/config') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/action_d') - for f in w[2]] - } install.finalize_options(self) def run(self): install.run(self) @@ -232,20 +208,35 @@ setup( license = "GPL", platforms = "Posix", cmdclass = { - 'build_py': build_py, 'build_scripts': build_scripts, + 'build_py': build_py, 'build_scripts': build_scripts, 'install_scripts': install_scripts_f2b, 'install': install_command_f2b }, scripts = [ 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', + 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper ], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + 'fail2ban.tests', + 'fail2ban.tests.action_d', ], + package_data = { + 'fail2ban.tests': + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/files') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/config') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/action_d') + for f in w[2]] + }, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From 5fa1f69264d3c23793f64c03c96737d54555e919 Mon Sep 17 00:00:00 2001 From: Angelo Compagnucci Date: Thu, 16 Jan 2020 12:28:42 +0100 Subject: [PATCH 009/136] setup.py: adding option to install without tests Tests files are not always needed especially when installing on low resource systems like an embedded one. This patch adds the --without-tests option to skip installing the tests files. Signed-off-by: Angelo Compagnucci --- setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8da29268..ce1eedf6 100755 --- a/setup.py +++ b/setup.py @@ -119,9 +119,11 @@ class install_scripts_f2b(install_scripts): class install_command_f2b(install): user_options = install.user_options + [ ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), + ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): self.disable_2to3 = None + self.without_tests = None install.initialize_options(self) def finalize_options(self): global _2to3 @@ -132,6 +134,13 @@ class install_command_f2b(install): cmdclass = self.distribution.cmdclass cmdclass['build_py'] = build_py_2to3 cmdclass['build_scripts'] = build_scripts_2to3 + if self.without_tests: + self.distribution.scripts.remove('bin/fail2ban-testcases') + + self.distribution.packages.remove('fail2ban.tests') + self.distribution.packages.remove('fail2ban.tests.action_d') + + del self.distribution.package_data['fail2ban.tests'] install.finalize_options(self) def run(self): install.run(self) @@ -208,7 +217,7 @@ setup( license = "GPL", platforms = "Posix", cmdclass = { - 'build_py': build_py, 'build_scripts': build_scripts, + 'build_py': build_py, 'build_scripts': build_scripts, 'install_scripts': install_scripts_f2b, 'install': install_command_f2b }, scripts = [ From 3befbb177017957869425c81a560edb8e27db75a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Jan 2020 16:45:01 +0100 Subject: [PATCH 010/136] improved wait for observer stop on server quit (second stop would force quit), this also cause reset db in observer (to avoid out of sequence errors) before database gets ultimately closed at end of server stop process (gh-2608) --- fail2ban/server/observer.py | 25 +++++++++++------- fail2ban/server/server.py | 17 +++++++------ fail2ban/tests/fail2banclienttestcase.py | 32 ++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index bd7cbe4a..c19549ba 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -146,9 +146,11 @@ class ObserverThread(JailThread): def pulse_notify(self): """Notify wakeup (sets /and resets/ notify event) """ - if not self._paused and self._notify: - self._notify.set() - #self._notify.clear() + if not self._paused: + n = self._notify + if n: + n.set() + #n.clear() def add(self, *event): """Add a event to queue and notify thread to wake up. @@ -237,6 +239,7 @@ class ObserverThread(JailThread): break ## end of main loop - exit logSys.info("Observer stopped, %s events remaining.", len(self._queue)) + self._notify = None #print("Observer stopped, %s events remaining." % len(self._queue)) except Exception as e: logSys.error('Observer stopped after error: %s', e, exc_info=True) @@ -262,9 +265,8 @@ class ObserverThread(JailThread): if not self.active: super(ObserverThread, self).start() - def stop(self): + def stop(self, wtime=5, forceQuit=True): if self.active and self._notify: - wtime = 5 logSys.info("Observer stop ... try to end queue %s seconds", wtime) #print("Observer stop ....") # just add shutdown job to make possible wait later until full (events remaining) @@ -276,10 +278,15 @@ class ObserverThread(JailThread): #self.pulse_notify() self._notify = None # wait max wtime seconds until full (events remaining) - self.wait_empty(wtime) - n.clear() - self.active = False - self.wait_idle(0.5) + if self.wait_empty(wtime) or forceQuit: + n.clear() + self.active = False; # leave outer (active) loop + self._paused = True; # leave inner (queue) loop + self.__db = None + else: + self._notify = n + return self.wait_idle(min(wtime, 0.5)) and not self.is_full + return True @property def is_full(self): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 15265822..feb3b399 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -193,23 +193,26 @@ class Server: signal.signal(s, sh) # Give observer a small chance to complete its work before exit - if Observers.Main is not None: - Observers.Main.stop() + obsMain = Observers.Main + if obsMain is not None: + if obsMain.stop(forceQuit=False): + obsMain = None + Observers.Main = None # Now stop all the jails self.stopAllJail() + # Stop observer ultimately + if obsMain is not None: + obsMain.stop() + # Explicit close database (server can leave in a thread, # so delayed GC can prevent commiting changes) if self.__db: self.__db.close() self.__db = None - # Stop observer and exit - if Observers.Main is not None: - Observers.Main.stop() - Observers.Main = None - # Stop async + # Stop async and exit if self.__asyncServer is not None: self.__asyncServer.stop() self.__asyncServer = None diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 5caa4dd9..95f73ed3 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -343,6 +343,7 @@ def with_foreground_server_thread(startextra={}): # to wait for end of server, default accept any exit code, because multi-threaded, # thus server can exit in-between... def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): + tearDownMyTime() # if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state): if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")): Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME) @@ -1570,6 +1571,37 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "192.0.2.11", "+ 600 =", all=True, wait=MID_WAITTIME) + # test stop with busy observer: + self.pruneLog("[test-phase end) stop on busy observer]") + tearDownMyTime() + a = {'state': 0} + obsMain = Observers.Main + def _long_action(): + logSys.info('++ observer enters busy state ...') + a['state'] = 1 + Utils.wait_for(lambda: a['state'] == 2, MAX_WAITTIME) + obsMain.db_purge(); # does nothing (db is already None) + logSys.info('-- observer leaves busy state.') + obsMain.add('call', _long_action) + obsMain.add('call', lambda: None) + # wait observer enter busy state: + Utils.wait_for(lambda: a['state'] == 1, MAX_WAITTIME) + # overwrite default wait time (normally 5 seconds): + obsMain_stop = obsMain.stop + def _stop(wtime=(0.01 if unittest.F2B.fast else 0.1), forceQuit=True): + return obsMain_stop(wtime, forceQuit) + obsMain.stop = _stop + # stop server and wait for end: + self.stopAndWaitForServerEnd(SUCCESS) + # check observer and db state: + self.assertNotLogged('observer leaves busy state') + self.assertFalse(obsMain.idle) + self.assertEqual(obsMain._ObserverThread__db, None) + # server is exited without wait for observer, stop it now: + a['state'] = 2 + self.assertLogged('observer leaves busy state', wait=True) + obsMain.join() + # test multiple start/stop of the server (threaded in foreground) -- if False: # pragma: no cover @with_foreground_server_thread() From 9e6d07d928d1016be8195613aac2c0a2b53b9342 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 22 Jan 2020 17:19:35 +0100 Subject: [PATCH 011/136] testSampleRegexsFactory: `time` is not mandatory anymore (check time only if set in json), allows usage of same line(s) matching different `logtype` option: `# filterOptions: [{"logtype": "file"}, {"logtype": "short"}, {"logtype": "journal"}]` --- fail2ban/tests/samplestestcase.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 1039b65e..0bbd05f5 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -223,11 +223,9 @@ def testSampleRegexsFactory(name, basedir): try: fail = {} # for logtype "journal" we don't need parse timestamp (simulate real systemd-backend handling): - checktime = True if opts.get('logtype') != 'journal': ret = flt.processLine(line) else: # simulate journal processing, time is known from journal (formatJournalEntry): - checktime = False if opts.get('test.prefix-line'): # journal backends creates common prefix-line: line = opts.get('test.prefix-line') + line ret = flt.processLine(('', TEST_NOW_STR, line.rstrip('\r\n')), TEST_NOW) @@ -271,7 +269,7 @@ def testSampleRegexsFactory(name, basedir): self.assertEqual(fv, v) t = faildata.get("time", None) - if checktime or t is not None: + if t is not None: try: jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") except ValueError: From 569dea2b197b5a6b4c9f7b6450d6a87da58586ae Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 22 Jan 2020 17:24:40 +0100 Subject: [PATCH 012/136] filter.d/mysqld-auth.conf: capture user name in filter (can be more strict if user switched, used in action or fail2ban-regex output); also add coverage for mariadb 10.4 log format (gh-2611) --- config/filter.d/mysqld-auth.conf | 2 +- fail2ban/tests/files/logs/mysqld-auth | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 97b37920..930c9b5a 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index 0b0827f9..29faeb71 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -33,3 +33,7 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for 2019-09-06T01:45:18 srv mysqld: 2019-09-06 1:45:18 140581192722176 [Warning] Access denied for user 'global'@'192.0.2.2' (using password: YES) # failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.0.2.3", "desc": "ISO timestamp within log message" } 2019-09-24T13:16:50 srv mysqld[1234]: 2019-09-24 13:16:50 8756 [Warning] Access denied for user 'root'@'192.0.2.3' (using password: YES) + +# filterOptions: [{"logtype": "file"}, {"logtype": "short"}, {"logtype": "journal"}] +# failJSON: { "match": true , "host": "192.0.2.1", "user":"root", "desc": "mariadb 10.4 log format, gh-2611" } +2020-01-16 21:34:14 4644 [Warning] Access denied for user 'root'@'192.0.2.1' (using password: YES) From cd42cb26d6aac9454ced9b0ae3362ad16f74f192 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 27 Jan 2020 12:57:29 +0100 Subject: [PATCH 013/136] database: try to fix `out of sequence` error on some old platform / sqlite versions (#2613) - repack iterator as long as in lock (although dirty read has no matter here and only writing operations should be serialized, but to be sure and exclude this as source of that errors). --- fail2ban/server/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 0dd9acb6..19049e13 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -754,7 +754,8 @@ class Fail2BanDb(object): if overalljails or jail is None: query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" cur = self._db.cursor() - return cur.execute(query, queryArgs) + # repack iterator as long as in lock: + return list(cur.execute(query, queryArgs)) def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None): queryArgs = [] From 12b3ac684a637e2adfef05997d4aa3ad36b68970 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Jan 2020 21:45:30 +0100 Subject: [PATCH 014/136] closes #2615: systemd backend would seek to last known position (or `now - findtime`) in journal at start. --- fail2ban/server/database.py | 64 +++++++++++++++++++++++------- fail2ban/server/filtersystemd.py | 36 +++++++++++------ fail2ban/server/jail.py | 4 ++ fail2ban/tests/databasetestcase.py | 9 +++++ fail2ban/tests/dummyjail.py | 11 +---- fail2ban/tests/filtertestcase.py | 49 ++++++++++++++++++++++- 6 files changed, 135 insertions(+), 38 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index f584d74f..ab0de565 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -455,22 +455,24 @@ class Fail2BanDb(object): If log was already present in database, value of last position in the log file; else `None` """ + return self._addLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) + + def _addLog(self, cur, jail, name, pos=0, md5=None): lastLinePos = None cur.execute( "SELECT firstlinemd5, lastfilepos FROM logs " "WHERE jail=? AND path=?", - (jail.name, container.getFileName())) + (jail.name, name)) try: firstLineMD5, lastLinePos = cur.fetchone() except TypeError: - firstLineMD5 = False + firstLineMD5 = None - cur.execute( - "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " - "VALUES(?, ?, ?, ?)", - (jail.name, container.getFileName(), - container.getHash(), container.getPos())) - if container.getHash() != firstLineMD5: + if not firstLineMD5 and (pos or md5): + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + if md5 is not None and md5 != firstLineMD5: lastLinePos = None return lastLinePos @@ -499,7 +501,7 @@ class Fail2BanDb(object): return set(row[0] for row in cur.fetchmany()) @commitandrollback - def updateLog(self, cur, *args, **kwargs): + def updateLog(self, cur, jail, container): """Updates hash and last position in log file. Parameters @@ -509,14 +511,48 @@ class Fail2BanDb(object): container : FileContainer File container of the log file being updated. """ - self._updateLog(cur, *args, **kwargs) + self._updateLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) - def _updateLog(self, cur, jail, container): + def _updateLog(self, cur, jail, name, pos, md5): cur.execute( "UPDATE logs SET firstlinemd5=?, lastfilepos=? " - "WHERE jail=? AND path=?", - (container.getHash(), container.getPos(), - jail.name, container.getFileName())) + "WHERE jail=? AND path=?", (md5, pos, jail.name, name)) + # be sure it is set (if not available): + if not cur.rowcount: + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + + @commitandrollback + def getJournalPos(self, cur, jail, name, time=0, iso=None): + """Get journal position from database. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + + Returns + ------- + int (or float) + Last position (as time) if it was already present in database; else `None` + """ + return self._addLog(cur, jail, name, time, iso); # no hash, just time as iso + + @commitandrollback + def updateJournal(self, cur, jail, name, time, iso): + """Updates last position (as time) of journal. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + """ + self._updateLog(cur, jail, name, time, iso); # no hash, just time as iso @commitandrollback def addBan(self, cur, jail, ticket): diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index c2a72598..870b3058 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -190,6 +190,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalReader(self): return self.__journal + def getJrnEntTime(self, logentry): + """ Returns time of entry as tuple (ISO-str, Posix).""" + date = logentry.get('_SOURCE_REALTIME_TIMESTAMP') + if date is None: + date = logentry.get('__REALTIME_TIMESTAMP') + return (date.isoformat(), time.mktime(date.timetuple()) + date.microsecond/1.0E6) + ## # Format journal log entry into syslog style # @@ -222,9 +229,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements[-1] += v logelements[-1] += ":" if logelements[-1] == "kernel:": - if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: - monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') - else: + monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') + if monotonic is None: monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] logelements.append("[%12.6f]" % monotonic.total_seconds()) msg = logentry.get('MESSAGE','') @@ -235,13 +241,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logline = " ".join(logelements) - date = logentry.get('_SOURCE_REALTIME_TIMESTAMP', - logentry.get('__REALTIME_TIMESTAMP')) + date = self.getJrnEntTime(logentry) logSys.log(5, "[%s] Read systemd journal entry: %s %s", self.jailName, - date.isoformat(), logline) + date[0], logline) ## use the same type for 1st argument: - return ((logline[:0], date.isoformat(), logline.replace('\n', '\\n')), - time.mktime(date.timetuple()) + date.microsecond/1.0E6) + return ((logline[:0], date[0], logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): if not isinstance(date, datetime.datetime): @@ -262,9 +266,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover "Jail regexs will be checked against all journal entries, " "which is not advised for performance reasons.") - # Seek to now - findtime in journal - start_time = datetime.datetime.now() - \ - datetime.timedelta(seconds=int(self.getFindTime())) + # Try to obtain the last known time (position of journal) + start_time = 0 + if self.jail.database is not None: + start_time = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0 + # Seek to max(last_known_time, now - findtime) in journal + start_time = max( start_time, MyTime.time() - int(self.getFindTime()) ) self.seekToTime(start_time) # Move back one entry to ensure do not end up in dead space # if start time beyond end of journal @@ -303,8 +310,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) self.ticks += 1 if logentry: - self.processLineAndAdd( - *self.formatJournalEntry(logentry)) + line = self.formatJournalEntry(logentry) + self.processLineAndAdd(*line) self.__modified += 1 if self.__modified >= 100: # todo: should be configurable break @@ -313,6 +320,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if self.__modified: self.performBan() self.__modified = 0 + # update position in log (time and iso string): + if self.jail.database is not None: + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 1deea49b..89912556 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -156,6 +156,10 @@ class Jail(object): """ return self.__db + @database.setter + def database(self, value): + self.__db = value; + @property def filter(self): """The filter which the jail is using to monitor log files. diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index af9298a2..06f92d8c 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -225,6 +225,15 @@ class DatabaseTest(LogCaptureTestCase): self.db.addLog(self.jail, self.fileContainer), None) os.remove(filename) + def testUpdateJournal(self): + self.testAddJail() # Jail required + # not yet updated: + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), None) + # update 3 times (insert and 2 updates) and check it was set (and overwritten): + for t in (1500000000, 1500000001, 1500000002): + self.db.updateJournal(self.jail, 'systemd-journal', t, 'TEST'+str(t)) + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), t) + def testAddBan(self): self.testAddJail() ticket = FailTicket("127.0.0.1", 0, ["abc\n"]) diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 19f97f4e..9e9aaeed 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -35,7 +35,6 @@ class DummyJail(Jail): self.lock = Lock() self.queue = [] super(DummyJail, self).__init__(name='DummyJail', backend=backend) - self.__db = None self.__actions = Actions(self) def __len__(self): @@ -63,7 +62,7 @@ class DummyJail(Jail): @property def name(self): - return "DummyJail #%s with %d tickets" % (id(self), len(self)) + return "DummyJail" + ("" if self.database else " #%s with %d tickets" % (id(self), len(self))) @property def idle(self): @@ -73,14 +72,6 @@ class DummyJail(Jail): def idle(self, value): pass - @property - def database(self): - return self.__db; - - @database.setter - def database(self, value): - self.__db = value; - @property def actions(self): return self.__actions; diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 6ca8162b..91cb7eb6 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -43,7 +43,8 @@ from ..server.failmanager import FailManagerEmpty from ..server.ipdns import asip, getfqdn, DNSUtils, IPAddr from ..server.mytime import MyTime from ..server.utils import Utils, uni_decode -from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase, \ +from .databasetestcase import getFail2BanDb +from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_alt_time, with_tmpdir, LogCaptureTestCase, \ logSys as DefLogSys, CONFIG_DIR as STOCK_CONF_DIR from .dummyjail import DummyJail @@ -1397,6 +1398,52 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=5, n=4) self.assert_correct_ban("193.168.0.128", 3) + @with_alt_time + def test_grow_file_with_db(self): + + def _gen_falure(ip): + # insert new failures ans check it is monitored: + fields = self.journal_fields + fields.update(TEST_JOURNAL_FIELDS) + journal.send(MESSAGE="error: PAM: Authentication failure for test from "+ip, **fields) + self.waitForTicks(1) + self.assert_correct_ban(ip, 1) + + # coverage for update log: + self.jail.database = getFail2BanDb(':memory:') + self.jail.database.addJail(self.jail) + MyTime.setTime(time.time()) + self._test_grow_file() + # stop: + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 2) + # update log manually (should cause a seek to end of log without wait for next second): + self.jail.database.updateJournal(self.jail, 'systemd-journal', MyTime.time(), 'TEST') + # check seek to last (simulated) position succeeds (without bans of previous copied tickets): + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(1) + # check new IP but no old IPs found: + _gen_falure("192.0.2.5") + self.assertFalse(self.jail.getFailTicket()) + + # now the same with increased time (check now - findtime case): + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 10000) + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(1) + MyTime.setTime(time.time() + 3) + # check new IP but no old IPs found: + _gen_falure("192.0.2.6") + self.assertFalse(self.jail.getFailTicket()) + def test_delJournalMatch(self): self._initFilter() self.filter.start() From 9c7bd8080762d63610b8134f84163e45547616ba Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Feb 2020 20:09:13 +0100 Subject: [PATCH 015/136] fail2ban-regex: stop endless logging on closed streams (redirected pipes like `... | head -n 100`), exit if stdout channel is closed --- fail2ban/client/fail2banregex.py | 1 + fail2ban/helpers.py | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 334c031f..513b765d 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -709,6 +709,7 @@ class Fail2banRegex(object): def exec_command_line(*args): + logging.exitOnIOError = True parser = get_opt_parser() (opts, args) = parser.parse_args(*args) errors = [] diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 241543c1..6f2bcdd7 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -208,6 +208,25 @@ class FormatterWithTraceBack(logging.Formatter): return logging.Formatter.format(self, record) +logging.exitOnIOError = False +def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover + if logSys and len(logSys.handlers): + logSys.removeHandler(logSys.handlers[0]) + if logHndlr: + logHndlr.close = lambda: None + logging.StreamHandler.flush = lambda self: None + #sys.excepthook = lambda *args: None + if logging.exitOnIOError: + try: + sys.stderr.close() + except: + pass + sys.exit(0) + +try: + BrokenPipeError +except NameError: # pragma: 3.x no cover + BrokenPipeError = IOError __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, @@ -223,6 +242,10 @@ def __safeLog(self, level, msg, args, **kwargs): try: # if isEnabledFor(level) already called... __origLog(self, level, msg, args, **kwargs) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(self) + raise except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version try: for args in ( @@ -237,6 +260,18 @@ def __safeLog(self, level, msg, args, **kwargs): pass logging.Logger._log = __safeLog +__origLogFlush = logging.StreamHandler.flush +def __safeLogFlush(self): + """Safe flush inject stopping endless logging on closed streams (redirected pipe). + """ + try: + __origLogFlush(self) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(None, self) + raise +logging.StreamHandler.flush = __safeLogFlush + def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ From 3f489070646b363aa0374681fe910f05521cd247 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 Feb 2020 11:08:01 +0100 Subject: [PATCH 016/136] amend to f3dbc9dda10e52610e3de26f538b5581fd905505: change main thread-name back to `fail2ban-server`; implements new command line option `--pname` to specify it by start of server (default `fail2ban-server`); closes gh-2623 (revert change of main thread-name, because it can affect process-name too, so `pgrep` & co. may be confused) --- fail2ban/client/fail2bancmdline.py | 3 ++- fail2ban/server/server.py | 5 +++-- man/fail2ban-client.1 | 5 ++++- man/fail2ban-server.1 | 5 ++++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 1268ee9f..3c6bd0bf 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -97,6 +97,7 @@ class Fail2banCmdLine(): output(" -c configuration directory") output(" -s socket path") output(" -p pidfile path") + output(" --pname name of the process (main thread) to identify instance (default fail2ban-server)") output(" --loglevel logging level") output(" --logtarget logging target, use file-name or stdout, stderr, syslog or sysout.") output(" --syslogsocket auto|") @@ -185,7 +186,7 @@ class Fail2banCmdLine(): try: cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'pname=', 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 7c820e49..22814280 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -80,8 +80,6 @@ class Server: 'Linux': '/dev/log', } self.__prev_signals = {} - # replace real thread name with short process name (for top/ps/pstree or diagnostic): - prctl_set_th_name('f2b/server') def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested logSys.debug("Caught signal %d. Exiting", signum) @@ -112,6 +110,9 @@ class Server: logSys.error(err) raise ServerInitializationError(err) # We are daemon. + + # replace main thread (and process) name to identify server (for top/ps/pstree or diagnostic): + prctl_set_th_name(conf.get("pname", "fail2ban-server")) # Set all logging parameters (or use default if not specified): self.__verbose = conf.get("verbose", None) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 9ea61084..32a90851 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "January 2020" "fail2ban-client v0.10.5" "User Commands" +.TH FAIL2BAN-CLIENT "1" "February 2020" "fail2ban-client v0.10.5" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS @@ -19,6 +19,9 @@ socket path \fB\-p\fR pidfile path .TP +\fB\-\-pname\fR +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR logging level .TP diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index ddf9b303..d75158a8 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "January 2020" "fail2ban-server v0.10.5" "User Commands" +.TH FAIL2BAN-SERVER "1" "February 2020" "fail2ban-server v0.10.5" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS @@ -19,6 +19,9 @@ socket path \fB\-p\fR pidfile path .TP +\fB\-\-pname\fR +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR logging level .TP From 7a28861fc709d488c59a28ecf58e4ef5e5b79f4d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 7 Feb 2020 13:52:45 +0100 Subject: [PATCH 017/136] review of command line: more long-named options can be supplied via command line --- fail2ban/client/fail2bancmdline.py | 33 ++++++++++++++++-------------- man/fail2ban-client.1 | 6 +++--- man/fail2ban-server.1 | 6 +++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 3c6bd0bf..53c86de6 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -35,7 +35,8 @@ logSys = getLogger("fail2ban") def output(s): # pragma: no cover print(s) -CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True @@ -94,9 +95,9 @@ class Fail2banCmdLine(): output("and bans the corresponding IP addresses using firewall rules.") output("") output("Options:") - output(" -c configuration directory") - output(" -s socket path") - output(" -p pidfile path") + output(" -c, --conf configuration directory") + output(" -s, --socket socket path") + output(" -p, --pidfile pidfile path") output(" --pname name of the process (main thread) to identify instance (default fail2ban-server)") output(" --loglevel logging level") output(" --logtarget logging target, use file-name or stdout, stderr, syslog or sysout.") @@ -130,17 +131,15 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "-c": + if o in ("-c", "--conf"): self._conf["conf"] = opt[1] - elif o == "-s": + elif o in ("-s", "--socket"): self._conf["socket"] = opt[1] - elif o == "-p": + elif o in ("-p", "--pidfile"): self._conf["pidfile"] = opt[1] - elif o.startswith("--log") or o.startswith("--sys"): - self._conf[ o[2:] ] = opt[1] - elif o in ["-d", "--dp", "--dump-pretty"]: + elif o in ("-d", "--dp", "--dump-pretty"): self._conf["dump"] = True if o == "-d" else 2 - elif o == "-t" or o == "--test": + elif o in ("-t", "--test"): self.cleanConfOnly = True self._conf["test"] = True elif o == "-v": @@ -164,12 +163,14 @@ class Fail2banCmdLine(): from ..server.mytime import MyTime output(MyTime.str2seconds(opt[1])) return True - elif o in ["-h", "--help"]: + elif o in ("-h", "--help"): self.dispUsage() return True - elif o in ["-V", "--version"]: + elif o in ("-V", "--version"): self.dispVersion(o == "-V") return True + elif o.startswith("--"): # other long named params (see also resetConf) + self._conf[ o[2:] ] = opt[1] return None def initCmdLine(self, argv): @@ -186,7 +187,8 @@ class Fail2banCmdLine(): try: cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', - 'pname=', 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'conf=', 'pidfile=', 'pname=', 'socket=', + 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -228,7 +230,8 @@ class Fail2banCmdLine(): if not conf: self.configurator.readEarly() conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + if o in conf: + self._conf[o] = conf[o] logSys.info("Using socket file %s", self._conf["socket"]) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 32a90851..ad4fa0ed 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -10,13 +10,13 @@ Fail2Ban v0.10.5 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR +\fB\-c\fR, \fB\-\-conf\fR configuration directory .TP -\fB\-s\fR +\fB\-s\fR, \fB\-\-socket\fR socket path .TP -\fB\-p\fR +\fB\-p\fR, \fB\-\-pidfile\fR pidfile path .TP \fB\-\-pname\fR diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index d75158a8..c7516cc8 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -10,13 +10,13 @@ Fail2Ban v0.10.5 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR +\fB\-c\fR, \fB\-\-conf\fR configuration directory .TP -\fB\-s\fR +\fB\-s\fR, \fB\-\-socket\fR socket path .TP -\fB\-p\fR +\fB\-p\fR, \fB\-\-pidfile\fR pidfile path .TP \fB\-\-pname\fR From 34d63fccfe794030bb044ee175aef8560b41f769 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Feb 2020 13:03:55 +0100 Subject: [PATCH 018/136] close gh-2629 - jail.conf (action_blocklist_de interpolation): replace service parameter (use jail name instead of filter, which can be empty) --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index fbc357f7..f7c84fac 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -202,7 +202,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in # corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] # Report ban via badips.com, and use as blacklist # From 774dda6105ee9e5c2107416bd2465b73bf69f25a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Feb 2020 13:29:16 +0100 Subject: [PATCH 019/136] filter.d/postfix.conf: extended mode ddos and aggressive covering multiple disconnects without auth --- config/filter.d/postfix.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 29866dfa..fb690fb0 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -37,7 +37,7 @@ mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unava mdpr-more = %(mdpr-normal)s mdre-more = %(mdre-normal)s -mdpr-ddos = lost connection after(?! DATA) [A-Z]+ +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))) mdre-ddos = ^from [^[]*\[\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) From 88cf5bcd930b47897fd8c8359230675abeef6f73 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Feb 2020 13:41:28 +0100 Subject: [PATCH 020/136] Update postfix --- fail2ban/tests/files/logs/postfix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index d7d37600..6e2dc460 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -137,6 +137,11 @@ Jan 14 16:18:16 xxx postfix/smtpd[14933]: warning: host[192.0.2.5]: SASL CRAM-MD # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.1" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.1] helo=1 auth=0/1 quit=1 commands=2/3 +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.2" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.2] ehlo=1 auth=0/1 rset=1 quit=1 commands=3/4 + # failJSON: { "time": "2005-02-18T09:45:10", "match": true , "host": "192.0.2.10" } Feb 18 09:45:10 xxx postfix/smtpd[42]: lost connection after CONNECT from spammer.example.com[192.0.2.10] # failJSON: { "time": "2005-02-18T09:45:12", "match": true , "host": "192.0.2.42" } From ac8e8db8141a9945c7f435ae042c198e23a6f945 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 11 Feb 2020 14:18:58 +0100 Subject: [PATCH 021/136] travis: switch 3.8-dev to 3.8 (released) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 158cff99..1f218c81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: - python: 3.5 - python: 3.6 - python: 3.7 - - python: 3.8-dev + - python: 3.8 - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" From 1492ab2247bc02deaa65b80479aa4070e5404b39 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Feb 2020 18:44:36 +0100 Subject: [PATCH 022/136] improve processing of pending failures (lines without ID/IP) - fail2ban-regex would show those in matched lines now (as well as increase count of matched RE); avoid overwrite of data with empty tags by ticket constructed from multi-line failures; amend to d1b7e2b5fb2b389d04845369d7d29db65425dcf2: better output (as well as ignoring of pending lines) using `--out msg`; filter.d/sshd.conf: don't forget mlf-cache on "disconnecting: too many authentication failures" - message does not have IP (must be followed by "closed [preauth]" to obtain host-IP). --- config/filter.d/sshd.conf | 2 +- fail2ban/client/fail2banregex.py | 8 +++--- fail2ban/server/filter.py | 34 +++++++++++--------------- fail2ban/tests/files/logs/sshd | 2 +- fail2ban/tests/files/logs/sshd-journal | 2 +- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index d764a076..b382ffc1 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -55,7 +55,7 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s ^Disconnecting(?: from)?(?: (?:invalid|authenticating)) user \S+ %(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ ^Received disconnect from %(__on_port_opt)s:\s*11: -other> ^Accepted \w+ for \S+ from (?:\s|$) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 513b765d..afcb4282 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -272,6 +272,8 @@ class Fail2banRegex(object): self._filter.returnRawHost = opts.raw self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out + # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) + self._filter.ignorePending = opts.out; self._backend = 'auto' def output(self, line): @@ -452,7 +454,6 @@ class Fail2banRegex(object): try: found = self._filter.processLine(line, date) lines = [] - line = self._filter.processedLine() ret = [] for match in found: # Append True/False flag depending if line was matched by @@ -488,7 +489,7 @@ class Fail2banRegex(object): self._line_stats.matched += 1 self._line_stats.missed -= 1 if lines: # pre-lines parsed in multiline mode (buffering) - lines.append(line) + lines.append(self._filter.processedLine()) line = "\n".join(lines) return line, ret, is_ignored @@ -523,7 +524,8 @@ class Fail2banRegex(object): output(ret[1]) elif self._opts.out == 'msg': for ret in ret: - output('\n'.join(map(lambda v:''.join(v for v in v), ret[3].get('matches')))) + for ret in ret[3].get('matches'): + output(''.join(v for v in ret)) elif self._opts.out == 'row': for ret in ret: output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches'))) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 0c44c7ac..3a100fbd 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -105,6 +105,8 @@ class Filter(JailThread): self.returnRawHost = False ## check each regex (used for test purposes): self.checkAllRegex = False + ## avoid finding of pending failures (without ID/IP, used in fail2ban-regex): + self.ignorePending = True ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True ## Ticks counter @@ -651,7 +653,7 @@ class Filter(JailThread): fail['users'] = users = set() users.add(user) return users - return None + return users # # ATM incremental (non-empty only) merge deactivated ... # @staticmethod @@ -680,25 +682,22 @@ class Filter(JailThread): if not fail.get('nofail'): fail['nofail'] = fail["mlfgained"] elif fail.get('nofail'): nfflgs |= 1 - if fail.get('mlfforget'): nfflgs |= 2 + if fail.pop('mlfforget', None): nfflgs |= 2 # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] # update users set (hold all users of connect): users = self._updateUsers(mlfidGroups, fail.get('user')) # be sure we've correct current state ('nofail' and 'mlfgained' only from last failure) - try: - del mlfidGroups['nofail'] - del mlfidGroups['mlfgained'] - except KeyError: - pass + mlfidGroups.pop('nofail', None) + mlfidGroups.pop('mlfgained', None) # # ATM incremental (non-empty only) merge deactivated (for future version only), # # it can be simulated using alternate value tags, like ..., # # so previous value 'val' will be overwritten only if 'alt_val' is not empty... # _updateFailure(mlfidGroups, fail) # # overwrite multi-line failure with all values, available in fail: - mlfidGroups.update(fail) + mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None)) # new merged failure data: fail = mlfidGroups # if forget (disconnect/reset) - remove cached entry: @@ -709,20 +708,14 @@ class Filter(JailThread): mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) # check users in order to avoid reset failure by multiple logon-attempts: - if users and len(users) > 1: + if fail.pop('mlfpending', 0) or users and len(users) > 1: # we've new user, reset 'nofail' because of multiple users attempts: - try: - del fail['nofail'] - nfflgs &= ~1 # reset nofail - except KeyError: - pass + fail.pop('nofail', None) + nfflgs &= ~1 # reset nofail # merge matches: if not (nfflgs & 1): # current nofail state (corresponding users) - try: - m = fail.pop("nofail-matches") - m += fail.get("matches", []) - except KeyError: - m = fail.get("matches", []) + m = fail.pop("nofail-matches", []) + m += fail.get("matches", []) if not (nfflgs & 8): # no gain signaled m += failRegex.getMatchedTupleLines() fail["matches"] = m @@ -888,7 +881,8 @@ class Filter(JailThread): if host is None: if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) - if not self.checkAllRegex: return failList + fail['mlfpending'] = 1; # mark failure is pending + if not self.checkAllRegex and self.ignorePending: return failList ips = [None] # if raw - add single ip or failure-id, # otherwise expand host to multiple ips using dns (or ignore it if not valid): diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index a5f64939..2b8e6621 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -134,7 +134,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +Sep 29 17:15:03 spaceman sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 diff --git a/fail2ban/tests/files/logs/sshd-journal b/fail2ban/tests/files/logs/sshd-journal index 07e34efe..ba755645 100644 --- a/fail2ban/tests/files/logs/sshd-journal +++ b/fail2ban/tests/files/logs/sshd-journal @@ -135,7 +135,7 @@ srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser # failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -srv sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +srv sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } srv sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 From 9137c7bb2312c9e18ffc87ac7e827d89faf7fba4 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 12 Feb 2020 21:27:45 +0100 Subject: [PATCH 023/136] filter processing: - avoid duplicates in "matches" (previously always added matches of pending failures to every next real failure, or nofail-helper recognized IP, now first failure only); - several optimizations of merge mechanism (multi-line parsing); fail2ban-regex: better output handling, extended with tag substitution (ex.: `-o 'fail , user : '`); consider a string containing new-line as multi-line log-excerpt (not as a single log-line) filter.d/sshd.conf: introduced parameter `publickey` (allowing change behavior of "Failed publickey" failures): - `nofail` (default) - consider failed publickey (legitimate users) as no failure (helper to get IP and user-name only) - `invalid` - consider failed publickey for invalid users only; - `any` - consider failed publickey for valid users too; - `ignore` - ignore "Failed publickey ..." failures (don't consider failed publickey at all) tests/samplestestcase.py: SampleRegexsFactory gets new failJSON option `constraint` to allow ignore of some tests depending on filter name, options and test parameters --- config/filter.d/sshd.conf | 21 +++- fail2ban/client/fail2banregex.py | 101 +++++++++++++----- fail2ban/server/action.py | 23 ++-- fail2ban/server/filter.py | 46 +++----- .../filter.d/zzz-sshd-obsolete-multiline.conf | 2 +- fail2ban/tests/fail2banregextestcase.py | 73 ++++++++++++- fail2ban/tests/files/logs/sshd | 16 +-- fail2ban/tests/files/logs/sshd-journal | 16 +-- fail2ban/tests/samplestestcase.py | 97 +++++++++-------- 9 files changed, 260 insertions(+), 135 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index b382ffc1..c61cf960 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -40,8 +40,8 @@ prefregex = ^%(__prefix_line)s%(__pref)s.+.* from ( via \S+)?%(__suff)s$ ^User not known to the underlying authentication module for .* from %(__suff)s$ - ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + > + ^Failed for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^ROOT LOGIN REFUSED FROM ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ @@ -60,6 +60,12 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* -other> ^Accepted \w+ for \S+ from (?:\s|$) +cmnfailed-any = \S+ +cmnfailed-ignore = \b(?!publickey)\S+ +cmnfailed-invalid = +cmnfailed-nofail = (?:publickey|\S+) +cmnfailed = > + mdre-normal = # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode) mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ @@ -84,6 +90,17 @@ mdre-aggressive = %(mdre-ddos)s # mdre-extra-other is fully included within mdre-ddos-other: mdre-aggressive-other = %(mdre-ddos-other)s +# Parameter "publickey": nofail (default), invalid, any, ignore +publickey = nofail +# consider failed publickey for invalid users only: +cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) +# consider failed publickey for valid users too (don't need RE, see cmnfailed): +cmnfailre-failed-pub-any = +# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed): +cmnfailre-failed-pub-nofail = +# don't consider failed publickey as failures (don't need RE, see cmnfailed): +cmnfailre-failed-pub-ignore = + cfooterre = ^Connection from failregex = %(cmnfailre)s diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index afcb4282..a03125c3 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -493,8 +493,69 @@ class Fail2banRegex(object): line = "\n".join(lines) return line, ret, is_ignored + def _prepaireOutput(self): + """Prepares output- and fetch-function corresponding given '--out' option (format)""" + ofmt = self._opts.out + if ofmt in ('id', 'ip'): + def _out(ret): + for r in ret: + output(r[1]) + elif ofmt == 'msg': + def _out(ret): + for r in ret: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + output(r) + elif ofmt == 'row': + def _out(ret): + for r in ret: + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + elif '<' not in ofmt: + def _out(ret): + for r in ret: + output(r[3].get(ofmt)) + else: # extended format with tags substitution: + from ..server.actions import Actions, CommandAction, BanTicket + def _escOut(t, v): + # use safe escape (avoid inject on pseudo tag "\x00msg\x00"): + if t not in ('msg',): + return v.replace('\x00', '\\x00') + return v + def _out(ret): + rows = [] + wrap = {'NL':0} + for r in ret: + ticket = BanTicket(r[1], time=r[2], data=r[3]) + aInfo = Actions.ActionInfo(ticket) + # if msg tag is used - output if single line (otherwise let it as is to wrap multilines later): + def _get_msg(self): + if not wrap['NL'] and len(r[3].get('matches', [])) <= 1: + return self['matches'] + else: # pseudo tag for future replacement: + wrap['NL'] = 1 + return "\x00msg\x00" + aInfo['msg'] = _get_msg + # not recursive interpolation (use safe escape): + v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut) + if wrap['NL']: # contains multiline tags (msg): + rows.append((r, v)) + continue + output(v) + # wrap multiline tag (msg) interpolations to single line: + for r, v in rows: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + r = v.replace("\x00msg\x00", r) + output(r) + return _out + + def process(self, test_lines): t0 = time.time() + if self._opts.out: # get out function + out = self._prepaireOutput() for line in test_lines: if isinstance(line, tuple): line_datetimestripped, ret, is_ignored = self.testRegex( @@ -509,49 +570,35 @@ class Fail2banRegex(object): if not is_ignored: is_ignored = self.testIgnoreRegex(line_datetimestripped) + if self._opts.out: # (formated) output: + if len(ret) > 0: out(ret) + continue + if is_ignored: self._line_stats.ignored += 1 if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1): self._line_stats.ignored_lines.append(line) if self._debuggex: self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) - - if len(ret) > 0: - assert(not is_ignored) - if self._opts.out: - if self._opts.out in ('id', 'ip'): - for ret in ret: - output(ret[1]) - elif self._opts.out == 'msg': - for ret in ret: - for ret in ret[3].get('matches'): - output(''.join(v for v in ret)) - elif self._opts.out == 'row': - for ret in ret: - output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches'))) - else: - for ret in ret: - output(ret[3].get(self._opts.out)) - continue + elif len(ret) > 0: self._line_stats.matched += 1 if self._print_all_matched: self._line_stats.matched_lines.append(line) if self._debuggex: self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) else: - if not is_ignored: - self._line_stats.missed += 1 - if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): - self._line_stats.missed_lines.append(line) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + self._line_stats.missed += 1 + if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): + self._line_stats.missed_lines.append(line) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 self._time_elapsed = time.time() - t0 def printLines(self, ltype): lstats = self._line_stats - assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) + assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] multiline = self._filter.getMaxLines() > 1 @@ -688,10 +735,10 @@ class Fail2banRegex(object): test_lines = journal_lines_gen(flt, myjournal) else: # if single line parsing (without buffering) - if self._filter.getMaxLines() <= 1: + if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log: self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) test_lines = [ cmd_log ] - else: # multi line parsing (with buffering) + else: # multi line parsing (with and without buffering) test_lines = cmd_log.split("\n") self.output( "Use multi line : %s line(s)" % len(test_lines) ) for i, l in enumerate(test_lines): diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 2a5bb704..3bc48fe0 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -772,7 +772,7 @@ class CommandAction(ActionBase): ESCAPE_VN_CRE = re.compile(r"\W") @classmethod - def replaceDynamicTags(cls, realCmd, aInfo): + def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None): """Replaces dynamical tags in `query` with property values. **Important** @@ -797,16 +797,17 @@ class CommandAction(ActionBase): # array for escaped vars: varsDict = dict() - def escapeVal(tag, value): - # if the value should be escaped: - if cls.ESCAPE_CRE.search(value): - # That one needs to be escaped since its content is - # out of our control - tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) - varsDict[tag] = value # add variable - value = '$'+tag # replacement as variable - # replacement for tag: - return value + if not escapeVal: + def escapeVal(tag, value): + # if the value should be escaped: + if cls.ESCAPE_CRE.search(value): + # That one needs to be escaped since its content is + # out of our control + tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) + varsDict[tag] = value # add variable + value = '$'+tag # replacement as variable + # replacement for tag: + return value # additional replacement as calling map: ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 3a100fbd..835e9b2b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -655,30 +655,12 @@ class Filter(JailThread): return users return users - # # ATM incremental (non-empty only) merge deactivated ... - # @staticmethod - # def _updateFailure(self, mlfidGroups, fail): - # # reset old failure-ids when new types of id available in this failure: - # fids = set() - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if fail.get(k): - # fids.add(k) - # if fids: - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if k not in fids: - # try: - # del mlfidGroups[k] - # except: - # pass - # # update not empty values: - # mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) - def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None users = None nfflgs = 0 if fail.get("mlfgained"): - nfflgs |= 9 + nfflgs |= (8|1) if not fail.get('nofail'): fail['nofail'] = fail["mlfgained"] elif fail.get('nofail'): nfflgs |= 1 @@ -689,13 +671,11 @@ class Filter(JailThread): # update users set (hold all users of connect): users = self._updateUsers(mlfidGroups, fail.get('user')) # be sure we've correct current state ('nofail' and 'mlfgained' only from last failure) - mlfidGroups.pop('nofail', None) - mlfidGroups.pop('mlfgained', None) - # # ATM incremental (non-empty only) merge deactivated (for future version only), - # # it can be simulated using alternate value tags, like ..., - # # so previous value 'val' will be overwritten only if 'alt_val' is not empty... - # _updateFailure(mlfidGroups, fail) - # + if mlfidGroups.pop('nofail', None): nfflgs |= 4 + if mlfidGroups.pop('mlfgained', None): nfflgs |= 4 + # if we had no pending failures then clear the matches (they are already provided): + if (nfflgs & 4) == 0 and not mlfidGroups.get('mlfpending', 0): + mlfidGroups.pop("matches", None) # overwrite multi-line failure with all values, available in fail: mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None)) # new merged failure data: @@ -709,17 +689,18 @@ class Filter(JailThread): self.mlfidCache.set(mlfid, mlfidFail) # check users in order to avoid reset failure by multiple logon-attempts: if fail.pop('mlfpending', 0) or users and len(users) > 1: - # we've new user, reset 'nofail' because of multiple users attempts: + # we've pending failures or new user, reset 'nofail' because of failures or multiple users attempts: fail.pop('nofail', None) - nfflgs &= ~1 # reset nofail + fail.pop('mlfgained', None) + nfflgs &= ~(8|1) # reset nofail and gained # merge matches: - if not (nfflgs & 1): # current nofail state (corresponding users) + if (nfflgs & 1) == 0: # current nofail state (corresponding users) m = fail.pop("nofail-matches", []) m += fail.get("matches", []) - if not (nfflgs & 8): # no gain signaled + if (nfflgs & 8) == 0: # no gain signaled m += failRegex.getMatchedTupleLines() fail["matches"] = m - elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail: + elif (nfflgs & 3) == 1: # not mlfforget and nofail: fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines() # return merged: return fail @@ -895,6 +876,9 @@ class Filter(JailThread): # otherwise, try to use dns conversion: else: ips = DNSUtils.textToIp(host, self.__useDns) + # if checkAllRegex we must make a copy (to be sure next RE doesn't change merged/cached failure): + if self.checkAllRegex and mlfid is not None: + fail = fail.copy() # append failure with match to the list: for ip in ips: failList.append([failRegexIndex, ip, date, fail]) diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 549797af..d61a6520 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -37,7 +37,7 @@ __pam_auth = pam_[a-z]+ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^%(__prefix_line_sl)sFailed (?:publickey|\S+) for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index c09c4171..8c6a0e47 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -83,13 +83,29 @@ def _test_exec_command_line(*args): STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " -RE_00_ID = r"Authentication failure for .*? from $" -RE_00_USER = r"Authentication failure for .*? from $" +RE_00_ID = r"Authentication failure for .*? from $" +RE_00_USER = r"Authentication failure for .*? from $" FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") +# STR_ML_SSHD -- multiline log-excerpt with two sessions: +# 192.0.2.1 (sshd[32307]) makes 2 failed attempts using public keys (without "Disconnecting: Too many authentication"), +# and delayed success on accepted (STR_ML_SSHD_OK) or no success by close on preauth phase (STR_ML_SSHD_FAIL) +# 192.0.2.2 (sshd[32310]) makes 2 failed attempts using public keys (with "Disconnecting: Too many authentication"), +# and closed on preauth phase +STR_ML_SSHD = """Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth] +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.2 [preauth]""" +STR_ML_SSHD_OK = "Nov 28 09:16:06 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:..." +STR_ML_SSHD_FAIL = "Nov 28 09:16:06 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]" + + FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log') @@ -291,10 +307,10 @@ class Fail2banRegexTest(LogCaptureTestCase): # self.assertTrue(_test_exec( "--usedns", "no", "-d", "^Epoch", "--print-all-matched", - "1490349000 FAIL: failure\nhost: 192.0.2.35", + "-L", "2", "1490349000 FAIL: failure\nhost: 192.0.2.35", r"^\s*FAIL:\s*.*\nhost:\s+$" )) - self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + self.assertLogged('Lines: 2 lines, 0 ignored, 2 matched, 0 missed') def testRegexEpochPatterns(self): self.assertTrue(_test_exec( @@ -340,6 +356,55 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER)) self.assertLogged('kevin') self.pruneLog() + # complex substitution using tags (ip, user, family): + self.assertTrue(_test_exec('-o', ', , ', STR_00, RE_00_USER)) + self.assertLogged('192.0.2.0, kevin, inet4') + self.pruneLog() + + def testFrmtOutputWrapML(self): + unittest.F2B.SkipIfCfgMissing(stock=True) + # complex substitution using tags and message (ip, user, msg): + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]')) + # be sure we don't have IP in one line and have it in another: + lines = STR_ML_SSHD.split("\n") + self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1]) + # but both are in output "merged" with IP and user: + self.assertLogged( + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (mode is not aggressive): + self.assertNotLogged('192.0.2.1, git, ') + + # test with publickey (nofail) - would not produce output for 192.0.2.1 because accepted: + self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)") + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]')) + self.assertLogged( + '192.0.2.2, git, '+lines[-4], + '192.0.2.2, git, '+lines[-3], + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (access gained so failures ignored): + self.assertNotLogged('192.0.2.1, git, ') + + # now same test but "accepted" replaced with "closed" on preauth phase: + self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)") + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]')) + # 192.0.2.1 should be found for every failure (2x failed key + 1x closed): + lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:] + self.assertLogged( + '192.0.2.1, git, '+lines[-3], + '192.0.2.1, git, '+lines[-2], + '192.0.2.1, git, '+lines[-1], + all=True) + def testWrongFilterFile(self): # use test log as filter file to cover eror cases... diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 2b8e6621..3b4f0a0a 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -166,9 +166,11 @@ Nov 28 09:16:03 srv sshd[32307]: Connection closed by 192.0.2.1 Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx # failJSON: { "match": false } Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx -# failJSON: { "match": false } +# failJSON: { "constraint": "name == 'sshd'", "time": "2004-11-28T09:16:05", "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] -# failJSON: { "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" } +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] # failJSON: { "match": false } @@ -215,7 +217,7 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems (gh-1024) # failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root -# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } 2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 [preauth] # disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...): @@ -238,7 +240,7 @@ Mar 7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113 Mar 7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 # failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 -# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } Mar 7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) @@ -248,14 +250,14 @@ Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 # failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 -# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116 # failJSON: { "time": "2005-03-19T16:47:48", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } Mar 19 16:47:48 test sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 -# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } Mar 19 16:47:49 test sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] # failJSON: { "time": "2005-03-19T16:47:50", "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } Mar 19 16:47:50 srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] @@ -332,7 +334,7 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] # gh-1943 (previous OpenSSH log-format) diff --git a/fail2ban/tests/files/logs/sshd-journal b/fail2ban/tests/files/logs/sshd-journal index ba755645..d19889d7 100644 --- a/fail2ban/tests/files/logs/sshd-journal +++ b/fail2ban/tests/files/logs/sshd-journal @@ -167,9 +167,11 @@ srv sshd[32307]: Connection closed by 192.0.2.1 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx # failJSON: { "match": false } srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx -# failJSON: { "match": false } +# failJSON: { "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] -# failJSON: { "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" } +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] # failJSON: { "match": false } @@ -216,7 +218,7 @@ srv sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank yo # Match sshd auth errors on OpenSUSE systems (gh-1024) # failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } srv sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root -# failJSON: { "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } srv sshd[2716]: Connection closed by 192.0.2.112 [preauth] # filterOptions: [{}] @@ -238,7 +240,7 @@ srv sshd[1556]: Connection closed by 192.0.2.113 srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 # failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 -# failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +# failJSON: { "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } srv sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) @@ -248,14 +250,14 @@ srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 # failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 -# failJSON: { "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +# failJSON: { "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } srv sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } srv sshd[1559]: Connection closed by 192.0.2.116 # failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } srv sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 -# failJSON: { "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } srv sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] # failJSON: { "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] @@ -325,7 +327,7 @@ srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching srv sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +# failJSON: { "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] # gh-1943 (previous OpenSSH log-format) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 0bbd05f5..9908cba3 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -216,9 +216,11 @@ def testSampleRegexsFactory(name, basedir): # process line using several filter options (if specified in the test-file): for fltName, flt, opts in self._filterTests: + # Bypass if constraint (as expression) is not valid: + if faildata.get('constraint') and not eval(faildata['constraint']): + continue flt, regexsUsedIdx = flt regexList = flt.getFailRegex() - failregex = -1 try: fail = {} @@ -229,20 +231,23 @@ def testSampleRegexsFactory(name, basedir): if opts.get('test.prefix-line'): # journal backends creates common prefix-line: line = opts.get('test.prefix-line') + line ret = flt.processLine(('', TEST_NOW_STR, line.rstrip('\r\n')), TEST_NOW) - if not ret: - # Bypass if filter constraint specified: - if faildata.get('filter') and name != faildata.get('filter'): - continue - # Check line is flagged as none match - self.assertFalse(faildata.get('match', True), - "Line not matched when should have") - continue + if ret: + # filter matched only (in checkAllRegex mode it could return 'nofail' too): + found = [] + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # bypass pending and nofail: + if fid is None or fail.get('nofail'): + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) + continue + found.append(ret) + ret = found - failregex, fid, fail2banTime, fail = ret[0] - # Bypass no failure helpers-regexp: - if not faildata.get('match', False) and (fid is None or fail.get('nofail')): - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) + if not ret: + # Check line is flagged as none match + self.assertFalse(faildata.get('match', False), + "Line not matched when should have") continue # Check line is flagged to match @@ -251,39 +256,41 @@ def testSampleRegexsFactory(name, basedir): self.assertEqual(len(ret), 1, "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) - # Verify match captures (at least fid/host) and timestamp as expected - for k, v in faildata.iteritems(): - if k not in ("time", "match", "desc", "filter"): - fv = fail.get(k, None) - if fv is None: - # Fallback for backwards compatibility (previously no fid, was host only): - if k == "host": - fv = fid - # special case for attempts counter: - if k == "attempts": - fv = len(fail.get('matches', {})) - # compare sorted (if set) - if isinstance(fv, (set, list, dict)): - self.assertSortedEqual(fv, v) - continue - self.assertEqual(fv, v) + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # Verify match captures (at least fid/host) and timestamp as expected + for k, v in faildata.iteritems(): + if k not in ("time", "match", "desc", "constraint"): + fv = fail.get(k, None) + if fv is None: + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host": + fv = fid + # special case for attempts counter: + if k == "attempts": + fv = len(fail.get('matches', {})) + # compare sorted (if set) + if isinstance(fv, (set, list, dict)): + self.assertSortedEqual(fv, v) + continue + self.assertEqual(fv, v) - t = faildata.get("time", None) - if t is not None: - try: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") - except ValueError: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") - jsonTime = time.mktime(jsonTimeLocal.timetuple()) - jsonTime += jsonTimeLocal.microsecond / 1000000.0 - self.assertEqual(fail2banTime, jsonTime, - "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % - (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), - jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), - fail2banTime - jsonTime) ) + t = faildata.get("time", None) + if t is not None: + try: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") + except ValueError: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") + jsonTime = time.mktime(jsonTimeLocal.timetuple()) + jsonTime += jsonTimeLocal.microsecond / 1000000.0 + self.assertEqual(fail2banTime, jsonTime, + "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % + (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), + jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), + fail2banTime - jsonTime) ) - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover import pprint raise AssertionError("%s: %s on: %s:%i, line:\n %sregex (%s):\n %s\n" From 14e68eed72e3d6874a30a0a523125ad2c5dc79c0 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 12 Feb 2020 21:38:16 +0100 Subject: [PATCH 024/136] performance: set fetch handler getGroups depending on presence of alternate tags in RE (simplest variant or merged with alt-tags) in regex constructor --- fail2ban/server/failregex.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index f7dafbef..0ae9acc5 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -138,6 +138,8 @@ class Regex: except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) + # set fetch handler depending on presence of alternate tags: + self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -277,11 +279,12 @@ class Regex: # Returns all matched groups. # - def getGroups(self): - if not self._altValues: - return self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + def _getGroups(self): + return self._matchCache.groupdict() + + def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): #fail = fail.copy() for k,n in self._altValues: v = fail.get(k) @@ -289,6 +292,9 @@ class Regex: fail[n] = v return fail + def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) + pass + ## # Returns skipped lines. # From 91eca4fdeb728fa0fd2d6cf55db8eb7baa2a39a2 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Feb 2020 13:50:17 +0100 Subject: [PATCH 025/136] automatically create not-existing path (last level folder only) for pidfile, socket and database (with default permissions) --- fail2ban/server/server.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 22814280..abb312da 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -57,6 +57,23 @@ except ImportError: # pragma: no cover def _thread_name(): return threading.current_thread().__class__.__name__ +try: + FileExistsError +except NameError: # pragma: 3.x no cover + FileExistsError = OSError + +def _make_file_path(name): + """Creates path of file (last level only) on demand""" + name = os.path.dirname(name) + # only if it is absolute (e. g. important for socket, so if unix path): + if os.path.isabs(name): + # be sure path exists (create last level of directory on demand): + try: + os.mkdir(name) + except (OSError, FileExistsError) as e: + if e.errno != 17: # pragma: no cover - not EEXIST is not covered + raise + class Server: @@ -96,7 +113,7 @@ class Server: def start(self, sock, pidfile, force=False, conf={}): # First set the mask to only allow access to owner - os.umask(0077) + os.umask(0o077) # Second daemonize before logging etc, because it will close all handles: if self.__daemon: # pragma: no cover logSys.info("Starting in daemon mode") @@ -141,6 +158,7 @@ class Server: # Creates a PID file. try: logSys.debug("Creating PID file %s", pidfile) + _make_file_path(pidfile) pidFile = open(pidfile, 'w') pidFile.write("%s\n" % os.getpid()) pidFile.close() @@ -150,6 +168,7 @@ class Server: # Start the communication logSys.debug("Starting communication") try: + _make_file_path(sock) self.__asyncServer = AsyncServer(self.__transm) self.__asyncServer.onstart = conf.get('onstart') self.__asyncServer.start(sock, force) @@ -741,6 +760,7 @@ class Server: self.__db = None else: if Fail2BanDb is not None: + _make_file_path(filename) self.__db = Fail2BanDb(filename) self.__db.delAllJails() else: # pragma: no cover From b3644ad4134c78592106bc6bec2ed18d6949a0ba Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Feb 2020 21:26:28 +0100 Subject: [PATCH 026/136] code normalization and optimization (strip of trailing new-line, date parsing, ignoreregex mechanism, etc) --- fail2ban/client/fail2banregex.py | 71 ++++++++-------- fail2ban/server/filter.py | 134 ++++++++++++++---------------- fail2ban/tests/filtertestcase.py | 19 ++--- fail2ban/tests/samplestestcase.py | 9 +- 4 files changed, 111 insertions(+), 122 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index a03125c3..98fd9799 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -273,7 +273,9 @@ class Fail2banRegex(object): self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) - self._filter.ignorePending = opts.out; + self._filter.ignorePending = opts.out + # callback to increment ignored RE's by index (during process): + self._filter.onIgnoreRegex = self._onIgnoreRegex self._backend = 'auto' def output(self, line): @@ -435,22 +437,17 @@ class Fail2banRegex(object): 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True - def testIgnoreRegex(self, line): - found = False - try: - ret = self._filter.ignoreLine([(line, "", "")]) - if ret is not None: - found = True - regex = self._ignoreregex[ret].inc() - except RegexException as e: # pragma: no cover - output( 'ERROR: %s' % e ) - return False - return found + def _onIgnoreRegex(self, idx, ignoreRegex): + self._lineIgnored = True + self._ignoreregex[idx].inc() def testRegex(self, line, date=None): orgLineBuffer = self._filter._Filter__lineBuffer + # duplicate line buffer (list can be changed inplace during processLine): + if self._filter.getMaxLines() > 1: + orgLineBuffer = orgLineBuffer[:] fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() - is_ignored = False + is_ignored = self._lineIgnored = False try: found = self._filter.processLine(line, date) lines = [] @@ -469,29 +466,30 @@ class Fail2banRegex(object): except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) return False - for bufLine in orgLineBuffer[int(fullBuffer):]: - if bufLine not in self._filter._Filter__lineBuffer: - try: - self._line_stats.missed_lines.pop( - self._line_stats.missed_lines.index("".join(bufLine))) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.pop( - self._line_stats.missed_lines_timeextracted.index( - "".join(bufLine[::2]))) - except ValueError: - pass - # if buffering - add also another lines from match: - if self._print_all_matched: - if not self._debuggex: - self._line_stats.matched_lines.append("".join(bufLine)) - else: - lines.append(bufLine[0] + bufLine[2]) - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + if self._filter.getMaxLines() > 1: + for bufLine in orgLineBuffer[int(fullBuffer):]: + if bufLine not in self._filter._Filter__lineBuffer: + try: + self._line_stats.missed_lines.pop( + self._line_stats.missed_lines.index("".join(bufLine))) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.pop( + self._line_stats.missed_lines_timeextracted.index( + "".join(bufLine[::2]))) + except ValueError: + pass + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 if lines: # pre-lines parsed in multiline mode (buffering) lines.append(self._filter.processedLine()) line = "\n".join(lines) - return line, ret, is_ignored + return line, ret, (is_ignored or self._lineIgnored) def _prepaireOutput(self): """Prepares output- and fetch-function corresponding given '--out' option (format)""" @@ -558,8 +556,7 @@ class Fail2banRegex(object): out = self._prepaireOutput() for line in test_lines: if isinstance(line, tuple): - line_datetimestripped, ret, is_ignored = self.testRegex( - line[0], line[1]) + line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1]) line = "".join(line[0]) else: line = line.rstrip('\r\n') @@ -567,11 +564,9 @@ class Fail2banRegex(object): # skip comment and empty lines continue line_datetimestripped, ret, is_ignored = self.testRegex(line) - if not is_ignored: - is_ignored = self.testIgnoreRegex(line_datetimestripped) if self._opts.out: # (formated) output: - if len(ret) > 0: out(ret) + if len(ret) > 0 and not is_ignored: out(ret) continue if is_ignored: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 835e9b2b..112569c2 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -107,6 +107,8 @@ class Filter(JailThread): self.checkAllRegex = False ## avoid finding of pending failures (without ID/IP, used in fail2ban-regex): self.ignorePending = True + ## callback called on ignoreregex match : + self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True ## Ticks counter @@ -170,7 +172,7 @@ class Filter(JailThread): # @param value the regular expression def addFailRegex(self, value): - multiLine = self.getMaxLines() > 1 + multiLine = self.__lineBufferSize > 1 try: regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, useDns=self.__useDns) @@ -575,20 +577,33 @@ class Filter(JailThread): """ if date: tupleLine = line + self.__lastTimeText = tupleLine[1] + self.__lastDate = date else: - l = line.rstrip('\r\n') logSys.log(7, "Working on line %r", line) - (timeMatch, template) = self.dateDetector.matchTime(l) - if timeMatch: - tupleLine = ( - l[:timeMatch.start(1)], - l[timeMatch.start(1):timeMatch.end(1)], - l[timeMatch.end(1):], - (timeMatch, template) - ) + # try to parse date: + timeMatch = self.dateDetector.matchTime(line) + m = timeMatch[0] + if m: + s = m.start(1) + e = m.end(1) + m = line[s:e] + tupleLine = (line[:s], m, line[e:]) + if m: # found and not empty - retrive date: + date = self.dateDetector.getTime(m, timeMatch) + + if date is None: + if m: logSys.error("findFailure failed to parse timeText: %s", m) + date = self.__lastDate + else: + # Lets get the time part + date = date[0] + self.__lastTimeText = m + self.__lastDate = date else: - tupleLine = (l, "", "", None) + tupleLine = (line, self.__lastTimeText, "") + date = self.__lastDate # save last line (lazy convert of process line tuple to string on demand): self.processedLine = lambda: "".join(tupleLine[::2]) @@ -630,20 +645,26 @@ class Filter(JailThread): self._errors //= 2 self.idle = True - ## - # Returns true if the line should be ignored. - # - # Uses ignoreregex. - # @param line: the line - # @return: a boolean - - def ignoreLine(self, tupleLines): - buf = Regex._tupleLinesBuf(tupleLines) + def _ignoreLine(self, buf, orgBuffer, failRegex=None): + # if multi-line buffer - use matched only, otherwise (single line) - original buf: + if failRegex and self.__lineBufferSize > 1: + orgBuffer = failRegex.getMatchedTupleLines() + buf = Regex._tupleLinesBuf(orgBuffer) + # search ignored: + fnd = None for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): - ignoreRegex.search(buf, tupleLines) + ignoreRegex.search(buf, orgBuffer) if ignoreRegex.hasMatched(): - return ignoreRegexIndex - return None + fnd = ignoreRegexIndex + logSys.log(7, " Matched ignoreregex %d and was ignored", fnd) + if self.onIgnoreRegex: self.onIgnoreRegex(fnd, ignoreRegex) + # remove ignored match: + if not self.checkAllRegex or self.__lineBufferSize > 1: + # todo: check ignoreRegex.getUnmatchedTupleLines() would be better (fix testGetFailuresMultiLineIgnoreRegex): + if failRegex: + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + if not self.checkAllRegex: break + return fnd def _updateUsers(self, fail, user=()): users = fail.get('users') @@ -713,7 +734,7 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date=None): + def findFailure(self, tupleLine, date): failList = list() ll = logSys.getEffectiveLevel() @@ -723,62 +744,38 @@ class Filter(JailThread): returnRawHost = True cidr = IPAddr.CIDR_RAW - # Checks if we mut ignore this line. - if self.ignoreLine([tupleLine[::2]]) is not None: - # The ignoreregex matched. Return. - if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", - "".join(tupleLine[::2])) - return failList - - timeText = tupleLine[1] - if date: - self.__lastTimeText = timeText - self.__lastDate = date - elif timeText: - - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) - - if dateTimeMatch is None: - logSys.error("findFailure failed to parse timeText: %s", timeText) - date = self.__lastDate - - else: - # Lets get the time part - date = dateTimeMatch[0] - - self.__lastTimeText = timeText - self.__lastDate = date - else: - timeText = self.__lastTimeText or "".join(tupleLine[::2]) - date = self.__lastDate - if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", date, MyTime.time(), self.getFindTime()) return failList if self.__lineBufferSize > 1: - orgBuffer = self.__lineBuffer = ( - self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] + self.__lineBuffer.append(tupleLine) + orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:] else: - orgBuffer = self.__lineBuffer = [tupleLine[:3]] - if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer) - buf = Regex._tupleLinesBuf(self.__lineBuffer) + orgBuffer = self.__lineBuffer = [tupleLine] + if ll <= 5: logSys.log(5, "Looking for match of %r", orgBuffer) + buf = Regex._tupleLinesBuf(orgBuffer) + + # Checks if we must ignore this line (only if fewer ignoreregex than failregex). + if self.__ignoreRegex and len(self.__ignoreRegex) < len(self.__failRegex) - 2: + if self._ignoreLine(buf, orgBuffer) is not None: + # The ignoreregex matched. Return. + return failList # Pre-filter fail regex (if available): preGroups = {} if self.__prefRegex: if ll <= 5: logSys.log(5, " Looking for prefregex %r", self.__prefRegex.getRegex()) - self.__prefRegex.search(buf, self.__lineBuffer) + self.__prefRegex.search(buf, orgBuffer) if not self.__prefRegex.hasMatched(): if ll <= 5: logSys.log(5, " Prefregex not matched") return failList preGroups = self.__prefRegex.getGroups() if ll <= 7: logSys.log(7, " Pre-filter matched %s", preGroups) - repl = preGroups.get('content') + repl = preGroups.pop('content', None) # Content replacement: if repl: - del preGroups['content'] self.__lineBuffer, buf = [('', '', repl)], None # Iterates over all the regular expressions. @@ -796,15 +793,12 @@ class Filter(JailThread): # The failregex matched. if ll <= 7: logSys.log(7, " Matched failregex %d: %s", failRegexIndex, fail) # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: + if self.__ignoreRegex and self._ignoreLine(buf, orgBuffer, failRegex) is not None: # The ignoreregex matched. Remove ignored match. - self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None - if ll <= 7: logSys.log(7, " Matched ignoreregex and was ignored") + buf = None if not self.checkAllRegex: break - else: - continue + continue if date is None: logSys.warning( "Found a match for %r but no valid date/time " @@ -814,10 +808,10 @@ class Filter(JailThread): "file a detailed issue on" " https://github.com/fail2ban/fail2ban/issues " "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), timeText) + "\n".join(failRegex.getMatchedLines()), tupleLine[1]) continue # we should check all regex (bypass on multi-line, otherwise too complex): - if not self.checkAllRegex or self.getMaxLines() > 1: + if not self.checkAllRegex or self.__lineBufferSize > 1: self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None # merge data if multi-line failure: raw = returnRawHost @@ -1056,7 +1050,7 @@ class FileFilter(Filter): if not line or not self.active: # The jail reached the bottom or has been stopped break - self.processLineAndAdd(line) + self.processLineAndAdd(line.rstrip('\r\n')) finally: log.close() db = self.jail.database diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 91cb7eb6..959d96b7 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -63,10 +63,7 @@ def open(*args): if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) - if sys.version_info >= (3,): - return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) - else: - return fopen(*args) + return fopen(*args) def _killfile(f, name): @@ -200,7 +197,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # polling filter could detect the change mtimesleep() if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Skip @@ -210,7 +207,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line i = 0 lines = [] while n is None or i < n: - l = fin.readline() + l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') if terminal_line is not None and l == terminal_line: break lines.append(l) @@ -238,7 +235,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p Returns None """ if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Required for filtering @@ -249,7 +246,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Read/Write i = 0 while n is None or i < n: - l = fin.readline() + l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') if terminal_line is not None and l == terminal_line: break journal.send(MESSAGE=l.strip(), **fields) @@ -1583,9 +1580,9 @@ class GetFailures(LogCaptureTestCase): # We first adjust logfile/failures to end with CR+LF fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') # poor man unix2dos: - fin, fout = open(GetFailures.FILENAME_01), open(fname, 'w') - for l in fin.readlines(): - fout.write('%s\r\n' % l.rstrip('\n')) + fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') + for l in fin.read().splitlines(): + fout.write(l + b'\r\n') fin.close() fout.close() diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 9908cba3..b99dd06c 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -32,7 +32,7 @@ import sys import time import unittest from ..server.failregex import Regex -from ..server.filter import Filter +from ..server.filter import Filter, FileContainer from ..client.filterreader import FilterReader from .utils import setUpMyTime, tearDownMyTime, TEST_NOW, CONFIG_DIR @@ -157,10 +157,11 @@ def testSampleRegexsFactory(name, basedir): while i < len(filenames): filename = filenames[i]; i += 1; logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", - filename)) + filename), mode='rb') ignoreBlock = False for line in logFile: + line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: try: @@ -202,6 +203,7 @@ def testSampleRegexsFactory(name, basedir): raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) line = next(logFile) + line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable @@ -214,6 +216,7 @@ def testSampleRegexsFactory(name, basedir): flt = self._readFilter(fltName, name, basedir, opts=None) self._filterTests = [(fltName, flt, {})] + line = line.rstrip('\r\n') # process line using several filter options (if specified in the test-file): for fltName, flt, opts in self._filterTests: # Bypass if constraint (as expression) is not valid: @@ -230,7 +233,7 @@ def testSampleRegexsFactory(name, basedir): else: # simulate journal processing, time is known from journal (formatJournalEntry): if opts.get('test.prefix-line'): # journal backends creates common prefix-line: line = opts.get('test.prefix-line') + line - ret = flt.processLine(('', TEST_NOW_STR, line.rstrip('\r\n')), TEST_NOW) + ret = flt.processLine(('', TEST_NOW_STR, line), TEST_NOW) if ret: # filter matched only (in checkAllRegex mode it could return 'nofail' too): found = [] From ab3a7fc6d2ea01cd4a17607398d151bb5fe2e63b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 17 Feb 2020 16:24:42 +0100 Subject: [PATCH 027/136] filter.d/sshd.conf: mode `ddos` (and aggressive) extended to detect port scanner sending unexpected ident string after connect --- config/filter.d/sshd.conf | 3 ++- fail2ban/tests/files/logs/sshd | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index c61cf960..12631cb3 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -52,7 +52,7 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ - ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ + ^maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s ^Disconnecting(?: from)?(?: (?:invalid|authenticating)) user \S+ %(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ @@ -71,6 +71,7 @@ mdre-normal = mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from + ^kex_exchange_identification: client sent invalid protocol identifier ^Bad protocol version identification '.*' from ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 3b4f0a0a..0385f38c 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -305,6 +305,11 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 # filterOptions: [{"test.condition":"name=='sshd'", "mode": "ddos"}, {"test.condition":"name=='sshd'", "mode": "aggressive"}] +# failJSON: { "match": false } +Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 +# failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } +Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" + # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-03-15T09:21:02", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } From e57e950ef57ab19ac8af315ecfc517039f0c5d6d Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Feb 2020 14:43:15 +0100 Subject: [PATCH 028/136] version bump (back to dev) --- ChangeLog | 9 ++++++++- fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-python.1 | 2 +- man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6c7b4bd9..ea0c4a64 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,9 +6,10 @@ Fail2Ban: Changelog =================== -Incompatibility list (compared to v.0.9): +ver. 0.10.6-dev (20??/??/??) - development edition ----------- +### Incompatibility list (v.0.10 compared to v.0.9): * Filter (or `failregex`) internal capture-groups: - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should @@ -30,6 +31,12 @@ Incompatibility list (compared to v.0.9): * Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are IPv6-capable now. +### Fixes + +### New Features + +### Enhancements + ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must ----------- diff --git a/fail2ban/version.py b/fail2ban/version.py index 89f6248c..e3c02e63 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.10.5" +version = "0.10.6-dev" def normVersion(): """ Returns fail2ban version in normalized machine-readable format""" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index ad4fa0ed..84f846f2 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "February 2020" "fail2ban-client v0.10.5" "User Commands" +.TH FAIL2BAN-CLIENT "1" "February 2020" "fail2ban-client v0.10.6-dev" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.10.5 reads log file that contains password failure report +Fail2Ban v0.10.6\-dev reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-python.1 b/man/fail2ban-python.1 index f38097c5..16ebf3ed 100644 --- a/man/fail2ban-python.1 +++ b/man/fail2ban-python.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-PYTHON "1" "January 2020" "fail2ban-python 0.10.5" "User Commands" +.TH FAIL2BAN-PYTHON "1" "February 2020" "fail2ban-python 0.10.6-dev" "User Commands" .SH NAME fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used .SH DESCRIPTION diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 44154b85..bb89ef8c 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "January 2020" "fail2ban-regex 0.10.5" "User Commands" +.TH FAIL2BAN-REGEX "1" "February 2020" "fail2ban-regex 0.10.6-dev" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index c7516cc8..bca729b5 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "February 2020" "fail2ban-server v0.10.5" "User Commands" +.TH FAIL2BAN-SERVER "1" "February 2020" "fail2ban-server v0.10.6-dev" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.10.5 reads log file that contains password failure report +Fail2Ban v0.10.6\-dev reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 56b02627..fd91d466 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "January 2020" "fail2ban-testcases 0.10.5" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "February 2020" "fail2ban-testcases 0.10.6-dev" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From df885586d490d1149efda1b59269c69e7185549f Mon Sep 17 00:00:00 2001 From: Christopher Gurnee Date: Wed, 19 Feb 2020 15:28:12 -0500 Subject: [PATCH 029/136] close Popen() pipes explicitly for PyPy Waiting for garbage collection to close pipes opened by Popen() can lead to "Too many open files" errors with PyPy; close them explicitly. --- ChangeLog | 1 + fail2ban/server/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ea0c4a64..007e4ffc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition IPv6-capable now. ### Fixes +* restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed ### New Features diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 2bde3f4d..8e8b0571 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -260,7 +260,6 @@ class Utils(): if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): for l in stdout.splitlines(): logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l)) - popen.stdout.close() if popen.stderr: try: if retcode is None or retcode < 0: @@ -271,7 +270,8 @@ class Utils(): if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): for l in stderr.splitlines(): logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) - popen.stderr.close() + popen.stdout.close() + popen.stderr.close() success = False if retcode in success_codes: From 6c6cf2a9562fa6de53432dd39debbec0985d2967 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Feb 2020 15:06:04 +0100 Subject: [PATCH 030/136] small amend (avoid possible error by close of not existing pipe) --- fail2ban/server/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 8e8b0571..053aa04f 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -270,8 +270,9 @@ class Utils(): if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): for l in stderr.splitlines(): logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) - popen.stdout.close() - popen.stderr.close() + + if popen.stdout: popen.stdout.close() + if popen.stderr: popen.stderr.close() success = False if retcode in success_codes: From 4766547e1f9a9311e9534459633bd9687d37af68 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Feb 2020 11:49:43 +0100 Subject: [PATCH 031/136] performance optimization of `datepattern` (better search algorithm); datetemplate: improved anchor detection for capturing groups `(^...)`; introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; datedetector: speedup special case if only one template is defined (every match wins - no collision, no sorting, no other best match possible) --- ChangeLog | 3 ++ fail2ban/server/datedetector.py | 69 +++++++++++++++----------- fail2ban/server/datetemplate.py | 21 +++++--- fail2ban/tests/datedetectortestcase.py | 21 ++++++++ 4 files changed, 78 insertions(+), 36 deletions(-) diff --git a/ChangeLog b/ChangeLog index 007e4ffc..2bb07385 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,9 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### New Features ### Enhancements +* introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; +* datetemplate: improved anchor detection for capturing groups `(^...)`; +* performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 5942e3e0..0a6451be 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -337,65 +337,76 @@ class DateDetector(object): # if no templates specified - default templates should be used: if not len(self.__templates): self.addDefaultTemplate() - logSys.log(logLevel-1, "try to match time for line: %.120s", line) - match = None + log = logSys.log if logSys.getEffectiveLevel() <= logLevel else lambda *args: None + log(logLevel-1, "try to match time for line: %.120s", line) + # first try to use last template with same start/end position: + match = None + found = None, 0x7fffffff, 0x7fffffff, -1 ignoreBySearch = 0x7fffffff i = self.__lastTemplIdx if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): - if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug - logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) + log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", - i, distance, endpos, - line[distance-1:distance], self.__lastPos[1], - line[distance:endpos], - line[endpos:endpos+1], self.__lastEndPos[1]) - # check same boundaries left/right, otherwise possible collision/pattern switch: - if (line[distance-1:distance] == self.__lastPos[1] and - line[endpos:endpos+1] == self.__lastEndPos[1] - ): + log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", + i, distance, endpos, + line[distance-1:distance], self.__lastPos[1], + line[distance:endpos], + line[endpos:endpos+1], self.__lastEndPos[2]) + # check same boundaries left/right, outside fully equal, inside only if not alnum (e. g. bound RE + # with space or some special char), otherwise possible collision/pattern switch: + if (( + line[distance-1:distance] == self.__lastPos[1] or + (line[distance] == self.__lastPos[2] and not self.__lastPos[2].isalnum()) + ) and ( + line[endpos:endpos+1] == self.__lastEndPos[2] or + (line[endpos-1] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum()) + )): + # search in line part only: + log(logLevel-1, " boundaries are correct, search in part %r", line[distance:endpos]) match = template.matchDate(line, distance, endpos) + else: + log(logLevel-1, " boundaries show conflict, try whole search") + match = template.matchDate(line) + ignoreBySearch = i if match: distance = match.start() endpos = match.end() # if different position, possible collision/pattern switch: if ( + len(self.__templates) == 1 or # single template: template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or (distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]) ): - logSys.log(logLevel, " matched last time template #%02i", i) + log(logLevel, " matched last time template #%02i", i) else: - logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") + log(logLevel, " ** last pattern collision - pattern change, reserve & search ...") + found = match, distance, endpos, i; # save current best alternative match = None else: - logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") + log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: - logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) - found = None, 0x7fffffff, 0x7fffffff, -1 + log(logLevel, " search template (%i) ...", len(self.__templates)) i = 0 for ddtempl in self.__templates: - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) if i == ignoreBySearch: i += 1 continue + log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) template = ddtempl.template match = template.matchDate(line) if match: distance = match.start() endpos = match.end() - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", - i, distance, ddtempl.distance, self.__lastPos[0], template.name) + log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", + i, distance, ddtempl.distance, self.__lastPos[0], template.name) ## last (or single) template - fast stop: if i+1 >= len(self.__templates): break @@ -408,7 +419,7 @@ class DateDetector(object): ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: if distance > ddtempl.distance or distance > self.__lastPos[0]: - logSys.log(logLevel, " ** distance collision - pattern change, reserve") + log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: found = match, distance, endpos, i @@ -422,7 +433,7 @@ class DateDetector(object): # check other template was found (use this one with shortest distance): if not match and found[0]: match, distance, endpos, i = found - logSys.log(logLevel, " use best time template #%02i", i) + log(logLevel, " use best time template #%02i", i) ddtempl = self.__templates[i] template = ddtempl.template # we've winner, incr hits, set distance, usage, reorder, etc: @@ -432,8 +443,8 @@ class DateDetector(object): ddtempl.distance = distance if self.__firstUnused == i: self.__firstUnused += 1 - self.__lastPos = distance, line[distance-1:distance] - self.__lastEndPos = endpos, line[endpos:endpos+1] + self.__lastPos = distance, line[distance-1:distance], line[distance] + self.__lastEndPos = endpos, line[endpos-1], line[endpos:endpos+1] # if not first - try to reorder current template (bubble up), they will be not sorted anymore: if i and i != self.__lastTemplIdx: i = self._reorderTemplate(i) @@ -442,7 +453,7 @@ class DateDetector(object): return (match, template) # not found: - logSys.log(logLevel, " no template.") + log(logLevel, " no template.") return (None, None) @property diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 973a8a51..a198e4ed 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -36,15 +36,16 @@ logSys = getLogger(__name__) RE_GROUPED = re.compile(r'(? Date: Mon, 2 Mar 2020 17:05:00 +0100 Subject: [PATCH 032/136] failmanager, ticket: avoid reset of retry count by pause between attempts near to findTime - adjust time of ticket will now change current attempts considering findTime as an estimation from rate by previous known interval (if it exceeds the findTime); this should avoid some false positives as well as provide more safe handling around `maxretry/findtime` relation especially on busy circumstances. --- fail2ban/server/failmanager.py | 9 +++---- fail2ban/server/ticket.py | 43 ++++++++++++++++---------------- fail2ban/tests/tickettestcase.py | 28 +++++++++++++-------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 80a6414a..eee979fd 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -92,10 +92,7 @@ class FailManager: if attempt <= 0: attempt += 1 unixTime = ticket.getTime() - fData.setLastTime(unixTime) - if fData.getLastReset() < unixTime - self.__maxTime: - fData.setLastReset(unixTime) - fData.setRetry(0) + fData.adjustTime(unixTime, self.__maxTime) fData.inc(matches, attempt, count) # truncate to maxMatches: if self.maxMatches: @@ -136,7 +133,7 @@ class FailManager: def cleanup(self, time): with self.__lock: todelete = [fid for fid,item in self.__failList.iteritems() \ - if item.getLastTime() + self.__maxTime <= time] + if item.getTime() + self.__maxTime <= time] if len(todelete) == len(self.__failList): # remove all: self.__failList = dict() @@ -150,7 +147,7 @@ class FailManager: else: # create new dictionary without items to be deleted: self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ - if item.getLastTime() + self.__maxTime > time) + if item.getTime() + self.__maxTime > time) self.__bgSvc.service() def delFailure(self, fid): diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 4f509ea9..8feeac9a 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -218,21 +218,20 @@ class FailTicket(Ticket): def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): # this class variables: - self.__retry = 0 - self.__lastReset = None + self._firstTime = None + self._retry = 1 # create/copy using default ticket constructor: Ticket.__init__(self, ip, time, matches, data, ticket) # init: - if ticket is None: - self.__lastReset = time if time is not None else self.getTime() - if not self.__retry: - self.__retry = self._data['failures']; + if not isinstance(ticket, FailTicket): + self._firstTime = time if time is not None else self.getTime() + self._retry = self._data.get('failures', 1) def setRetry(self, value): """ Set artificial retry count, normally equal failures / attempt, used in incremental features (BanTimeIncr) to increase retry count for bad IPs """ - self.__retry = value + self._retry = value if not self._data['failures']: self._data['failures'] = 1 if not value: @@ -243,10 +242,23 @@ class FailTicket(Ticket): """ Returns failures / attempt count or artificial retry count increased for bad IPs """ - return max(self.__retry, self._data['failures']) + return self._retry + + def adjustTime(self, time, maxTime): + """ Adjust time of ticket and current attempts count considering given maxTime + as estimation from rate by previous known interval (if it exceeds the findTime) + """ + if time > self._time: + # expand current interval and attemps count (considering maxTime): + if self._firstTime < time - maxTime: + # adjust retry calculated as estimation from rate by previous known interval: + self._retry = int(round(self._retry / float(time - self._firstTime) * maxTime)) + self._firstTime = time - maxTime + # last time of failure: + self._time = time def inc(self, matches=None, attempt=1, count=1): - self.__retry += count + self._retry += count self._data['failures'] += attempt if matches: # we should duplicate "matches", because possibly referenced to multiple tickets: @@ -255,19 +267,6 @@ class FailTicket(Ticket): else: self._data['matches'] = matches - def setLastTime(self, value): - if value > self._time: - self._time = value - - def getLastTime(self): - return self._time - - def getLastReset(self): - return self.__lastReset - - def setLastReset(self, value): - self.__lastReset = value - ## # Ban Ticket. # diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py index 277c2f28..d7d5f19a 100644 --- a/fail2ban/tests/tickettestcase.py +++ b/fail2ban/tests/tickettestcase.py @@ -69,10 +69,10 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft.getTime(), tm) self.assertEqual(ft.getMatches(), matches2) ft.setAttempt(2) - self.assertEqual(ft.getAttempt(), 2) - # retry is max of set retry and failures: - self.assertEqual(ft.getRetry(), 2) ft.setRetry(1) + self.assertEqual(ft.getAttempt(), 2) + self.assertEqual(ft.getRetry(), 1) + ft.setRetry(2) self.assertEqual(ft.getRetry(), 2) ft.setRetry(3) self.assertEqual(ft.getRetry(), 3) @@ -86,13 +86,21 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft.getRetry(), 14) self.assertEqual(ft.getMatches(), matches3) # last time (ignore if smaller as time): - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm-60) self.assertEqual(ft.getTime(), tm) - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm+60) + ft.adjustTime(tm-60, 3600) + self.assertEqual(ft.getTime(), tm) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+60, 3600) self.assertEqual(ft.getTime(), tm+60) - self.assertEqual(ft.getLastTime(), tm+60) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+3600, 3600) + self.assertEqual(ft.getTime(), tm+3600) + self.assertEqual(ft.getRetry(), 14) + # adjust time so interval is larger than find time (3600), so reset retry count: + ft.adjustTime(tm+7200, 3600) + self.assertEqual(ft.getTime(), tm+7200) + self.assertEqual(ft.getRetry(), 7); # estimated attempts count + self.assertEqual(ft.getAttempt(), 4); # real known failure count ft.setData('country', 'DE') self.assertEqual(ft.getData(), {'matches': ['first', 'second', 'third'], 'failures': 4, 'country': 'DE'}) @@ -102,10 +110,10 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft, ft2) self.assertEqual(ft.getData(), ft2.getData()) self.assertEqual(ft2.getAttempt(), 4) - self.assertEqual(ft2.getRetry(), 14) + self.assertEqual(ft2.getRetry(), 7) self.assertEqual(ft2.getMatches(), matches3) self.assertEqual(ft2.getTime(), ft.getTime()) - self.assertEqual(ft2.getLastTime(), ft.getLastTime()) + self.assertEqual(ft2.getTime(), ft.getTime()) self.assertEqual(ft2.getBanTime(), ft.getBanTime()) def testTicketFlags(self): From 15158e4474593aff797222fb0984658b59d5d31f Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 2 Mar 2020 18:58:59 +0100 Subject: [PATCH 033/136] closes gh-2647: add ban to database is moved from jail.putFailTicket to actions.__CheckBan; be sure manual ban is written to database, so can be restored by restart; reload/restart test extended --- ChangeLog | 1 + fail2ban/server/actions.py | 5 ++++- fail2ban/server/jail.py | 2 -- fail2ban/tests/databasetestcase.py | 1 + fail2ban/tests/fail2banclienttestcase.py | 23 ++++++++++++++++------- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2bb07385..a81ba0aa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### Fixes * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed +* manual ban is written to database, so can be restored by restart (gh-2647) ### New Features diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 5488465e..d1f46ac0 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -452,7 +452,7 @@ class Actions(JailThread, Mapping): logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip) for name, action in self._actions.iteritems(): try: - if ticket.restored and getattr(action, 'norestored', False): + if bTicket.restored and getattr(action, 'norestored', False): continue if not aInfo.immutable: aInfo.reset() action.ban(aInfo) @@ -495,6 +495,9 @@ class Actions(JailThread, Mapping): cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: cnt += self.__reBan(bTicket) + # add ban to database: + if not bTicket.restored and self._jail.database is not None: + self._jail.database.addBan(self._jail, bTicket) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 89912556..b8a1c1f4 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -197,8 +197,6 @@ class Jail(object): Used by filter to add a failure for banning. """ self.__queue.put(ticket) - if not ticket.restored and self.database is not None: - self.database.addBan(self, ticket) def getFailTicket(self): """Get a fail ticket from the jail. diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 06f92d8c..d06927b1 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -491,6 +491,7 @@ class DatabaseTest(LogCaptureTestCase): # test action together with database functionality self.testAddJail() # Jail required self.jail.database = self.db + self.db.addJail(self.jail) actions = Actions(self.jail) actions.add( "action_checkainfo", diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 770e3ffb..6c79800e 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -973,8 +973,8 @@ class Fail2banServerTest(Fail2banClientServerBase): # leave action2 just to test restored interpolation: _write_jail_cfg(actions=[2,3]) - # write new failures: self.pruneLog("[test-phase 2b]") + # write new failures: _write_file(test2log, "w+", *( (str(int(MyTime.time())) + " error 403 from 192.0.2.2: test 2",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.3: test 2",) * 3 + @@ -987,13 +987,19 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "2 ticket(s) in 'test-jail2", "5 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME) + # ban manually to cover restore in restart (phase 2c): + self.execCmd(SUCCESS, startparams, + "set", "test-jail2", "banip", "192.0.2.9") + self.assertLogged( + "3 ticket(s) in 'test-jail2", wait=MID_WAITTIME) self.assertLogged( "[test-jail1] Ban 192.0.2.2", "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", "[test-jail1] Ban 192.0.2.8", "[test-jail2] Ban 192.0.2.4", - "[test-jail2] Ban 192.0.2.8", all=True) + "[test-jail2] Ban 192.0.2.8", + "[test-jail2] Ban 192.0.2.9", all=True) # test ips at all not visible for jail2: self.assertNotLogged( "[test-jail2] Found 192.0.2.2", @@ -1013,15 +1019,17 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Reload finished.", "Restore Ban", - "2 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) + "3 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) # stop/start and unban/restore ban: self.assertLogged( - "Jail 'test-jail2' stopped", - "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", + "Jail 'test-jail2' stopped", + "Jail 'test-jail2' started", "[test-jail2] Restore Ban 192.0.2.4", - "[test-jail2] Restore Ban 192.0.2.8", all=True + "[test-jail2] Restore Ban 192.0.2.8", + "[test-jail2] Restore Ban 192.0.2.9", all=True ) # test restored is 1 (only test-action2): self.assertLogged( @@ -1055,7 +1063,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail2' stopped", "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", - "[test-jail2] Unban 192.0.2.8", all=True + "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", all=True ) # test unban (action2): self.assertLogged( From 42714d0849cff8a1502e0fabfed376e40187fca8 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Mar 2020 13:47:11 +0100 Subject: [PATCH 034/136] filter.d/common.conf: closes gh-2650, avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (after the config considers all sections that can overwrite it); amend to 62b1712d223269b6201366e55c796a15a2f2211d (PR #2387, backend-related option `logtype`); testSampleRegexsZZZ-GENERIC-EXAMPLE covering now negative case also (other daemon in prefix line) --- ChangeLog | 2 ++ config/filter.d/common.conf | 8 ++++---- config/filter.d/monit.conf | 8 ++++++-- fail2ban/tests/files/logs/zzz-generic-example | 3 +++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index a81ba0aa..3d7e73be 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### Fixes * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) +* `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` + should be interpolated in definition section (inside the filter-config, gh-2650) ### New Features diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index 16897c8e..13286038 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -25,7 +25,7 @@ __pid_re = (?:\[\d+\]) # Daemon name (with optional source_file:line or whatever) # EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix) -__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? +__daemon_re = [\[\(]?<_daemon>(?:\(\S+\))?[\]\)]?:? # extra daemon info # EXAMPLE: [ID 800047 auth.info] @@ -33,7 +33,7 @@ __daemon_extra_re = \[ID \d+ \S+\] # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] -__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?) +__daemon_combs_re = (?:<__pid_re>?:\s+<__daemon_re>|<__daemon_re><__pid_re>?:?) # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] @@ -69,12 +69,12 @@ datepattern = /datepattern> [lt_file] # Common line prefixes for logtype "file": -__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)? +__prefix_line = <__date_ambit>?\s*(?:<__bsd_syslog_verbose>\s+)?(?:<__hostname>\s+)?(?:<__kernel_prefix>\s+)?(?:<__vserver>\s+)?(?:<__daemon_combs_re>\s+)?(?:<__daemon_extra_re>\s+)? datepattern = {^LN-BEG} [lt_short] # Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry): -__prefix_line = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)? +__prefix_line = \s*(?:<__hostname>\s+)?(?:<_daemon><__pid_re>?:?\s+)?(?:<__kernel_prefix>\s+)? datepattern = %(lt_file/datepattern)s [lt_journal] __prefix_line = %(lt_short/__prefix_line)s diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index b652a1f4..fdaee9c3 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -8,13 +8,17 @@ # common.local before = common.conf +# [DEFAULT] +# logtype = short + [Definition] _daemon = monit +_prefix = Warning|HttpRequest + # Regexp for previous (accessing monit httpd) and new (access denied) versions -failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ - ^%(__prefix_line)s\w+: access denied -- client : (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$ +failregex = ^%(__prefix_line)s(?:error\s*:\s+)?(?:%(_prefix)s):\s+(?:access denied\s+--\s+)?[Cc]lient '?'?(?:\s+supplied|\s*:)\s+(?:unknown user '[^']+'|wrong password for user '[^']*'|empty password) # Ignore login with empty user (first connect, no user specified) # ignoreregex = %(__prefix_line)s\w+: access denied -- client : (?:unknown user '') diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index d0c31740..d0bd3322 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -60,3 +60,6 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ [Jun 21 16:56:03] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.251 # failJSON: { "match": false, "desc": "test 2nd ignoreregex" } [Jun 21 16:56:04] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.252 + +# failJSON: { "match": false, "desc": "ignore other daemon" } +[Jun 21 16:56:04] machine captain-nemo(pam_unix)[55555] F2B: error from 192.0.2.2 From 781a25512b107828aff71998c19f2fa4dbf471c1 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 6 Mar 2020 19:04:39 +0100 Subject: [PATCH 035/136] travis CI: add 3.9-dev as target --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1f218c81..59a50313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ matrix: - python: 3.6 - python: 3.7 - python: 3.8 + - python: 3.9-dev - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" From 55e76c0b807e87f6a04d459bb9c59da33c98572b Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 6 Mar 2020 19:41:16 +0100 Subject: [PATCH 036/136] restore isAlive method removed in python 3.9 --- fail2ban/server/jailthread.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index d0430367..94f34542 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -120,3 +120,6 @@ class JailThread(Thread): ## python 2.x replace binding of private __bootstrap method: if sys.version_info < (3,): # pragma: 3.x no cover JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap +## python 3.9, restore isAlive method: +elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover + JailThread.isAlive = JailThread.is_alive From 9d7388e68448e9294e568a8ad21599e719c914b0 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 6 Mar 2020 20:04:18 +0100 Subject: [PATCH 037/136] Thread: is_alive instead of isAlive (removed in py-3.9) --- fail2ban/tests/sockettestcase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 4e14ece5..8cd22a41 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -86,7 +86,7 @@ class Socket(LogCaptureTestCase): def _stopServerThread(self): serverThread = self.serverThread # wait for end of thread : - Utils.wait_for(lambda: not serverThread.isAlive() + Utils.wait_for(lambda: not serverThread.is_alive() or serverThread.join(Utils.DEFAULT_SLEEP_TIME), unittest.F2B.maxWaitTime(10)) self.serverThread = None @@ -97,7 +97,7 @@ class Socket(LogCaptureTestCase): self.server.close() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) # clean : self.server.stop() self.assertFalse(self.server.isActive()) @@ -138,7 +138,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) @@ -179,7 +179,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) def testLoopErrors(self): # replace poll handler to produce error in loop-cycle: @@ -215,7 +215,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) From d4da9afd7fefcbba94f95d636832c3821073657e Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 6 Mar 2020 20:29:48 +0100 Subject: [PATCH 038/136] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 3d7e73be..36c15891 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition IPv6-capable now. ### Fixes +* python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` From e3737bb7c095f7c029e1b087027995653db4535c Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 17:20:19 +0100 Subject: [PATCH 039/136] filter stability fix: prevent race condition - no ban if filter (backend) is continuously busy if too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) --- ChangeLog | 2 ++ fail2ban/server/filter.py | 8 +++++++- fail2ban/server/filtergamin.py | 3 ++- fail2ban/server/filterpoll.py | 3 ++- fail2ban/server/filterpyinotify.py | 3 ++- fail2ban/server/filtersystemd.py | 3 ++- fail2ban/tests/filtertestcase.py | 5 +++++ 7 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3d7e73be..6bb95a8b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition IPv6-capable now. ### Fixes +* [stability] prevent race condition - no ban if filter (backend) is continuously busy if + too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 112569c2..c668e77d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -111,6 +111,8 @@ class Filter(JailThread): self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): + self.banASAP = True ## Ticks counter self.ticks = 0 ## Thread name: @@ -625,7 +627,11 @@ class Filter(JailThread): logSys.info( "[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - self.failManager.addFailure(tick) + attempts = self.failManager.addFailure(tick) + # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, + # we can speedup ban, so do it as soon as possible: + if self.banASAP and attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 77c81757..078246de 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -79,7 +79,8 @@ class FilterGamin(FileFilter): this is a common logic and must be shared/provided by FileFilter """ self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False ## diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 228a2c8b..b4d8ab14 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -117,7 +117,8 @@ class FilterPoll(FileFilter): self.ticks += 1 if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index ca6b253f..185305ca 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -140,7 +140,8 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False def _addPending(self, path, reason, isDir=False): diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 870b3058..47fc891e 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -318,7 +318,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover else: break if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = 0 # update position in log (time and iso string): if self.jail.database is not None: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 959d96b7..bd1d26eb 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -399,6 +399,7 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addFailRegex('^') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') self.filter.setFindTime(10); # max 10 seconds back + self.filter.setMaxRetry(5); # don't ban here # self.pruneLog('[phase 1] DST time jump') # check local time jump (DST hole): @@ -757,6 +758,7 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -974,6 +976,7 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1272,6 +1275,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1525,6 +1529,7 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') From ab363a2c0e691bf966179dfb3df0e7c938e0f08d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 17:28:33 +0100 Subject: [PATCH 040/136] small amend with fix still one test (ban unexpected in this old artificial test-cases, todo - such tests should be rewritten or removed) --- fail2ban/tests/filtertestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index bd1d26eb..202f3fbb 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1719,6 +1719,7 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) + filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures From 68f827e1f3945b735f214e15b224a926bb4bf170 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 18:03:27 +0100 Subject: [PATCH 041/136] small optimization for manually (via client / protocol) signaled attempt (performBan only if maxretry gets reached) --- fail2ban/server/filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index c668e77d..e7f3e01d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -457,10 +457,10 @@ class Filter(JailThread): logSys.info( "[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - self.failManager.addFailure(ticket, len(matches) or 1) - + attempts = self.failManager.addFailure(ticket, len(matches) or 1) # Perform the ban if this attempt is resulted to: - self.performBan(ip) + if attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) return 1 From bc2b81133c2c5a460673ddab54fe30bcc2af9ecd Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 22:07:32 +0100 Subject: [PATCH 042/136] pyinotify backend: guarantees initial scanning of log-file by start (retarded via pending event if filter not yet active) --- ChangeLog | 1 + fail2ban/server/filterpyinotify.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6a6b4451..3781f467 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### Fixes * [stability] prevent race condition - no ban if filter (backend) is continuously busy if too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) +* pyinotify-backend sporadically avoided initial scanning of log-file by start * python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 185305ca..6d0172da 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -271,7 +271,13 @@ class FilterPyinotify(FileFilter): def _addLogPath(self, path): self._addFileWatcher(path) - self._process_file(path) + # initial scan: + if self.active: + # we can execute it right now: + self._process_file(path) + else: + # retard until filter gets started: + self._addPending(path, ('INITIAL', path)) ## # Delete a log path From b43dc147b5177019a1dcb51de9833139e904cc21 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 22:20:01 +0100 Subject: [PATCH 043/136] amend to RC-fix 9f1c6f1617018a0e00fc8bf7bfd62db2c17fa11a (gh-2660): resolves bottleneck by initial scanning of a lot of messages (or evildoers generating many messages) causes repeated ban, that will be ignored but could cause entering of "long" sleep in actions thread previously; speedup recognition banning queue has entries to begin check-ban process in actions thread --- fail2ban/server/actions.py | 10 ++++++---- fail2ban/server/jail.py | 6 ++++++ fail2ban/tests/dummyjail.py | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index d1f46ac0..902d7aa6 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -318,8 +318,10 @@ class Actions(JailThread, Mapping): logSys.debug("Actions: leave idle mode") continue # wait for ban (stop if gets inactive): - bancnt = Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime) - cnt += bancnt + bancnt = 0 + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, self.sleeptime): + bancnt = self.__checkBan() + cnt += bancnt # unban if nothing is banned not later than banned tickets >= banPrecedence if not bancnt or cnt >= self.banPrecedence: if self.active: @@ -495,8 +497,8 @@ class Actions(JailThread, Mapping): cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: cnt += self.__reBan(bTicket) - # add ban to database: - if not bTicket.restored and self._jail.database is not None: + # add ban to database (and ignore too old tickets, replace it with inOperation later): + if not bTicket.restored and self._jail.database is not None and bTicket.getTime() >= MyTime.time() - 60: self._jail.database.addBan(self._jail, bTicket) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index b8a1c1f4..048aded9 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -191,6 +191,12 @@ class Jail(object): ("Actions", self.actions.status(flavor=flavor)), ] + @property + def hasFailTickets(self): + """Retrieve whether queue has tickets to ban. + """ + return not self.__queue.empty() + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 9e9aaeed..eaa4a564 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -49,6 +49,10 @@ class DummyJail(Jail): with self.lock: return bool(self.queue) + @property + def hasFailTickets(self): + return bool(self.queue) + def putFailTicket(self, ticket): with self.lock: self.queue.append(ticket) From b64a435b0eac0a1298235e320f1450382a2ac9fe Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 22:34:15 +0100 Subject: [PATCH 044/136] ignore only not banned old (repeated and ignored) tickets --- fail2ban/server/actions.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 902d7aa6..35b027ae 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -497,9 +497,12 @@ class Actions(JailThread, Mapping): cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: cnt += self.__reBan(bTicket) - # add ban to database (and ignore too old tickets, replace it with inOperation later): - if not bTicket.restored and self._jail.database is not None and bTicket.getTime() >= MyTime.time() - 60: - self._jail.database.addBan(self._jail, bTicket) + # add ban to database: + if not bTicket.restored and self._jail.database is not None: + # ignore too old (repeated and ignored) tickets, + # [todo] replace it with inOperation later (once it gets back-ported): + if not reason and bTicket.getTime() >= MyTime.time() - 60: + self._jail.database.addBan(self._jail, bTicket) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) From 8547ea7ea057dad5708e11b365c8f060f038a1d0 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Mar 2020 23:16:04 +0100 Subject: [PATCH 045/136] resolve sporadic minor issue - check pending can refresh watcher (monitor) that gets deleting, and there may be no wdInt to delete --- fail2ban/server/filterpyinotify.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 6d0172da..9796e26f 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -188,7 +188,8 @@ class FilterPyinotify(FileFilter): for path, isDir in found.iteritems(): self._delPending(path) # refresh monitoring of this: - self._refreshWatcher(path, isDir=isDir) + if isDir is not None: + self._refreshWatcher(path, isDir=isDir) if isDir: # check all files belong to this dir: for logpath in self.__watchFiles: @@ -276,8 +277,8 @@ class FilterPyinotify(FileFilter): # we can execute it right now: self._process_file(path) else: - # retard until filter gets started: - self._addPending(path, ('INITIAL', path)) + # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): + self._addPending(path, ('INITIAL', path), isDir=None) ## # Delete a log path @@ -285,9 +286,9 @@ class FilterPyinotify(FileFilter): # @param path the log file to delete def _delLogPath(self, path): + self._delPending(path) if not self._delFileWatcher(path): # pragma: no cover logSys.error("Failed to remove watch on path: %s", path) - self._delPending(path) path_dir = dirname(path) for k in self.__watchFiles: @@ -297,8 +298,8 @@ class FilterPyinotify(FileFilter): if path_dir: # Remove watches for the directory # since there is no other monitored file under this directory - self._delDirWatcher(path_dir) self._delPending(path_dir) + self._delDirWatcher(path_dir) # pyinotify.ProcessEvent default handler: def __process_default(self, event): From 606bf110c99c0b491b10f336b67675311f279f1a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 16 Mar 2020 17:29:09 +0100 Subject: [PATCH 046/136] filter.d/sshd.conf (mode `ddos`): fixed "connection reset" regex (seems to have same syntax now as closed), so both regex's combined now to single RE (closes gh-2662) --- config/filter.d/sshd.conf | 3 +-- .../tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 3 +-- fail2ban/tests/files/logs/sshd | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 12631cb3..7a7f5e48 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -73,11 +73,10 @@ mdre-normal-other = ^(Connection closed|Disconnected) ^kex_exchange_identification: client sent invalid protocol identifier ^Bad protocol version identification '.*' from - ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer # same as mdre-normal-other, but as failure (without ) and [preauth] only: -mdre-ddos-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ +mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index d61a6520..4ff4ac68 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -57,8 +57,7 @@ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from ^%(__prefix_line_sl)sBad protocol version identification '.*' from - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 0385f38c..e45ca90d 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -296,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } From 343ec1cdd296530f331637c725bd2bb0549e01e6 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 18 Mar 2020 20:37:25 +0100 Subject: [PATCH 047/136] test-causes: avoid host-depending issue (mistakenly ignoring IP 127.0.0.2 as own address) - replace loop-back addr with test sub-net addr (and disable ignoreself) --- fail2ban/tests/observertestcase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 8e944454..e379ccd1 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -224,7 +223,7 @@ class BanTimeIncrDB(LogCaptureTestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -385,10 +384,12 @@ class BanTimeIncrDB(LogCaptureTestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -477,7 +478,7 @@ class BanTimeIncrDB(LogCaptureTestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) failManager = FailManager() failManager.setMaxRetry(3) From fc175fa78a2c0c91f6f90745c12a56e67605a279 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:10:32 +0200 Subject: [PATCH 048/136] performance: optimize simplest case whether the ignoreip is a single IP (not subnet/dns) - uses a set instead of list (holds single IPs and subnets/dns in different lists); decrease log level for ignored duplicates (warning is too heavy here) --- fail2ban/server/filter.py | 24 ++++++++++++++++++------ fail2ban/server/ipdns.py | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e7f3e01d..a92acb8b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -80,6 +80,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -489,28 +490,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -550,8 +559,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 6648dac6..335fc473 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -379,6 +379,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other From d21a24de8e1bb998fe8a54908b650250ba524fd0 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:39:36 +0200 Subject: [PATCH 049/136] more test cases for IP/DNS (and use dummies if no-network set by testing) --- fail2ban/tests/filtertestcase.py | 16 ++++++++++++---- fail2ban/tests/utils.py | 12 ++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 202f3fbb..a511b5d0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1899,7 +1899,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip4 = IPAddr('192.0.2.1') ip6 = IPAddr('2001:DB8::') self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) self.assertTrue(asip('192.0.2.1').isIPv4) self.assertTrue(id(asip(ip4)) == id(ip4)) @@ -1908,6 +1910,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1916,6 +1919,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1938,7 +1942,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertSortedEqual(res, []) - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1947,7 +1951,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1971,7 +1975,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: @@ -1993,8 +1997,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -2019,6 +2025,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -2114,7 +2122,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index f5ffb978..dc12a5be 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -331,13 +331,21 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), ('192.168.0.', set()), ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): From 136781d627aa70ab88f3fac3b6df398c21dc7387 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Apr 2020 12:17:59 +0200 Subject: [PATCH 050/136] filter.d/sshd.conf: fixed regex for mode `extra` - "No authentication methods available" (supported seems to be optional now, gh-2682) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 7a7f5e48..31e61b96 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -78,7 +78,7 @@ mdre-ddos = ^Did not receive identification string from # same as mdre-normal-other, but as failure (without ) and [preauth] only: mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 4ff4ac68..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -60,7 +60,7 @@ mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e45ca90d..1bf9d913 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -330,6 +330,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } From 2912bc640b3335bffe20d03afa74d84c7a3c4f56 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Thu, 9 Apr 2020 16:42:08 +0200 Subject: [PATCH 051/136] New Gitlab jail --- config/filter.d/gitlab.conf | 6 ++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/gitlab | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 config/filter.d/gitlab.conf create mode 100644 fail2ban/tests/files/logs/gitlab diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=.+ ip=$ diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..e5d16656 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -821,6 +821,10 @@ udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + [bitwarden] port = http,https logpath = /home/*/bwdata/logs/identity/Identity/log.txt diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..222df642 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 78651de7e504bad84e73040dfd7ea530591d0385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 14 Apr 2020 12:25:18 +0200 Subject: [PATCH 052/136] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 3781f467..c722db2b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition should be interpolated in definition section (inside the filter-config, gh-2650) ### New Features +* new filter and jail for GitLab recognizing failed application logins (gh-2689) ### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; From 7e3061e7ace0e973378a17de04a2692142e1a6c1 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Apr 2020 17:35:04 +0200 Subject: [PATCH 053/136] fail2ban.service systemd unit template: don't add user site directory to python system path (avoids accessing of `/root/.local` directory, prevents SE linux audit warning at daemon startup, gh-2688) --- files/fail2ban.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 5e540545..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftabl [Service] Type=simple +Environment="PYTHONNOUSERSITE=1" ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local From 06b46e92eb3fc11c29974fea1753a96edaf8ec20 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Apr 2020 21:34:30 +0200 Subject: [PATCH 054/136] jail.conf: don't specify `action` directly in jails (use `action_` or `banaction` instead); no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified per jail or in default section in jail.local), closes gh-2357; ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686); don't use %(banaction)s interpolation because it can be complex value (containing `[...]`), so would bother the action interpolation. --- ChangeLog | 6 ++++++ config/jail.conf | 30 ++++++++++++++---------------- fail2ban/client/actionreader.py | 10 +++++++--- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3781f467..6329d39a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,12 @@ ver. 0.10.6-dev (20??/??/??) - development edition * python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..bbf8740f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -174,19 +174,19 @@ banaction_allports = iptables-allports action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mwl = %(action_)s %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines @@ -333,7 +333,7 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log @@ -706,8 +706,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -718,9 +718,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -728,9 +727,8 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 @@ -818,8 +816,8 @@ logpath = /opt/cstrike/logs/L[0-9]*.log # Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] [bitwarden] port = http,https @@ -871,8 +869,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index e5bee154..131e37cb 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -52,13 +52,17 @@ class ActionReader(DefinitionInitConfigReader): } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) From affd9cef5f2ddb5c596e1aa3789ba18b5c987ba1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Apr 2020 13:32:17 +0200 Subject: [PATCH 055/136] filter.d/courier-smtp.conf: prefregex extended to consider port in log-message (closes gh-2697) --- ChangeLog | 1 + config/filter.d/courier-smtp.conf | 2 +- fail2ban/tests/files/logs/courier-smtp | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6329d39a..b001ae58 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) ### New Features diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=,.+$ +prefregex = ^%(__prefix_line)serror,relay=,(?:port=\d+,)?.+$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..cea73073 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -12,3 +12,5 @@ Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com From 6b90ca820f5244fd88e8b407419da359d5d068b9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Apr 2020 13:08:24 +0200 Subject: [PATCH 056/136] filter.d/traefik-auth.conf: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle the match of username differently: - `normal`: matches 401 with supplied username only - `ddos`: matches 401 without supplied username only - `aggressive`: matches 401 and any variant (with and without username) closes gh-2693 --- ChangeLog | 5 +++++ config/filter.d/traefik-auth.conf | 22 +++++++++++++++++++++- fail2ban/tests/files/logs/traefik-auth | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b001ae58..f5d3dd6d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,6 +47,11 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) ### New Features diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf index 8321a138..8022fee1 100644 --- a/config/filter.d/traefik-auth.conf +++ b/config/filter.d/traefik-auth.conf @@ -51,6 +51,26 @@ [Definition] -failregex = ^ \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^ \- > \[\] \"(?:) [^\"]+\" 401\b ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )\S+ +usrre-ddos = - +usrre-aggressive = \S+ \ No newline at end of file diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth index 3e7a8987..edfe7306 100644 --- a/fail2ban/tests/files/logs/traefik-auth +++ b/fail2ban/tests/files/logs/traefik-auth @@ -1,6 +1,23 @@ +# filterOptions: [{"mode": "normal"}] + # failJSON: { "match": false } 10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "ddos"}] + +# failJSON: { "match": false } +10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}] + # failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" } 10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms +# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" } +10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms # failJSON: { "match": false } 10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" } +10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms From 87a1a2f1a112ce5bfef8357758435266c4f6b43d Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 14:52:38 +0200 Subject: [PATCH 057/136] action.d/*-ipset*.conf: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only) --- config/action.d/firewallcmd-ipset.conf | 22 +++++++++++------ .../iptables-ipset-proto6-allports.conf | 24 ++++++++++++------- config/action.d/iptables-ipset-proto6.conf | 24 ++++++++++++------- config/action.d/shorewall-ipset-proto6.conf | 22 ++++++++++------- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index dcf20375..9dd9fbb2 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist actionunban = ipset del -exist @@ -40,11 +40,19 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype # Notes.: defines additions to the blocking rule @@ -69,7 +77,7 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 # DEV NOTES: diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index dc7d63a7..4f200db0 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index f88777b8..8956ec6a 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index fc7dd24e..cbcc5524 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -76,10 +76,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -[Init] +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) From 12be3ed77d9cbe0ef6fed4ac26f9859527f9c448 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 15:17:42 +0200 Subject: [PATCH 058/136] test cases fixed --- fail2ban/tests/servertestcase.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index a51c4f85..b771ab50 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1509,14 +1509,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0 `", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1532,27 +1532,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain=""]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1568,13 +1568,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", @@ -1852,14 +1852,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1875,27 +1875,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=, protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=, protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1911,13 +1911,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", From da1652d0d71074a2af8ae69512fbca24fafdf385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 26 Apr 2020 12:26:55 +0200 Subject: [PATCH 059/136] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index f5d3dd6d..2ae56d99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 5da2422f616f6245982f38693df2e61ccf24dbb4 Mon Sep 17 00:00:00 2001 From: Ilya Date: Wed, 11 Mar 2020 14:43:45 +0300 Subject: [PATCH 060/136] Fix actionunban Add command to remove new line character. Needed for working removing rule from cloudflare firewall. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..70e5ee3f 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) [Init] From 8b3b9addd10dec9fec47b8f7fd3a971c326fe430 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 20 Mar 2020 13:52:17 +0300 Subject: [PATCH 061/136] Change tool from 'cut' to 'sed' Sed regex was tested - it works. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 70e5ee3f..d00db98b 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) [Init] From 852670bc99be2f30a64dd5d096d5ff375b6a2e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Sun, 27 May 2018 08:15:33 +0200 Subject: [PATCH 062/136] CloudFlare started to indent their API responses We need to use https://github.com/stedolan/jq to parse it. --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index d00db98b..27a0b6b5 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl and jq. # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') [Init] From 5b8fc3b51a203a7428dd66f07fb8480ec894860d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 Dec 2018 14:35:17 +0100 Subject: [PATCH 063/136] cloudflare: fixes ip to id conversion by unban using jq normalized URIs and parameters, notes gets a jail-name (should be possible to differentiate the same IP across several jails) --- config/action.d/cloudflare.conf | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 27a0b6b5..5125777c 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + | jq -r '.result[0].id') + if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 1c1b671c745dbe0e1f9a096fd1953d0257e8b958 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Dec 2018 11:27:53 +0100 Subject: [PATCH 064/136] Update cloudflare.conf --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 5125777c..4c0a3810 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -59,13 +59,13 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | jq -r '.result[0].id') if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" _cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules -_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 01e92ce4a617ff8cf95d47b67cdd77481a3efdd7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 16 Mar 2020 18:28:45 +0100 Subject: [PATCH 065/136] added fallback using tr and sed (jq is optional now) --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4c0a3810..361cb177 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl and jq. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ - | jq -r '.result[0].id') + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" From 42aef09d695f98794066118015db7b72442c6116 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 27 Apr 2020 19:38:48 +0200 Subject: [PATCH 066/136] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2ae56d99..d744dba0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 43f699b872ab75132a2188be29706d602830254e Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 May 2020 17:32:13 +0200 Subject: [PATCH 067/136] grammar / typos --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c7177f13..4d236b34 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,7 +60,7 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # From afb7a931632cded1312facf2f82d2a33caf7ffd2 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:27:48 +0200 Subject: [PATCH 068/136] amend to 368aa9e77570519b37fb57c9dbc5112d4c4b7382: fix time in gitlab test (GMT in log due to TZ-suffix `Z`, CEST in test-suite) --- fail2ban/tests/files/logs/gitlab | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab index 222df642..70ddc0e8 100644 --- a/fail2ban/tests/files/logs/gitlab +++ b/fail2ban/tests/files/logs/gitlab @@ -1,5 +1,5 @@ # Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log -# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } 2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 -# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } 2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 0ae2ef68be57f056038adaf064570e6725c6f20f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:36:06 +0200 Subject: [PATCH 069/136] ensure iterator is safe (traverse over the list in snapshot created within a lock), avoids getting modified state as well as "dictionary changed size during iteration" errors --- fail2ban/server/banmanager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 8c0a6965..479ba26f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -104,7 +104,7 @@ class BanManager: def getBanList(self): with self.__lock: - return self.__banList.keys() + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,8 +112,9 @@ class BanManager: # @return ban list iterator def __iter__(self): + # ensure iterator is safe (traverse over the list in snapshot created within lock): with self.__lock: - return self.__banList.itervalues() + return iter(list(self.__banList.values())) ## # Returns normalized value From 54b2208690e3c2fff00fbd9b197984d880e29a02 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 16:31:54 +0200 Subject: [PATCH 070/136] extends protocol/client with banned status (retrieve information whether an IP is banned and/or in which jails), implements FR gh-2725 --- fail2ban/protocol.py | 4 +++ fail2ban/server/actions.py | 8 ++++++ fail2ban/server/server.py | 26 ++++++++++++++++++ fail2ban/server/transmitter.py | 8 +++++- fail2ban/tests/fail2banclienttestcase.py | 34 +++++++++++++++++++++++- man/fail2ban-client.1 | 16 +++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 92b0dcc0..18b901e8 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban ... ", "unbans (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned ... ]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -120,6 +122,8 @@ protocol = [ ["set action ", "sets the of for the action for "], ["set action [ ]", "calls the with for the action for "], ['', "JAIL INFORMATION", ""], +["get banned", "return banned IPs of "], +["get banned ... ]", "return 1 if IP is banned in otherwise 0, or a list of 1/0 for multiple IPs"], ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 35b027ae..6123605d 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -210,6 +210,14 @@ class Actions(JailThread, Mapping): def getBanTime(self): return self.__banManager.getBanTime() + def getBanned(self, ids): + lst = self.__banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + def addBannedIP(self, ip): """Ban an IP or list of IPs.""" unixTime = MyTime.time() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index abb312da..b12c8f9f 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -521,6 +521,32 @@ class Server: cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index de80f624..aff9071c 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -118,6 +118,9 @@ class Transmitter: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) elif name == "echo": return command[1:] elif name == "server-status": @@ -424,7 +427,10 @@ class Transmitter: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 6c79800e..a334b568 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -421,6 +421,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -1007,6 +1015,30 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail2] Found 192.0.2.3", "[test-jail2] Ban 192.0.2.3", all=True) + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # rotate logs: _write_file(test1log, "w+") diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 84f846f2..6a3247e1 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -111,6 +111,14 @@ jails and database) unbans (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned ... ]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -356,6 +364,14 @@ for .IP JAIL INFORMATION .TP +\fBget banned\fR +return banned IPs of +.TP +\fBget banned ... ]\fR +return 1 if IP is banned in +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget logpath\fR gets the list of the monitored files for From fa1ff4c5d8757bf8e07bd43794d27290293192e5 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 May 2020 13:36:09 +0200 Subject: [PATCH 071/136] assertSortedEqual: fixed sort of nested lists, switch default of nestedOnly to False (comparison of unsorted lists is rarely needed) --- fail2ban/tests/misctestcase.py | 10 +++++++++- fail2ban/tests/utils.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 9b986f53..43d76802 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -390,7 +390,15 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index dc12a5be..47e5b909 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -556,7 +556,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -566,6 +566,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -584,8 +591,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) From 9b6da03c90139db1a8820a24736462c5c3477ba8 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 26 May 2020 13:14:37 +0200 Subject: [PATCH 072/136] amend to e786dbf132689133c29671871718a97f93b8912a: removes space between name and [pid] by normal non-verbose logging, padding without truncate now; test coverage for getVerbosityFormat; closes #2734 --- fail2ban/helpers.py | 2 +- fail2ban/tests/misctestcase.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 6f2bcdd7..3ef7d543 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -302,7 +302,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True if addtime: fmt = ' %(asctime)-15s' + fmt else: # default (not verbose): - fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt + fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt if addtime: fmt = "%(asctime)s " + fmt # remove padding if not needed: diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 43d76802..c1a6a345 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -34,7 +34,7 @@ from StringIO import StringIO from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ - splitwords, uni_decode, uni_string + getVerbosityFormat, splitwords, uni_decode, uni_string from ..server.mytime import MyTime @@ -404,6 +404,14 @@ class TestsUtilsTest(LogCaptureTestCase): self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", self.assertSortedEqual, ['A', 'B'], ['C', 'B']) + def testVerbosityFormat(self): + self.assertEqual(getVerbosityFormat(1), + '%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s') + self.assertEqual(getVerbosityFormat(1, padding=False), + '%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s') + self.assertEqual(getVerbosityFormat(1, addtime=False, padding=False), + '%(name)s[%(process)d]: %(levelname)s %(message)s') + def testFormatterWithTraceBack(self): strout = StringIO() Formatter = FormatterWithTraceBack From 5a0edf61c96998a29d4e7afd2105cf8f5b855318 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 8 Jun 2020 14:38:26 +0200 Subject: [PATCH 073/136] filter.d/sshd.conf: normalizing of user pattern in all RE's, allowing empty user (gh-2749) --- ChangeLog | 1 + config/filter.d/sshd.conf | 16 ++++++++-------- fail2ban/tests/files/logs/sshd | 5 +++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index c5be981b..013cba89 100644 --- a/ChangeLog +++ b/ChangeLog @@ -56,6 +56,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition - `normal`: matches 401 with supplied username only - `ddos`: matches 401 without supplied username only - `aggressive`: matches 401 and any variant (with and without username) +* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) ### New Features * new filter and jail for GitLab recognizing failed application logins (gh-2689) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 31e61b96..4c86dca0 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* __on_port_opt = (?: (?:port \d+|on \S+)){0,2} # close by authenticating user: -__authng_user = (?: (?:invalid|authenticating) user \S+|.+?)? +__authng_user = (?: (?:invalid|authenticating) user \S+|.*?)? # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. @@ -44,18 +44,18 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^Failed for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^ROOT LOGIN REFUSED FROM ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ - ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ - ^User .+ from not allowed because listed in DenyUsers%(__suff)s$ - ^User .+ from not allowed because not in any group%(__suff)s$ + ^User \S+|.*? from not allowed because not listed in AllowUsers%(__suff)s$ + ^User \S+|.*? from not allowed because listed in DenyUsers%(__suff)s$ + ^User \S+|.*? from not allowed because not in any group%(__suff)s$ ^refused connect from \S+ \(\) ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ - ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ + ^User \S+|.*? from not allowed because a group is listed in DenyGroups%(__suff)s$ + ^User \S+|.*? from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ ^maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ - ^User .+ not allowed because account is locked%(__suff)s + ^User \S+|.*? not allowed because account is locked%(__suff)s ^Disconnecting(?: from)?(?: (?:invalid|authenticating)) user \S+ %(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for \S+|.*?)?%(__suff)s$ ^Received disconnect from %(__on_port_opt)s:\s*11: -other> ^Accepted \w+ for \S+ from (?:\s|$) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 1bf9d913..9fff416a 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -321,6 +321,11 @@ Mar 15 09:21:02 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-07-18T17:19:11", "match": true , "host": "192.0.2.4", "desc": "ddos: disconnect on preauth phase, gh-2115" } Jul 18 17:19:11 srv sshd[2101]: Disconnected from 192.0.2.4 port 36985 [preauth] +# failJSON: { "time": "2005-06-06T04:17:04", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:04 host sshd[1189074]: Invalid user from 192.0.2.68 port 34916 +# failJSON: { "time": "2005-06-06T04:17:09", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:09 host sshd[1189074]: Connection closed by invalid user 192.0.2.68 port 34916 [preauth] + # filterOptions: [{"mode": "extra"}, {"mode": "aggressive"}] # several other cases from gh-864: From dd8081ade5ace23782efcf0c96b7f8c9f16228a7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 20:00:42 +0200 Subject: [PATCH 074/136] extends capturing alternate tags in filter, implementing new tag prefix `` with all value of tags), for examples see new tests in fail2banregextestcase; closes gh-2755 (extends #1454 and #1698). --- fail2ban/server/action.py | 2 +- fail2ban/server/failregex.py | 56 +++++++++++++++++-------- fail2ban/server/ipdns.py | 2 +- fail2ban/tests/fail2banregextestcase.py | 17 ++++++++ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d0262171..1c313cf0 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -868,7 +868,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 0ae9acc5..9bbf5779 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -87,20 +87,24 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -128,18 +132,23 @@ class Regex: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex self._altValues = {} + self._tupleValues = {} for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n + n = COMPLNAME_CRE.match(k) + if n: + if n.group(1) == ALTNAME_PRE: + self._altValues[k] = mapTag2Opt(n.group(2)) + else: + self._tupleValues[k] = mapTag2Opt(n.group(2)) self._altValues = list(self._altValues.items()) if len(self._altValues) else None + self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) # set fetch handler depending on presence of alternate tags: - self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -284,12 +293,23 @@ class Regex: def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..18e1bd02 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -337,7 +337,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..334fad1c 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -340,6 +340,23 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) self.assertLogged('kevin') self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s* \S+')) + self.assertLogged(str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left 192.0.2.3 right', + r'^\s*\S+ \S+')) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*\S+ : \S+')) + self.assertLogged(str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() # row with id : self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) self.assertLogged("['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) From ec3000798d9b42c3f05d40e810f43ae444c14d97 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Jun 2020 21:25:42 +0200 Subject: [PATCH 075/136] ensure that set of alternate tags or combine tuple tags take place ordered (sort the lists by its name or index) --- fail2ban/server/failregex.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 9bbf5779..4032ecdb 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -131,23 +131,26 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} - self._tupleValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): n = COMPLNAME_CRE.match(k) if n: - if n.group(1) == ALTNAME_PRE: - self._altValues[k] = mapTag2Opt(n.group(2)) + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) else: - self._tupleValues[k] = mapTag2Opt(n.group(2)) - self._altValues = list(self._altValues.items()) if len(self._altValues) else None - self._tupleValues = list(self._tupleValues.items()) if len(self._tupleValues) else None + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - # set fetch handler depending on presence of alternate tags: + # set fetch handler depending on presence of alternate (or tuple) tags: self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): From 309c8dddd7adc2de140ed5a72088cd4f2dcc9b91 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Jun 2020 19:20:36 +0200 Subject: [PATCH 076/136] action.d/nftables.conf (type=multiport only): fixed port range selector (replacing `:` with `-`) --- config/action.d/nftables.conf | 2 +- fail2ban/tests/servertestcase.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf index c1fb8550..77cf3661 100644 --- a/config/action.d/nftables.conf +++ b/config/action.d/nftables.conf @@ -34,7 +34,7 @@ type = multiport rule_match-custom = rule_match-allports = meta l4proto \{ \} -rule_match-multiport = $proto dport \{ \} +rule_match-multiport = $proto dport \{ $(echo '' | sed s/:/-/g) \} match = > # Option: rule_stat diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index b771ab50..f1b667b1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1296,11 +1296,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), 'ip4-start': ( r"`nft add set inet f2b-table addr-set-j-w-nft-mp \{ type ipv4_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip saddr @addr-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip saddr @addr-set-j-w-nft-mp reject`", ), 'ip6-start': ( r"`nft add set inet f2b-table addr6-set-j-w-nft-mp \{ type ipv6_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip6 saddr @addr6-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip6 saddr @addr6-set-j-w-nft-mp reject`", ), 'flush': ( "`{ nft flush set inet f2b-table addr-set-j-w-nft-mp 2> /dev/null; } || ", From 08dbe4abd593c09be8be9101964c2ee1915374f5 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 3 Jul 2020 13:45:29 +0200 Subject: [PATCH 077/136] fixed comment for loglevel, default is INFO --- config/fail2ban.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index ba0e9204..f3867839 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -19,7 +19,7 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # loglevel = INFO From ea35f2ad754810c51f5cf01d5fa3018002917a50 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 3 Jul 2020 13:47:46 +0200 Subject: [PATCH 078/136] default loglevel is INFO --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 662b3f48..4d01b6a1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -130,7 +130,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com The items that can be set in section [Definition] are: .TP .B loglevel -verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) +verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: INFO (equal 20) .TP .B logtarget log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local From 73a8175bb00ed2f456448f6edcfbf78ab7f0630e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 13:22:02 +0200 Subject: [PATCH 079/136] resolves names conflict (command action timeout and ipset timeout); closes gh-2790 --- config/action.d/firewallcmd-ipset.conf | 14 +++++++------- .../action.d/iptables-ipset-proto6-allports.conf | 14 +++++++------- config/action.d/iptables-ipset-proto6.conf | 14 +++++++------- config/action.d/shorewall-ipset-proto6.conf | 14 +++++++------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 9dd9fbb2..42513933 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist actionunban = ipset del -exist @@ -40,18 +40,18 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index 4f200db0..addb2b95 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,18 +61,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 8956ec6a..7677564f 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,18 +61,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index cbcc5524..75eef218 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -76,16 +76,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) From 0ef8f6675d7c7d68e0b19d42071e1d1838f94627 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:25:31 +0200 Subject: [PATCH 080/136] fix travis builds (pipy in xenial, don't error if doc missing in default path after install) --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59a50313..064b678b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ matrix: - python: 2.7 name: 2.7 (xenial) - python: pypy - dist: trusty - python: 3.3 dist: trusty - python: 3.4 @@ -70,8 +69,8 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) - - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi + # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) + - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' # Test initd script - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: From 9510346507e70da6b25f27a20b3b6f6a84e8eeeb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:31:11 +0200 Subject: [PATCH 081/136] typo in skip message --- fail2ban/tests/fail2banregextestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..2da0098e 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -485,7 +485,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testLogtypeSystemdJournal(self): # pragma: no cover if not fail2banregex.FilterSystemd: - raise unittest.SkipTest('Skip test because no systemd backand available') + raise unittest.SkipTest('Skip test because no systemd backend available') self.assertTrue(_test_exec( "systemd-journal", FILTER_ZZZ_GEN +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' From 253d47d33c40c8f8f52b5d5fced6f2be7d942524 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 14:52:15 +0200 Subject: [PATCH 082/136] compat: some 2.x pypy versions produce UnicodeEncodeError: 'ascii' codec can't encode character on surrogates (uni_string must be fixed also for UTF-8 system encoding) --- fail2ban/tests/misctestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c1a6a345..458e9a23 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -201,7 +201,8 @@ class TestsUtilsTest(LogCaptureTestCase): uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') - uni_string(u'test\xcf') + if sys.version_info < (3,) and 'PyPy' not in sys.version: + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) From 98983adf761282ba4e6ad4d83a77fffcc6030573 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Aug 2020 17:14:13 +0200 Subject: [PATCH 083/136] update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 266ca614..65568f62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,8 @@ ver. 0.11.2-dev (20??/??/??) - development edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `` with all value of `` tags (gh-2755) ver. 0.11.1 (2020/01/11) - this-is-the-way From a7ad3e00ddc3a3a71e8ab5a20dcfb33f63daa0e2 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 11:54:54 +0200 Subject: [PATCH 084/136] amend to 91eca4fdeb728fa0fd2d6cf55db8eb7baa2a39a2 (#2634): server creates a RTM-directory for socket/pid file automatically (don't check its existence in client) --- fail2ban/client/fail2banclient.py | 13 ------------- fail2ban/tests/fail2banclienttestcase.py | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 7c90ca40..6ea18fda 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -168,19 +168,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): if not ret: return None - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self._conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self._conf["socket"])) - return None - if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return None - # Check already running if not self._conf["force"] and os.path.exists(self._conf["socket"]): logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index a334b568..fd6d6bbd 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -623,12 +623,6 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start") - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## not running self.execCmd(FAILED, (), "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload") @@ -724,12 +718,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock")) - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## already exists: open(pjoin(tmp, "f2b.sock"), 'a').close() self.execCmd(FAILED, (), From 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 13:57:36 +0200 Subject: [PATCH 085/136] closes gh-2758: no explicit flush (close std-channels on exit, it would cause implicit flush without to produce an error 32 "Broken pipe" on closed pipe) --- fail2ban/client/fail2bancmdline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 53c86de6..8936e03f 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -308,6 +308,10 @@ class Fail2banCmdLine(): # since method is also exposed in API via globally bound variable @staticmethod def _exit(code=0): + # implicit flush without to produce broken pipe error (32): + sys.stderr.close() + sys.stdout.close() + # exit: if hasattr(os, '_exit') and os._exit: os._exit(code) else: @@ -318,8 +322,6 @@ class Fail2banCmdLine(): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() # exit Fail2banCmdLine._exit(code) From 7d172faa50db8153265b7ac0e26f1034d7d6119f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 15:37:19 +0200 Subject: [PATCH 086/136] implements gh-2791: fail2ban-client extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS) --- ChangeLog | 1 + fail2ban/server/actions.py | 14 +++++++++++++- fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++++++----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index fb38aca1..9b30bd6a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 6123605d..4689b9d7 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -251,7 +251,7 @@ class Actions(JailThread, Mapping): if ip is None: return self.__flushBan(db) # Multiple IPs: - if isinstance(ip, list): + if isinstance(ip, (list, tuple)): missed = [] cnt = 0 for i in ip: @@ -273,6 +273,18 @@ class Actions(JailThread, Mapping): # Unban the IP. self.__unBan(ticket) else: + # Multiple IPs by subnet or dns: + if not isinstance(ip, IPAddr): + ipa = IPAddr(ip) + if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): + ips = filter( + lambda i: ( + isinstance(i, IPAddr) and (i == ipa or i.isSingle and i.isInNet(ipa)) + ), self.__banManager.getBanList() + ) + if ips: + return self.removeBannedIP(ips, db, ifexists) + # not found: msg = "%s is not banned" % ip logSys.log(logging.MSG, msg) if ifexists: diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index fd6d6bbd..104e4c57 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1158,13 +1158,26 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertNotLogged("[test-jail1] Found 192.0.2.5") # unban single ips: - self.pruneLog("[test-phase 6]") + self.pruneLog("[test-phase 6a]") self.execCmd(SUCCESS, startparams, "--async", "unban", "192.0.2.5", "192.0.2.6") self.assertLogged( "192.0.2.5 is not banned", "[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME ) + # unban ips by subnet (cidr/mask): + self.pruneLog("[test-phase 6b]") + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.2/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.2", + "[test-jail1] Unban 192.0.2.3", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.8/31", "192.0.2.100/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.8", + "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) # reload all (one jail) with unban all: self.pruneLog("[test-phase 7]") @@ -1175,8 +1188,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Jail 'test-jail1' reloaded", "[test-jail1] Unban 192.0.2.1", - "[test-jail1] Unban 192.0.2.2", - "[test-jail1] Unban 192.0.2.3", "[test-jail1] Unban 192.0.2.4", all=True ) # no restart occurred, no more ban (unbanned all using option "--unban"): @@ -1184,8 +1195,6 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", "[test-jail1] Ban 192.0.2.1", - "[test-jail1] Ban 192.0.2.2", - "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", all=True ) From 3ca69c8c0a756b58c751707e72905d3f50c4a620 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 11 Aug 2020 17:14:21 +0200 Subject: [PATCH 087/136] amend to #2791: unban subnet when subnet is in supplied subnet --- fail2ban/server/actions.py | 6 +----- fail2ban/server/ipdns.py | 5 +++++ fail2ban/tests/fail2banclienttestcase.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 4689b9d7..3308d4b2 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -277,11 +277,7 @@ class Actions(JailThread, Mapping): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter( - lambda i: ( - isinstance(i, IPAddr) and (i == ipa or i.isSingle and i.isInNet(ipa)) - ), self.__banManager.getBanList() - ) + ips = filter(ipa.contains, self.__banManager.getBanList()) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..571ccc4f 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -517,6 +517,11 @@ class IPAddr(object): return (self.addr & mask) == net.addr + def contains(self, ip): + """Return whether the object (as network) contains given IP + """ + return isinstance(ip, IPAddr) and (ip == self or ip.isInNet(self)) + # Pre-calculated map: addr to maskplen def __getMaskMap(): m6 = (1 << 128)-1 diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 104e4c57..bbd6964a 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1179,6 +1179,21 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail1] Unban 192.0.2.8", "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) + # ban/unban subnet(s): + self.pruneLog("[test-phase 6c]") + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "banip", "192.0.2.96/28", "192.0.2.112/28") + self.assertLogged( + "[test-jail1] Ban 192.0.2.96/28", + "[test-jail1] Ban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "unbanip", "192.0.2.64/26"); # contains both subnets .96/28 and .112/28 + self.assertLogged( + "[test-jail1] Unban 192.0.2.96/28", + "[test-jail1] Unban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + # reload all (one jail) with unban all: self.pruneLog("[test-phase 7]") self.execCmd(SUCCESS, startparams, From 7e8d98c4edf7ecd5b1805eb850225a0a8435320d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 13 Aug 2020 19:20:27 +0200 Subject: [PATCH 088/136] code review, fix simplest TZ issue - avoid date adjustment by assuming of last year (date without year in the future) by wrong zone (don't adjust by offset up to +24 hours) --- fail2ban/server/filter.py | 7 +++++-- fail2ban/server/filterpoll.py | 4 +++- fail2ban/server/strptime.py | 5 ++--- fail2ban/tests/files/logs/courier-smtp | 6 +++--- fail2ban/tests/filtertestcase.py | 8 +++++--- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a92acb8b..1ff81d65 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -633,6 +633,9 @@ class Filter(JailThread): fail = element[3] logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) + # ensure the time is not in the future, e. g. by some estimated (assumed) time: + if self.checkFindTime and unixTime > MyTime.time(): + unixTime = MyTime.time() tick = FailTicket(ip, unixTime, data=fail) if self._inIgnoreIPList(ip, tick): continue @@ -936,7 +939,7 @@ class FileFilter(Filter): log.setPos(lastpos) self.__logs[path] = log logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) - if autoSeek: + if autoSeek and not tail: self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific @@ -1206,7 +1209,7 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail = False): + def __init__(self, filename, encoding, tail=False): self.__filename = filename self.setEncoding(encoding) self.__tail = tail diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index b4d8ab14..7bbdfc5c 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -111,6 +111,8 @@ class FilterPoll(FileFilter): modlst = [] Utils.wait_for(lambda: not self.active or self.getModified(modlst), self.sleeptime) + if not self.active: # pragma: no cover - timing + break for filename in modlst: self.getFailures(filename) self.__modified = True @@ -140,7 +142,7 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) stats = logStats.st_mtime, logStats.st_ino, logStats.st_size - pstats = self.__prevStats.get(filename, (0)) + pstats = self.__prevStats.get(filename, (0,)) if logSys.getEffectiveLevel() <= 4: # we do not want to waste time on strftime etc if not necessary dt = logStats.st_mtime - pstats[0] diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 498d284b..1464a96d 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -291,9 +291,8 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): date_result -= datetime.timedelta(days=1) if assume_year: if not now: now = MyTime.now() - if date_result > now: - # Could be last year? - # also reset month and day as it's not yesterday... + if date_result > now + datetime.timedelta(days=1): # ignore by timezone issues (+24h) + # assume last year - also reset month and day as it's not yesterday... date_result = date_result.replace( year=year-1, month=month, day=day) diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index cea73073..6da0d0a4 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -8,9 +8,9 @@ Jul 4 18:39:39 mail courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin -# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +# failJSON: { "time": "2005-08-14T12:51:05", "match": true , "host": "192.0.2.3" } Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index a511b5d0..e8634929 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1606,7 +1606,7 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 7, 1124013544.0) + output = ('203.162.223.135', 9, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1615,7 +1615,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... - output = ('203.162.223.135', 5, 1124013544.0) + output = ('203.162.223.135', 3, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1624,7 +1624,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... - output = ('203.162.223.135', 1, 1124013544.0) + output = ('203.162.223.135', 2, 1124013600.0) self.filter.setMaxRetry(1) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) @@ -1652,6 +1652,7 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresWrongChar(self): + self.filter.checkFindTime = False # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') @@ -1672,6 +1673,7 @@ class GetFailures(LogCaptureTestCase): for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') From d2cef96f33d13acad7d3dd1129de44c0f723efd6 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 18:52:00 +0200 Subject: [PATCH 089/136] filter: implement mode `inOperation`, which gets activated if filter starts processing of new messages; better interaction with non-matching optional datepattern or invalid timestamps (or timezone) - assuming now instead of bypass; fixed test cases gathering new failures now in operation mode --- fail2ban/server/filter.py | 118 ++++++++++++------ fail2ban/server/utils.py | 4 + fail2ban/tests/fail2banregextestcase.py | 6 +- fail2ban/tests/files/logs/zzz-generic-example | 8 +- fail2ban/tests/filtertestcase.py | 44 ++++--- 5 files changed, 118 insertions(+), 62 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 1ff81d65..40bfeef5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -112,6 +112,8 @@ class Filter(JailThread): self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## shows that filter is in operation mode (processing new messages): + self.inOperation = True ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): self.banASAP = True ## Ticks counter @@ -586,16 +588,26 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, False) return False + def _logWarnOnce(self, nextLTM, *args): + """Log some issue as warning once per day, otherwise level 7""" + if MyTime.time() < getattr(self, nextLTM, 0): + if logSys.getEffectiveLevel() <= 7: logSys.log(7, *(args[0])) + else: + setattr(self, nextLTM, MyTime.time() + 24*60*60) + for args in args: + logSys.warning('[%s] ' + args[0], self.jailName, *args[1:]) + def processLine(self, line, date=None): """Split the time portion from log msg and return findFailures on them """ + logSys.log(7, "Working on line %r", line) + + noDate = False if date: tupleLine = line self.__lastTimeText = tupleLine[1] self.__lastDate = date else: - logSys.log(7, "Working on line %r", line) - # try to parse date: timeMatch = self.dateDetector.matchTime(line) m = timeMatch[0] @@ -606,22 +618,51 @@ class Filter(JailThread): tupleLine = (line[:s], m, line[e:]) if m: # found and not empty - retrive date: date = self.dateDetector.getTime(m, timeMatch) - - if date is None: - if m: logSys.error("findFailure failed to parse timeText: %s", m) + if date is not None: + # Lets get the time part + date = date[0] + self.__lastTimeText = m + self.__lastDate = date + else: + logSys.error("findFailure failed to parse timeText: %s", m) + else: + tupleLine = ("", "", line) + # still no date - try to use last known: + if date is None: + noDate = True + if self.__lastDate and self.__lastDate > MyTime.time() - 60: + tupleLine = ("", self.__lastTimeText, line) date = self.__lastDate - else: - # Lets get the time part - date = date[0] - self.__lastTimeText = m + + if self.checkFindTime: + # if in operation (modifications have been really found): + if self.inOperation: + # if weird date - we'd simulate now for timeing issue (too large deviation from now): + if (date is None or date < MyTime.time() - 60 or date > MyTime.time() + 60): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_simByTimeWarn", + ("Simulate NOW in operation since found time has too large deviation %s ~ %s +/- %s", + date, MyTime.time(), 60), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # simulate now as date: + date = MyTime.time() self.__lastDate = date else: - tupleLine = (line, self.__lastTimeText, "") - date = self.__lastDate + # in initialization (restore) phase, if too old - ignore: + if date is not None and date < MyTime.time() - self.getFindTime(): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_ignByTimeWarn", + ("Ignore line since time %s < %s - %s", + date, MyTime.time(), self.getFindTime()), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # ignore - too old (obsolete) entry: + return [] # save last line (lazy convert of process line tuple to string on demand): self.processedLine = lambda: "".join(tupleLine[::2]) - return self.findFailure(tupleLine, date) + return self.findFailure(tupleLine, date, noDate=noDate) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager @@ -755,7 +796,7 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date): + def findFailure(self, tupleLine, date, noDate=False): failList = list() ll = logSys.getEffectiveLevel() @@ -765,11 +806,6 @@ class Filter(JailThread): returnRawHost = True cidr = IPAddr.CIDR_RAW - if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): - if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", - date, MyTime.time(), self.getFindTime()) - return failList - if self.__lineBufferSize > 1: self.__lineBuffer.append(tupleLine) orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:] @@ -820,17 +856,17 @@ class Filter(JailThread): if not self.checkAllRegex: break continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " + if noDate: + self._logWarnOnce("_next_noTimeWarn", + ("Found a match but no valid date/time found for %r.", tupleLine[1]), + ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), + ("Please try setting a custom " "date pattern (see man page jail.conf(5)). " "If format is complex, please " "file a detailed issue on" " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), tupleLine[1]) - continue + "in order to get support for this format.",)) + if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): if not self.checkAllRegex or self.__lineBufferSize > 1: self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None @@ -1023,7 +1059,7 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename): + def getFailures(self, filename, inOperation=None): log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) @@ -1068,9 +1104,14 @@ class FileFilter(Filter): if has_content: while not self.idle: line = log.readline() - if not line or not self.active: - # The jail reached the bottom or has been stopped + if not self.active: break; # jail has been stopped + if not line: + # The jail reached the bottom, simply set in operation for this log + # (since we are first time at end of file, growing is only possible after modifications): + log.inOperation = True break + # acquire in operation from log and process: + self.inOperation = inOperation if inOperation is not None else log.inOperation self.processLineAndAdd(line.rstrip('\r\n')) finally: log.close() @@ -1230,6 +1271,8 @@ class FileContainer: self.__pos = 0 finally: handler.close() + ## shows that log is in operation mode (expecting new messages only from here): + self.inOperation = tail def getFileName(self): return self.__filename @@ -1303,16 +1346,17 @@ class FileContainer: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: global _decode_line_warn - lev = logging.DEBUG - if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = 7 + if not _decode_line_warn.get(filename, 0): lev = logging.WARNING - _decode_line_warn[filename] = MyTime.time() + 24*60*60 + _decode_line_warn.set(filename, 1) logSys.log(lev, - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", - filename, enc, line) + "Error decoding line from '%s' with '%s'.", filename, enc) + if logSys.getEffectiveLevel() <= lev: + logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r", + line) # decode with replacing error chars: line = line.decode(enc, 'replace') return line @@ -1333,7 +1377,7 @@ class FileContainer: ## print "D: Closed %s with pos %d" % (handler, self.__pos) ## sys.stdout.flush() -_decode_line_warn = {} +_decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); ## diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 053aa04f..4e64ca0b 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -125,6 +125,10 @@ class Utils(): with self.__lock: self._cache.pop(k, None) + def clear(self): + with self.__lock: + self._cache.clear() + @staticmethod def setFBlockMode(fhandle, value): diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 2da0098e..2e1d18db 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -172,7 +172,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') self.assertLogged('Error decoding line'); self.assertLogged('Continuing to process line ignoring invalid characters') @@ -186,7 +186,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 19 matched, 0 missed') def testDirectRE_1raw_noDns(self): self.assertTrue(_test_exec( @@ -194,7 +194,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", "--usedns=no", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') # usage of \S+ causes raw handling automatically: self.pruneLog() self.assertTrue(_test_exec( diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index d0bd3322..118c7e12 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } +# -- wrong time direct in journal-line (used last known date or now, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } +# -- wrong time direct in journal-line (used last known date, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index e8634929..63f43b21 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -215,7 +215,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) - fout.write('\n'.join(lines)) + fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases # Opened earlier, therefore must close it @@ -878,7 +878,7 @@ class LogFileMonitor(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -897,7 +897,7 @@ class LogFileMonitor(LogCaptureTestCase): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.filter.getFailures(self.name) #self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -916,7 +916,7 @@ class LogFileMonitor(LogCaptureTestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1027,13 +1027,13 @@ def get_monitor_failures_testcase(Filter_): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=12) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and our dummy jail is empty as well self.assertFalse(len(self.jail)) # since it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) if idle: self.waitForTicks(1) self.assertTrue(self.isEmpty(1)) @@ -1052,7 +1052,7 @@ def get_monitor_failures_testcase(Filter_): #return # just for fun let's copy all of them again and see if that results # in a new ban - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.assert_correct_last_attempt(GetFailures.FAILURES_01) def test_rewrite_file(self): @@ -1066,7 +1066,7 @@ def get_monitor_failures_testcase(Filter_): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) def _wait4failures(self, count=2): @@ -1087,13 +1087,13 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1169,8 +1169,7 @@ def get_monitor_failures_testcase(Filter_): def _test_move_into_file(self, interim_kill=False): # if we move a new file into the location of an old (monitored) file - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() # make sure that it is monitored first self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1181,14 +1180,14 @@ def get_monitor_failures_testcase(Filter_): # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', - n=100).close() + skip=12, n=3).close() os.rename(self.name + '.new', self.name) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) # and to make sure that it now monitored for changes _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 9) @@ -1207,7 +1206,7 @@ def get_monitor_failures_testcase(Filter_): # create a bogus file in the same directory and see if that doesn't affect open(self.name + '.bak2', 'w').close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') @@ -1239,8 +1238,8 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above # now copy and get even more - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) - # check for 3 failures (not 9), because 6 already get above... + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) + # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: self.assertEqual(self.filter.failManager.getFailTotal(), 12) @@ -1606,13 +1605,21 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 9, 1124013600.0) + output = ('203.162.223.135', 6, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_InOperation(self): + output = ('203.162.223.135', 9, 1124013600.0) + + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) + _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... output = ('203.162.223.135', 3, 1124013600.0) @@ -1673,6 +1680,7 @@ class GetFailures(LogCaptureTestCase): for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + if DefLogSys.getEffectiveLevel() > 7: DefLogSys.setLevel(7); # ensure decode_line logs always self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: From b82f584a96fd3a19b157fc85c51a72c952809713 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 19:33:40 +0200 Subject: [PATCH 090/136] added test case covering new date handling (simulation, unknown format, warnings, etc) --- fail2ban/server/filter.py | 9 +++---- fail2ban/tests/filtertestcase.py | 44 +++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 40bfeef5..3ea0a601 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -651,6 +651,7 @@ class Filter(JailThread): else: # in initialization (restore) phase, if too old - ignore: if date is not None and date < MyTime.time() - self.getFindTime(): + print('**********') # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", ("Ignore line since time %s < %s - %s", @@ -860,12 +861,8 @@ class Filter(JailThread): self._logWarnOnce("_next_noTimeWarn", ("Found a match but no valid date/time found for %r.", tupleLine[1]), ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), - ("Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.",)) + ("Please try setting a custom date pattern (see man page jail.conf(5)).",) + ) if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): if not self.checkAllRegex or self.__lineBufferSize > 1: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 63f43b21..2dac91d1 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -394,12 +394,13 @@ class IgnoreIP(LogCaptureTestCase): finally: tearDownMyTime() - def testTimeJump(self): + def _testTimeJump(self, inOperation=False): try: self.filter.addFailRegex('^') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') self.filter.setFindTime(10); # max 10 seconds back self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = inOperation # self.pruneLog('[phase 1] DST time jump') # check local time jump (DST hole): @@ -430,6 +431,47 @@ class IgnoreIP(LogCaptureTestCase): self.assertNotLogged('Ignore line') finally: tearDownMyTime() + def testTimeJump(self): + self._testTimeJump(inOperation=False) + def testTimeJump_InOperation(self): + self._testTimeJump(inOperation=True) + + def testWrongTimeZone(self): + try: + self.filter.addFailRegex('fail from $') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = True; # real processing (all messages are new) + # current time is 1h later than log-entries: + MyTime.setTime(1572138000+3600) + # + self.pruneLog("[phase 1] simulate wrong TZ") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 + self.assertLogged( + "Simulate NOW in operation since found time has too large deviation", + "Please check jail has possibly a timezone issue.", + "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", + "Total # of detected failures: 3.", wait=True) + # + self.pruneLog("[phase 2] wrong TZ given in log") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 + self.assertLogged( + "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", + "Total # of detected failures: 6.", all=True, wait=True) + self.assertNotLogged("Found a match but no valid date/time found") + # + self.pruneLog("[phase 3] other timestamp (don't match datepattern), regex matches") + for i in range(3): + self.filter.processLineAndAdd('27.10.2019 04:00:00 fail from 192.0.2.17'); # +3 = 9 + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "192.0.2.17:1", "192.0.2.17:2", "192.0.2.17:3", + "Total # of detected failures: 9.", all=True, wait=True) + finally: + tearDownMyTime() def testAddAttempt(self): self.filter.setMaxRetry(3) From f21c58dc7249fca1c35b50d5cd6d31bbb0fdcd42 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 20 Aug 2020 20:27:39 +0200 Subject: [PATCH 091/136] implements special datepattern `{NONE}` - allow to find failures without date-time in log messages (filter use now as timestamp) closes gh-2802 --- fail2ban/server/datedetector.py | 2 ++ fail2ban/server/filter.py | 6 +++++- fail2ban/tests/fail2banregextestcase.py | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 0a6451be..90a70b0d 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -282,6 +282,8 @@ class DateDetector(object): elif "{DATE}" in key: self.addDefaultTemplate(preMatch=pattern, allDefaults=False) return + elif key == "{NONE}": + template = _getPatternTemplate('{UNB}^', key) else: template = _getPatternTemplate(pattern, key) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 3ea0a601..d10da7ab 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -625,6 +625,11 @@ class Filter(JailThread): self.__lastDate = date else: logSys.error("findFailure failed to parse timeText: %s", m) + else: + # matched empty value - date is optional or not available - set it to now: + date = MyTime.time() + self.__lastTimeText = "" + self.__lastDate = date else: tupleLine = ("", "", line) # still no date - try to use last known: @@ -651,7 +656,6 @@ class Filter(JailThread): else: # in initialization (restore) phase, if too old - ignore: if date is not None and date < MyTime.time() - self.getFindTime(): - print('**********') # log time zone issue as warning once per day: self._logWarnOnce("_next_ignByTimeWarn", ("Ignore line since time %s < %s - %s", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 2e1d18db..4d878a24 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -81,6 +81,7 @@ def _test_exec_command_line(*args): return _exit_code STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" +STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " RE_00_ID = r"Authentication failure for .*? from $" @@ -361,6 +362,24 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('192.0.2.0, kevin, inet4') self.pruneLog() + def testNoDateTime(self): + # datepattern doesn't match: + self.assertTrue(_test_exec('-d', '{^LN-BEG}EPOCH', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "Found-ID:kevin", all=True) + self.pruneLog() + # explicitly no datepattern: + self.assertTrue(_test_exec('-d', '{NONE}', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found-ID:kevin", all=True) + self.assertNotLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", all=True) + + self.pruneLog() + def testFrmtOutputWrapML(self): unittest.F2B.SkipIfCfgMissing(stock=True) # complex substitution using tags and message (ip, user, msg): From 76e5d2b1998265d33f7ae1b30bb72b02d0e1515f Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 21 Aug 2020 17:11:56 +0200 Subject: [PATCH 092/136] amend to f21c58dc7249fca1c35b50d5cd6d31bbb0fdcd42, better follow previous handling with last known datetime (compatibility for multi-line logs, in case of second line without a timestamp) --- fail2ban/server/filter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d10da7ab..4e947d27 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -625,11 +625,14 @@ class Filter(JailThread): self.__lastDate = date else: logSys.error("findFailure failed to parse timeText: %s", m) + # matched empty value - date is optional or not available - set it to last known or now: + elif self.__lastDate and self.__lastDate > MyTime.time() - 60: + # set it to last known: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate else: - # matched empty value - date is optional or not available - set it to now: + # set it to now: date = MyTime.time() - self.__lastTimeText = "" - self.__lastDate = date else: tupleLine = ("", "", line) # still no date - try to use last known: From 295630cccfb8409a4e590414d968487eb8596ccb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Aug 2020 16:12:55 +0200 Subject: [PATCH 093/136] documentation and changelog --- ChangeLog | 13 ++++++++++--- man/jail.conf.5 | 22 +++++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9b30bd6a..59624d42 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,12 +59,19 @@ ver. 0.10.6-dev (20??/??/??) - development edition - `aggressive`: matches 401 and any variant (with and without username) * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) -### New Features +### New Features and Enhancements * new filter and jail for GitLab recognizing failed application logins (gh-2689) - -### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; +* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) + as well as some warnings signaling user about invalid pattern or zone (gh-2814): + - filter gets mode in-operation, which gets activated if filter starts processing of new messages; + in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much + from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected + bypass of failure (previously exceeding `findtime`); + - better interaction with non-matching optional datepattern or invalid timestamps; + - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages, + whereas filter will use now as timestamp (gh-2802) * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); * fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 4d01b6a1..830c8aed 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -460,11 +460,27 @@ Similar to actions, filters have an [Init] section which can be overridden in \f specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .TP \fBdatepattern\fR -specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %Y-%m-%d %H:%M(?::%S)?. For a list of valid format directives, see Python library documentation for strptime behaviour. -.br -Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used. +specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %%Y-%%m-%%d %%H:%%M(?::%%S)?. +For a list of valid format directives, see Python library documentation for strptime behaviour. .br \fBNOTE:\fR due to config file string substitution, that %'s must be escaped by an % in config files. +.br +Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used as datepattern. +.br +Normally the regexp generated for datepattern additionally gets word-start and word-end boundaries to avoid accidental match inside of some word in a message. +There are several prefixes and words with special meaning that could be specified with custom datepattern to control resulting regex: +.RS +.IP +\fI{DEFAULT}\fR - can be used to add default date patterns of fail2ban. +.IP +\fI{DATE}\fR - can be used as part of regex that will be replaced with default date patterns. +.IP +\fI{^LN-BEG}\fR - prefix (similar to \fI^\fR) changing word-start boundary to line-start boundary (ignoring up to 2 characters). If used as value (not as a prefix), it will also set all default date patterns (similar to \fI{DEFAULT}\fR), but anchored at begin of message line. +.IP +\fI{UNB}\fR - prefix to disable automatic word boundaries in regex. +.IP +\fI{NONE}\fR - value would allow to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). +.RE .TP \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. From ad51fb7e1ea42c795135fe97970c31cfce51e446 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 24 Aug 2020 16:41:22 +0200 Subject: [PATCH 094/136] partial cherry-pick fd25c4cbb813db2562e5a2e54d4510eac291a897 (#2768) --- fail2ban/server/failmanager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index eee979fd..3458aed5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value From 1e3da21c680ac9a282f72348869e8cdb3d1e0399 Mon Sep 17 00:00:00 2001 From: TorontoMedia <10255876+TorontoMedia@users.noreply.github.com> Date: Sun, 28 Jun 2020 12:58:41 -0400 Subject: [PATCH 095/136] Remove duplicate method and rename invalid parameter (cherry picked from commit fd25c4cbb813db2562e5a2e54d4510eac291a897) --- fail2ban/server/failmanager.py | 4 ---- fail2ban/server/observer.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index bc09c0e2..97b4c7a4 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -59,10 +59,6 @@ class FailManager: with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c19549ba..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: From 1707560df8033341e4fb8a2e79db3ea684b42386 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Wed, 26 Feb 2020 10:41:55 +0100 Subject: [PATCH 096/136] Enhance Guacamole jail --- config/filter.d/guacamole.conf | 50 ++++++++++++++++++++++------- config/jail.conf | 1 + fail2ban/tests/files/logs/guacamole | 5 +++ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 09b4e7b0..bc6dbea9 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -5,21 +5,47 @@ [Definition] -# Option: failregex -# Notes.: regex to match the password failures messages in the logfile. -# Values: TEXT -# +logging = catalina +failregex = /failregex> +maxlines = /maxlines> +datepattern = /datepattern> + +[L_catalina] + failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" failed\.$ -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# -ignoreregex = - -# "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p ^WARNING:()** - {^LN-BEG} \ No newline at end of file + {^LN-BEG} + +[L_webapp] + +failregex = ^ \[\S+\] WARN \S+ - Authentication attempt from for user "[^"]+" failed. + +maxlines = 1 + +datepattern = ^%%H:%%M:%%S.%%f + +# DEV Notes: +# +# failregex is based on the default pattern given in Guacamole documentation : +# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging +# +# The following logback.xml Guacamole configuration file can then be used accordingly : +# +# +# /var/log/guacamole.log +# +# /var/log/guacamole.%d.log.gz +# 32 +# +# +# %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n +# +# +# +# +# +# diff --git a/config/jail.conf b/config/jail.conf index 39ebbe6d..40827707 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -440,6 +440,7 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole index 3de67454..ebb7afb0 100644 --- a/fail2ban/tests/files/logs/guacamole +++ b/fail2ban/tests/files/logs/guacamole @@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" } WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. + +# filterOptions: {"logging": "webapp"} + +# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" } +12:57:32.907 [http-nio-8080-exec-10] WARN o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed. From 7b05c1ce7a487f2c269815f8a17b7444213ae555 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 14:52:22 +0200 Subject: [PATCH 097/136] do type-convert only in getCombined (otherwise int/bool conversion prevents substitution or section-related interpolation of tags) --- fail2ban/client/actionreader.py | 11 ++---- fail2ban/client/configreader.py | 54 ++++++++++++++++++++------ fail2ban/tests/clientreadertestcase.py | 7 ++-- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 131e37cb..011a213d 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,17 +38,17 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], - "actionrepair_on_unban": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): @@ -83,11 +83,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 20709b72..f9a6d288 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -228,7 +228,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} @@ -244,12 +244,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue + if convert: + if opttype == "bool": + v = self.getboolean(sec, optname) + if v is None: continue + elif opttype == "int": + v = self.getint(sec, optname) + if v is None: continue + else: + v = self.get(sec, optname, vars=pOptions) else: v = self.get(sec, optname, vars=pOptions) values[optname] = v @@ -267,7 +270,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue @@ -324,8 +327,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -346,10 +350,34 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, pOptions={}): + options = self._configOpts + for optname in options: + if isinstance(options, (list,tuple)): + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + else: + opttype, optvalue = options[optname] + if optname in pOptions: + continue + try: + if opttype == "bool": + v = opts.get(optname) + if v is None or isinstance(v, bool): continue + v = _as_bool(v) + opts[optname] = v + elif opttype == "int": + v = opts.get(optname) + if v is None or isinstance(v, (int, long)): continue + v = int(v) + opts[optname] = v + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -384,6 +412,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 2c1d0a0e..ae39c157 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -490,7 +490,9 @@ class FilterReaderTest(unittest.TestCase): self.__share_cfg = {} def testConvert(self): - output = [['multi-set', 'testcase01', 'addfailregex', [ + output = [ + ['set', 'testcase01', 'maxlines', 1], + ['multi-set', 'testcase01', 'addfailregex', [ "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" @@ -512,7 +514,6 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addjournalmatch', "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], ['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"], - ['set', 'testcase01', 'maxlines', 1], # Last for overide test ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) @@ -529,7 +530,7 @@ class FilterReaderTest(unittest.TestCase): filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - output[-1][-1] = "5" + output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) def testFilterReaderSubstitionDefault(self): From d9b8796792ff2d876fe02e5a469767d33469c276 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:01:34 +0200 Subject: [PATCH 098/136] amend with better (common) handling, documentation and tests --- fail2ban/client/configreader.py | 93 +++++++++++++------------- fail2ban/tests/clientreadertestcase.py | 30 +++++++-- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index f9a6d288..d8817bed 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -233,29 +257,17 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if convert: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue - else: - v = self.get(sec, optname, vars=pOptions) - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -350,33 +362,20 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) - def convertOptions(self, opts, pOptions={}): - options = self._configOpts - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] - if optname in pOptions: - continue - try: - if opttype == "bool": - v = opts.get(optname) - if v is None or isinstance(v, bool): continue - v = _as_bool(v) - opts[optname] = v - elif opttype == "int": - v = opts.get(optname) - if v is None or isinstance(v, (int, long)): continue - v = int(v) - opts[optname] = v - except ValueError: - logSys.warning("Wrong %s value %r for %r. Using default one: %r", - opttype, v, optname, optvalue) - opts[optname] = optvalue + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init @@ -413,7 +412,7 @@ class DefinitionInitConfigReader(ConfigReader): if not opts: raise ValueError('recursive tag definitions unable to be resolved') # convert options after all interpolations: - self.convertOptions(opts) + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ae39c157..8abfd4a5 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -87,6 +87,21 @@ option = %s self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + def testConvert(self): + self.c.add_section("Definition") + self.c.set("Definition", "a", "1") + self.c.set("Definition", "b", "1") + self.c.set("Definition", "c", "test") + opts = self.c.getOptions("Definition", + (('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + opts = self.c.getOptions("Definition", + (('int', 'a'), ('bool', 'b'), ('int', 'c'))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None}) + opts = self.c.getOptions("Definition", + {'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)}) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + def testInaccessibleFile(self): f = os.path.join(self.d, "d.conf") # inaccessible file self._write('d.conf', 0) @@ -483,11 +498,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertRaises(NoSectionError, c.getOptions, 'test', {}) -class FilterReaderTest(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(FilterReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} +class FilterReaderTest(LogCaptureTestCase): def testConvert(self): output = [ @@ -533,6 +544,15 @@ class FilterReaderTest(unittest.TestCase): output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) + def testConvertOptions(self): + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '', 'test': 'X'}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined(); + self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines' + self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:") + def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {}, From 2945fe8cbd39b6e02ae8065878ec2dffc68f0021 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 25 Aug 2020 18:25:32 +0200 Subject: [PATCH 099/136] changelog --- ChangeLog | 3 +++ fail2ban/client/configreader.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59624d42..cacfdee7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* fixed type conversion in config readers (take place after all interpolations get ready), that allows to + specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters. * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing @@ -61,6 +63,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition ### New Features and Enhancements * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d8817bed..1b5a56a2 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -282,7 +282,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): values[optname] = optvalue # elif logSys.getEffectiveLevel() <= logLevel: # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) - except ValueError: + except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue From 5a2cc4e1c5219645ca801c8ba90bab5c609340ec Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:05:20 +0200 Subject: [PATCH 100/136] substituteRecursiveTags: more precise self- or cyclic-recursion prevention (don't clear replacement counts of tags, rather consider replacement count by tax X in tag Y) --- fail2ban/helpers.py | 14 ++++++++------ fail2ban/tests/actiontestcase.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 3ef7d543..dc7852ae 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -398,8 +398,8 @@ def splitWithOptions(option): # tags () in tagged options. # -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 +# max tag replacement count (considering tag X in tag Y repeat): +MAX_TAG_REPLACE_COUNT = 25 # compiled RE for tag name (replacement name) TAG_CRE = re.compile(r'<([^ <>]+)>') @@ -433,6 +433,7 @@ def substituteRecursiveTags(inptags, conditional='', done = set() noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) + repCounts = {} while True: repFlag = False # substitute each value: @@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='', value = orgval = uni_string(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) - refCounts = {} + rplc = repCounts.get(tag, {}) #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: # found replacement tag: @@ -454,13 +455,13 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue #logSys.log(5, 'found: %s' % rtag) - if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: + if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, rtag, refCounts, value)) + (tag, rtag, rplc, value)) repl = None if conditional: repl = tags.get(rtag + '?' + conditional) @@ -480,7 +481,7 @@ def substituteRecursiveTags(inptags, conditional='', value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[rtag] = refCounts.get(rtag, 0) + 1 + rplc[rtag] = rplc.get(rtag, 0) + 1 # the next match for replace: m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) @@ -488,6 +489,7 @@ def substituteRecursiveTags(inptags, conditional='', if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): if tre_search(value): + repCounts[tag] = rplc repFlag = True # copy return tags dict to prevent modifying of inptags: if id(tags) == id(inptags): diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 3425667c..125706af 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase): delattr(self.__action, 'ac') # produce self-referencing query except: self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", - lambda: self.__action.replaceTag(">>>>>>>>>>>>>>>>>>>>", + lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) From e569281d6b9880fdf3d8084dae295880f6700897 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 12:08:04 +0200 Subject: [PATCH 101/136] avoids overwrite of `known/option` with unmodified (not available) value of `option` from .local config file, so it wouldn't cause self-recursion if `option` already has a reference to `known/option` (from some include) in .conf file; closes gh-2751 --- fail2ban/client/configparserinc.py | 7 ++++--- fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/files/filter.d/testcase02.conf | 12 ++++++++++++ fail2ban/tests/files/filter.d/testcase02.local | 4 ++++ 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 fail2ban/tests/files/filter.d/testcase02.conf create mode 100644 fail2ban/tests/files/filter.d/testcase02.local diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index e0f39579..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -372,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 8abfd4a5..bb42b7a0 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -562,6 +562,17 @@ class FilterReaderTest(LogCaptureTestCase): c = filterReader.convert() self.assertSortedEqual(c, output) + def testFilterReaderSubstKnown(self): + # testcase02.conf + testcase02.local, test covering that known/option is not overridden + # with unmodified (not available) value of option from .local config file, so wouldn't + # cause self-recursion if option already has a reference to known/option in .conf file. + filterReader = FilterReader('testcase02', "jailname", {}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined() + self.assertTrue('sshd' in opts['failregex']) + def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, diff --git a/fail2ban/tests/files/filter.d/testcase02.conf b/fail2ban/tests/files/filter.d/testcase02.conf new file mode 100644 index 00000000..99b3bb45 --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.conf @@ -0,0 +1,12 @@ +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + +[Definition] + +_daemon = sshd +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? + +failregex = %(__prefix_line)s test \ No newline at end of file diff --git a/fail2ban/tests/files/filter.d/testcase02.local b/fail2ban/tests/files/filter.d/testcase02.local new file mode 100644 index 00000000..bfc81d4b --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.local @@ -0,0 +1,4 @@ +[Definition] + +# no options here, coverage for testFilterReaderSubstKnown: +# avoid to overwrite known/option with unmodified (not available) value of option from .local config file \ No newline at end of file From be3115cda0c5ab7def871256995974880221683d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 13:31:29 +0200 Subject: [PATCH 102/136] fix year overflow (9999) by format of datetime (time2str for end of ban of persistent ticket); closes gh-2817 --- fail2ban/server/mytime.py | 7 +++++-- fail2ban/tests/banmanagertestcase.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 98b69bd4..e4b091a7 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -121,8 +121,11 @@ class MyTime: @return ISO-capable string representation of given unixTime """ - return datetime.datetime.fromtimestamp( - unixTime).replace(microsecond=0).strftime(format) + # consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ) + dt = datetime.datetime.fromtimestamp( + unixTime).replace(microsecond=0 + ) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59) + return dt.strftime(format) ## precreate/precompile primitives used in str2seconds: diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a5b37ef6..3be31bc5 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -154,6 +154,21 @@ class AddFailure(unittest.TestCase): finally: self.__banManager.setBanTime(btime) + def testBanList(self): + tickets = [ + BanTicket('192.0.2.1', 1167605999.0), + BanTicket('192.0.2.2', 1167605999.0), + ] + tickets[1].setBanTime(-1) + for t in tickets: + self.__banManager.addBanTicket(t) + self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True), + [ + '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59', + '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59' + ] + ) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): From d0d1f8c362eb927a90abd6c86bab31a8a7717a79 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 26 Aug 2020 16:54:18 +0200 Subject: [PATCH 103/136] improve result for get/set prefregex --- fail2ban/server/transmitter.py | 6 ++++-- fail2ban/tests/servertestcase.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index aff9071c..10cfd163 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -277,7 +277,8 @@ class Transmitter: value = command[2] self.__server.setPrefRegex(name, value) if self.__quiet: return - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) @@ -446,7 +447,8 @@ class Transmitter: elif command[1] == "ignorecache": return self.__server.getIgnoreCache(name) elif command[1] == "prefregex": - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index f1b667b1..3b2552dd 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -513,6 +513,9 @@ class Transmitter(TransmitterBase): jail=self.jailName) self.setGetTest("ignorecache", '', None, jail=self.jailName) + def testJailPrefRegex(self): + self.setGetTest("prefregex", "^Test", jail=self.jailName) + def testJailRegex(self): self.jailAddDelRegexTest("failregex", [ From 07fa9f29129f53924f6850f01adeb8fb7c699060 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 17:04:19 +0200 Subject: [PATCH 104/136] fixes gh-2787: allow to match `did not issue MAIL/EXPN/VRFY/ETRN during connection` non-anchored with extra mode (default names may deviate); additionally provides common addr-tag for IPv4/IPv6 (`(?:IPv6:|)`) and test-coverage for IPv6 --- config/filter.d/sendmail-reject.conf | 13 +++++++------ fail2ban/tests/files/logs/sendmail-reject | 4 ++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index ca171915..e8b766c5 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,19 +21,20 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=(?:IPv6:|), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^rejecting commands from (\S* )?\[(?:IPv6:|)\] due to pre-greeting traffic after \d+ seconds$ - ^(?:\S+ )?\[(?:IPv6:|)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ - ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:|)\]$ + ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$ mdre-normal = -mdre-extra = ^(?:\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$ +mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection mdre-aggressive = %(mdre-extra)s diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index f69e4531..99c1877c 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -103,3 +103,7 @@ Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52 # failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } +Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 +# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file From 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 17:44:25 +0200 Subject: [PATCH 105/136] filter `sendmail-auth` extended to follow new authentication failure message introduced in sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757) --- config/filter.d/sendmail-auth.conf | 5 +++-- fail2ban/tests/files/logs/sendmail-auth | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 14995fed..c15a2e0c 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -9,10 +9,11 @@ before = common.conf _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) # "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:|)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - +failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index a7ddd6f8..93bf0b14 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -17,3 +17,8 @@ Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SM # failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" } +Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) +# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } +Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) From db1f3477cc415e5b89191b25f36406613012fc6d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 27 Aug 2020 18:07:42 +0200 Subject: [PATCH 106/136] amend to 3f04cba9f92a1827d0cb3dcb51e57d9f60900b4a: sendmail-auth has 2 failregex now, so rewritten with prefregex --- config/filter.d/sendmail-auth.conf | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index c15a2e0c..84fcbdda 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -8,12 +8,14 @@ before = common.conf [Definition] _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) +# "\w{14,20}" will give support for IDs from 14 up to 20 characters long __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? addr = (?:IPv6:|) -# "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - ^%(__prefix_line)sAUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ +prefregex = ^%(__prefix_line)s.+$ + +failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service From 17a6ba44b3fc9e60b6905ea2694a56f2c58c783b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Aug 2020 13:26:26 +0200 Subject: [PATCH 107/136] fail2ban-regex: speedup formatted output (bypass unneeded stats creation); fail2ban-regex: extended with prefregex statistic --- ChangeLog | 3 +++ fail2ban/client/fail2banregex.py | 43 ++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index cacfdee7..c7f9c759 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,6 +62,9 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) ### New Features and Enhancements +* fail2ban-regex: + - speedup formatted output (bypass unneeded stats creation) + - extended with prefregex statistic * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 98fd9799..1adf9761 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -252,6 +252,8 @@ class Fail2banRegex(object): self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -453,19 +455,33 @@ class Fail2banRegex(object): lines = [] ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False + return None, 0, None if self._filter.getMaxLines() > 1: for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -651,7 +667,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) From a3a148078e1875fd7b33daa1f07e0bb0c2000342 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 28 Aug 2020 14:12:57 +0200 Subject: [PATCH 108/136] fail2ban-regex: more informative output for `datepattern` (e. g. set from filter) - pattern : description, example: Use datepattern : ^%Y-%m-%d %H:%M:%S : ^Year-Month-Day 24hour:Minute:Second --- ChangeLog | 1 + fail2ban/client/fail2banregex.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c7f9c759..fabfed68 100644 --- a/ChangeLog +++ b/ChangeLog @@ -65,6 +65,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * fail2ban-regex: - speedup formatted output (bypass unneeded stats creation) - extended with prefregex statistic + - more informative output for `datepattern` (e. g. set from filter) - pattern : description * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 1adf9761..d0fe55dc 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -294,8 +294,8 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - self.output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: From f09ba1b3993c804aaa279b4542fa8ffd87c4dd53 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Sep 2020 19:59:27 +0200 Subject: [PATCH 109/136] action in jail-config extended to consider space as separator now (splitWithOptions separates by space between mains words, but not in options), so defining `action = a b` would specify 2 actions `a` and `b`; it is additionally more precise now (see fixed typo with closed bracket `]` instead of comma in testServerReloadTest) --- ChangeLog | 2 ++ fail2ban/helpers.py | 2 +- fail2ban/tests/clientreadertestcase.py | 11 +++++++++++ fail2ban/tests/fail2banclienttestcase.py | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index fabfed68..8e9b7612 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition - speedup formatted output (bypass unneeded stats creation) - extended with prefregex statistic - more informative output for `datepattern` (e. g. set from filter) - pattern : description +* parsing of action in jail-configs considers space between action-names as separator also + (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index dc7852ae..f381576e 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -373,7 +373,7 @@ OPTION_EXTRACT_CRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( - r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL) + r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) def extractOptions(option): match = OPTION_CRE.match(option) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index bb42b7a0..2cfaff77 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -264,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) + def testSplitWithOptions(self): + # covering all separators - new-line and spaces: + for sep in ('\n', '\t', ' '): + self.assertEqual(splitWithOptions('a%sb' % (sep,)), ['a', 'b']) + self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)), ['a[x=y]', 'b']) + self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b']) + self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)), ['a[x="y z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b']) + def testIncorrectJail(self): jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index bbd6964a..03b1d7ce 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -856,7 +856,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: '," " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, From ed20d457b21d298cf8a8569ab2eb09546e8065c4 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Sep 2020 20:14:31 +0200 Subject: [PATCH 110/136] jail.conf: removed action parameter `name` that set on jail-name (`name=%(__name__)s` is default in action reader) --- config/jail.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 40827707..8fbd23a1 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -171,16 +171,16 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. action_mw = %(action_)s - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(action_)s - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # @@ -192,7 +192,7 @@ action_xarf = %(action_)s # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # From a038fd5dfe8cb0714472833604735b83462a217d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 3 Sep 2020 16:41:23 +0200 Subject: [PATCH 111/136] `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-`; small optimizations on `firewallcmd-rich-rules.conf` and `firewallcmd-rich-logging.conf` simplifying both and provide a dependency (rich-logging is a derivative of rich-rules); closes gh-2821 --- ChangeLog | 1 + config/action.d/firewallcmd-ipset.conf | 2 +- config/action.d/firewallcmd-multiport.conf | 4 +- config/action.d/firewallcmd-new.conf | 4 +- config/action.d/firewallcmd-rich-logging.conf | 30 ++---------- config/action.d/firewallcmd-rich-rules.conf | 8 ++-- fail2ban/tests/servertestcase.py | 48 +++++++++++++++---- 7 files changed, 55 insertions(+), 42 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8e9b7612..361b81d5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -51,6 +51,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing with `jq`, gh-2140, gh-2656) * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 42513933..66358f23 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -69,7 +69,7 @@ allports = -p # Option: multiport # Notes.: addition to block access only to specific ports # Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -multiport = -p -m multiport --dports +multiport = -p -m multiport --dports "$(echo '' | sed s/:/-/g)" ipmset = f2b- familyopt = diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf index 81540e5b..0c401f1b 100644 --- a/config/action.d/firewallcmd-multiport.conf +++ b/config/action.d/firewallcmd-multiport.conf @@ -11,9 +11,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index b06f5ccd..7b08603c 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -10,9 +10,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index badfee83..21e45087 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -1,6 +1,6 @@ # Fail2Ban configuration file # -# Author: Donald Yandt +# Authors: Donald Yandt, Sergey G. Brester # # Because of the rich rule commands requires firewalld-0.3.1+ # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not @@ -10,36 +10,15 @@ # # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/.xml that can be shared and parsed easliy # -# Example commands to view rules: -# firewall-cmd [--zone=] --list-rich-rules -# firewall-cmd [--zone=] --list-all -# firewall-cmd [--zone=zone] --query-rich-rule='rule' +# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters. [INCLUDES] -before = firewallcmd-common.conf +before = firewallcmd-rich-rules.conf [Definition] -actionstart = - -actionstop = - -actioncheck = - -# you can also use zones and/or service names. -# -# zone example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' port port='' protocol='' log prefix='f2b-' level='' limit value='/m' " -# -# service name example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' service name='' log prefix='f2b-' level='' limit value='/m' " -# -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp - -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done - -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done +rich-suffix = log prefix='f2b-' level='' limit value='/m' [Init] @@ -48,4 +27,3 @@ level = info # log rate per minute rate = 1 - diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index bed71797..803d7d12 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -35,8 +35,10 @@ actioncheck = # # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done +fwcmd_rich_rule = rule family='' source address='' port port='$p' protocol='' %(rich-suffix)s + +actionban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done - +actionunban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done +rich-suffix = \ No newline at end of file diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 3b2552dd..eaf1b346 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1780,18 +1780,18 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`", ), @@ -1859,21 +1859,21 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", ), 'ip6-start': ( "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", ), 'flush': ( "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset flush f2b-j-w-fwcmd-ipset6`", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset destroy f2b-j-w-fwcmd-ipset`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset6`", "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), @@ -1926,6 +1926,38 @@ class ServerConfigReaderTests(LogCaptureTestCase): r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", ), }), + # firewallcmd-rich-rules -- + ('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + }), + # firewallcmd-rich-logging -- + ('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + }), ) server = TestServer() transm = server._Server__transm From f555ff45e99c804a3fa48a2db1af56fa9200da0a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 19:08:52 +0200 Subject: [PATCH 112/136] attempt to speedup ban- and fail-manager (e. g. fail2ban-client status, see gh-2819), remove unneeded lock (GIL is enough here) --- fail2ban/server/banmanager.py | 20 +++++++------------- fail2ban/server/failmanager.py | 9 +++------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 479ba26f..fe38dec6 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -66,8 +66,7 @@ class BanManager: # @param value the time def setBanTime(self, value): - with self.__lock: - self.__banTime = int(value) + self.__banTime = int(value) ## # Get the ban time. @@ -76,8 +75,7 @@ class BanManager: # @return the time def getBanTime(self): - with self.__lock: - return self.__banTime + return self.__banTime ## # Set the total number of banned address. @@ -85,8 +83,7 @@ class BanManager: # @param value total number def setBanTotal(self, value): - with self.__lock: - self.__banTotal = value + self.__banTotal = value ## # Get the total number of banned address. @@ -94,8 +91,7 @@ class BanManager: # @return the total number def getBanTotal(self): - with self.__lock: - return self.__banTotal + return self.__banTotal ## # Returns a copy of the IP list. @@ -103,8 +99,7 @@ class BanManager: # @return IP list def getBanList(self): - with self.__lock: - return list(self.__banList.keys()) + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,9 +107,8 @@ class BanManager: # @return ban list iterator def __iter__(self): - # ensure iterator is safe (traverse over the list in snapshot created within lock): - with self.__lock: - return iter(list(self.__banList.values())) + # ensure iterator is safe - traverse over the list in snapshot created within lock (GIL): + return iter(list(self.__banList.values())) ## # Returns normalized value diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3458aed5..3e81e8b5 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -47,12 +47,10 @@ class FailManager: self.__bgSvc = BgService() def setFailTotal(self, value): - with self.__lock: - self.__failTotal = value + self.__failTotal = value def getFailTotal(self): - with self.__lock: - return self.__failTotal + return self.__failTotal def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... @@ -123,8 +121,7 @@ class FailManager: return attempts def size(self): - with self.__lock: - return len(self.__failList) + return len(self.__failList) def cleanup(self, time): with self.__lock: From 5abc4ba4ae280b6ec89c6bffc9dd2140d0d56dc4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 7 Sep 2020 22:11:51 +0200 Subject: [PATCH 113/136] amend to 39d4bb3c35ffb3bc6cdead5ecb58b3377f87867c (#2758): better reaction on broken pipe (on long output), don't close stdout explicitly (allows usage of modules like cProfile, which outputs result on exit), just flush it before exit. --- fail2ban/client/fail2bancmdline.py | 24 ++++++++++++++++-------- fail2ban/helpers.py | 5 +++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 8936e03f..03683cad 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,13 +27,17 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise # Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") @@ -310,12 +314,16 @@ class Fail2banCmdLine(): def _exit(code=0): # implicit flush without to produce broken pipe error (32): sys.stderr.close() - sys.stdout.close() - # exit: - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f381576e..c45be849 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -224,9 +224,10 @@ def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover sys.exit(0) try: - BrokenPipeError + BrokenPipeError = BrokenPipeError except NameError: # pragma: 3.x no cover - BrokenPipeError = IOError + BrokenPipeError = IOError + __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, From e8ee3ba544c7eefd7b41e34ec3a44550c1280fbb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:36:54 +0200 Subject: [PATCH 114/136] resolves a bottleneck within transmitting of large data between server and client: speedup search of communications end-marker and increase max buffer size (up to 32KB) --- fail2ban/client/csocket.py | 14 +++++++++----- fail2ban/tests/sockettestcase.py | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 8cd22a41..2d414e5c 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -152,7 +152,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(RuntimeError, r"socket connection broken", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler @@ -168,7 +168,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: TestMsgError() - #self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage) + #self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage) self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error') finally: RequestHandler.found_terminator = org_handler From f381b982465e3e9bf58f565cfea006fedc1989f1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2020 11:44:55 +0200 Subject: [PATCH 115/136] introduces new flavor `short` for `fail2ban-client status $jail short`: output total and current counts only, without banned IPs list in order to speedup it and to provide more clear output (gh-2819), flavor `basic` (still default) is unmodified for backwards compatibility; it can be changed later to `short`, so for full list of IPs in newer version one should better use: - `fail2ban-client status $jail basic` - `fail2ban-client get $jail banned` or `fail2ban-client banned` --- fail2ban/server/actions.py | 14 ++++++++++---- fail2ban/tests/actionstestcase.py | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3308d4b2..f14d8d7b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -661,13 +661,19 @@ class Actions(JailThread, Mapping): """Status of current and total ban counts and current banned IP list. """ # TODO: Allow this list to be printed as 'status' output - supported_flavors = ["basic", "cymru"] + supported_flavors = ["short", "basic", "cymru"] if flavor is None or flavor not in supported_flavors: logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) - ret = [("Currently banned", self.__banManager.size()), - ("Total banned", self.__banManager.getBanTotal()), - ("Banned IP list", self.__banManager.getBanList())] + if flavor != "short": + banned = self.__banManager.getBanList() + cnt = len(banned) + else: + cnt = self.__banManager.size() + ret = [("Currently banned", cnt), + ("Total banned", self.__banManager.getBanTotal())] + if flavor != "short": + ret += [("Banned IP list", banned)] if flavor == "cymru": cymru_info = self.__banManager.getBanListExtendedCymruInfo() ret += \ diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index d97d9921..532fe6ed 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase): self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop') self.assertEqual(self.__actions.status(),[("Currently banned", 0 ), ("Total banned", 0 ), ("Banned IP list", [] )]) + self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ), + ("Total banned", 0 )]) def testAddActionPython(self): self.__actions.add( From d977d81ef75f96d41e8aa994aa8efd81f13bf795 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 17 Sep 2020 12:39:08 +0200 Subject: [PATCH 116/136] action.d/abuseipdb.conf: removed broken link, simplified usage example, fixed typos --- config/action.d/abuseipdb.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index d36cb3a9..ed958c86 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -21,14 +21,13 @@ # # Example, for ssh bruteforce (in section [sshd] of `jail.local`): # action = %(known/action)s -# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] # -# See below for catagories. +# See below for categories. # -# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) -## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call. # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] # ID Title Description # 3 Fraud Orders @@ -101,5 +100,5 @@ actionunban = # Notes Your API key from abuseipdb.com # Values: STRING Default: None # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. -# You will need to set the catagory in the action call. +# You will need to set the category in the action call. abuseipdb_apikey = From f518d42c590f4ea3d8632937dc6ad7e875419b00 Mon Sep 17 00:00:00 2001 From: Nathan Henrie Date: Wed, 12 Aug 2020 14:11:33 -0600 Subject: [PATCH 117/136] Add a note about `journalflags` options to `systemd-journal` backend Also adds systemd backend configuration examples to jail.conf(5) Closes #2696 --- fail2ban/client/fail2banregex.py | 4 +++- man/jail.conf.5 | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index d0fe55dc..56da17dd 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -113,7 +113,9 @@ class _f2bOptParser(OptionParser): LOG: string a string representing a log line filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + "systemd-journal" search systemd journal. Optionally specify + `systemd-journal[journalflags=X]` to determine + which journals are used (systemd-python required) REGEX: string a string representing a 'failregex' diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 830c8aed..d7722124 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst uses a polling algorithm which does not require external libraries. .TP .B systemd -uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples: +.PP +.RS +.nf +backend = systemd[journalpath=/run/log/journal/machine-1] +backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"] +backend = systemd[journalflags=1] +.fi .SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: From 24093de32daa05af5f178639095ac7106d55b544 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 23 Sep 2020 19:35:17 +0200 Subject: [PATCH 118/136] small amend (simplifying formatted help and man) --- fail2ban/client/fail2banregex.py | 22 +++++++++++----------- man/fail2ban-regex.1 | 9 +++++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 56da17dd..e7a4e214 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,7 +21,6 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" @@ -109,21 +108,22 @@ class _f2bOptParser(OptionParser): def format_help(self, *args, **kwargs): """ Overwritten format helper with full ussage.""" self.usage = '' - return "Usage: " + usage() + __doc__ + """ + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal. Optionally specify - `systemd-journal[journalflags=X]` to determine - which journals are used (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n Report bugs to https://github.com/fail2ban/fail2ban/issues\n """ + __copyright__ + "\n" diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index bb89ef8c..3964126f 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -18,13 +18,18 @@ a string representing a log line filename path to a log file (\fI\,/var/log/auth.log\/\fP) .TP -"systemd\-journal" -search systemd journal (systemd\-python required) +systemd\-journal +search systemd journal (systemd\-python required), +optionally with backend parameters, see `man jail.conf` +for usage and examples (systemd\-journal[journalflags=1]). .SS "REGEX:" .TP string a string representing a 'failregex' .TP +filter +name of filter, optionally with options (sshd[mode=aggressive]) +.TP filename path to a filter file (filter.d/sshd.conf) .SS "IGNOREREGEX:" From 1418bcdf5bb01dd210f4d37b5c2de8ccd0c658e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 12:35:49 +0200 Subject: [PATCH 119/136] `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num`, exit code can't be larger than 255 (gh-2836) --- ChangeLog | 1 + config/action.d/bsd-ipfw.conf | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 361b81d5..c3e2c6d4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition with `jq`, gh-2140, gh-2656) * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) * `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) +* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 5116b0d8..7f04fe7c 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,10 @@ # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipfw show | fgrep -c -m 1 -s 'table()' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num from table\(
\) to me ; echo $num > "" ) +actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" + ) # Option: actionstop From 2817a8144c685d3ca18b7ca27064179ac318924e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 13:33:40 +0200 Subject: [PATCH 120/136] `action.d/bsd-ipfw.conf`: small amend (gh-2836) simplifying awk condition/code (position starts from `` and increases whilst used) --- config/action.d/bsd-ipfw.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 7f04fe7c..444192d3 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -15,7 +15,7 @@ # Values: CMD # actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( - num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 == b) { b = $1 + 1 } } END { print b }'); ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" ) From c8059bf9b35a167ad7bd4579f83495f9dda50c79 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 16:27:17 +0200 Subject: [PATCH 121/136] ban/unban: increase responsiveness of actions thread by (un)banning process, better waiting timeout considering pending tickets for unban (_nextUnbanTime) --- fail2ban/server/actions.py | 46 ++++++++++++++++++++--------------- fail2ban/server/banmanager.py | 16 +++++------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index f14d8d7b..b7b95b44 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -327,25 +327,33 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if self.idle: - logSys.debug("Actions: enter idle mode") - Utils.wait_for(lambda: not self.active or not self.idle, - lambda: False, self.sleeptime) - logSys.debug("Actions: leave idle mode") - continue - # wait for ban (stop if gets inactive): - bancnt = 0 - if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, self.sleeptime): - bancnt = self.__checkBan() - cnt += bancnt - # unban if nothing is banned not later than banned tickets >= banPrecedence - if not bancnt or cnt >= self.banPrecedence: - if self.active: - # let shrink the ban list faster - bancnt *= 2 - self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) - cnt = 0 - + try: + if self.idle: + logSys.debug("Actions: enter idle mode") + Utils.wait_for(lambda: not self.active or not self.idle, + lambda: False, self.sleeptime) + logSys.debug("Actions: leave idle mode") + continue + # wait for ban (stop if gets inactive, pending ban or unban): + bancnt = 0 + wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): + bancnt = self.__checkBan() + cnt += bancnt + # unban if nothing is banned not later than banned tickets >= banPrecedence + if not bancnt or cnt >= self.banPrecedence: + if self.active: + # let shrink the ban list faster + bancnt *= 2 + logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount) + self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) + cnt = 0 + except Exception as e: # pragma: no cover + logSys.error("[%s] unhandled error in actions thread: %s", + self._jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + self.__flushBan(stop=True) self.stopActions() return True diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index fe38dec6..575d648b 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -57,7 +57,7 @@ class BanManager: ## Total number of banned IP address self.__banTotal = 0 ## The time for next unban process (for performance and load reasons): - self.__nextUnbanTime = BanTicket.MAX_TIME + self._nextUnbanTime = BanTicket.MAX_TIME ## # Set the ban time. @@ -290,8 +290,8 @@ class BanManager: self.__banList[fid] = ticket self.__banTotal += 1 # correct next unban time: - if self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if self._nextUnbanTime > eob: + self._nextUnbanTime = eob return True ## @@ -322,12 +322,8 @@ class BanManager: def unBanList(self, time, maxCount=0x7fffffff): with self.__lock: - # Permanent banning - if self.__banTime < 0: - return list() - # Check next unban time: - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime if nextUnbanTime > time: return list() @@ -340,12 +336,12 @@ class BanManager: if time > eob: unBanList[fid] = ticket if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime break elif nextUnbanTime > eob: nextUnbanTime = eob - self.__nextUnbanTime = nextUnbanTime + self._nextUnbanTime = nextUnbanTime # Removes tickets. if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: From 02525d7b6f96c2841c24c4dbfcfe14e4a3d1ef63 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 8 Oct 2020 21:07:51 +0200 Subject: [PATCH 122/136] filter.d/sshd.conf: mode `ddos` (and `aggressive`) extended with new rule closing flood attack vector, matching: error: kex_exchange_identification: Connection closed by remote host (gh-2850) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 4c86dca0..e7942262 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -71,7 +71,7 @@ mdre-normal = mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from - ^kex_exchange_identification: client sent invalid protocol identifier + ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host) ^Bad protocol version identification '.*' from ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 9fff416a..5d23f96f 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -312,6 +312,8 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 # failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" +# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" } +Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] From a07e6fe1a2d1b8945d8a84abf3022061c9cc4528 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 30 Oct 2020 13:19:56 +0100 Subject: [PATCH 123/136] reduce default `maxmatches` from 50 to 5: avoid too large memory consumption if `maxretry` is large and many failures don't cause ban (but accumulated in fail-manager with all the matched lines); closes gh-2843 --- fail2ban/server/failmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3e81e8b5..4173a233 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -43,7 +43,7 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 - self.maxMatches = 50 + self.maxMatches = 5 self.__bgSvc = BgService() def setFailTotal(self, value): From 7cb6412f68afd2590bb86d3d9939e0d21ad00367 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 3 Nov 2020 13:15:42 +0100 Subject: [PATCH 124/136] 1st try of GH actions flow (CI only, no coverage atm) --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..154a75ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + python -m pip install --upgrade pip + pip install 2to3 + fi + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 7f0010be68d0ca778ae97698bdd893c859b282eb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 3 Nov 2020 15:51:49 +0100 Subject: [PATCH 125/136] attempt to install systemd-python module --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 154a75ae..7a1d31df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,10 +44,12 @@ jobs: - name: Install dependencies run: | + python -m pip install --upgrade pip if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then - python -m pip install --upgrade pip pip install 2to3 fi + pip install systemd-python || echo 'systemd not available' + pip install pyinotify || echo 'inotify not available' - name: Before scripts run: | From 55d6408b13c75100134f123cc283a801a384fd6c Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Nov 2020 15:31:11 +0100 Subject: [PATCH 126/136] tweaks to speedup test-cases (test-suite seems to be time stable now, so we could shorten sleeping intervals) --- fail2ban/tests/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 47e5b909..b54581f5 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -292,15 +292,15 @@ def initTests(opts): unittest.F2B.SkipIfFast = F2B_SkipIfFast else: # smaller inertance inside test-cases (litle speedup): - Utils.DEFAULT_SLEEP_TIME = 0.25 - Utils.DEFAULT_SLEEP_INTERVAL = 0.025 + Utils.DEFAULT_SLEEP_TIME = 0.025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.005 Utils.DEFAULT_SHORT_INTERVAL = 0.0005 # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover + if v > 0.25: # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) - _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + _org_sleep(v) time.sleep = _new_sleep # --no-network : if unittest.F2B.no_network: # pragma: no cover From e700ccc6671d64aa725905763c66d74522bbd6ca Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Nov 2020 16:51:49 +0100 Subject: [PATCH 127/136] filter apache-modsecurity: added coverage for different log-format (apache 2.4 and php-fpm, gh-2717) --- fail2ban/tests/files/logs/apache-modsecurity | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index dbb14863..7e2f8c86 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -6,3 +6,6 @@ # failJSON: { "time": "2018-09-28T09:18:06", "match": true , "host": "192.0.2.1", "desc": "two client entries in message (gh-2247)" } [Sat Sep 28 09:18:06 2018] [error] [client 192.0.2.1:55555] [client 192.0.2.1] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] + +# failJSON: { "time": "2020-05-09T00:35:52", "match": true , "host": "192.0.2.2", "desc": "new format - apache 2.4 and php-fpm (gh-2717)" } +[Sat May 09 00:35:52.389262 2020] [:error] [pid 22406:tid 139985298601728] [client 192.0.2.2:47762] [client 192.0.2.2] ModSecurity: Access denied with code 401 (phase 2). Operator EQ matched 1 at IP:blocked. [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_wp_login.conf"] [line "14"] [id "500000"] [msg "Ip address blocked for 15 minutes, more than 5 login attempts in 3 minutes."] [hostname "example.com"] [uri "/wp-login.php"] [unique_id "XrYlGL5IY3I@EoLOgAAAA8"], referer: https://example.com/wp-login.php From ec873e2dc38af51d7da41d65fb28a1daf707b557 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Thu, 5 Nov 2020 23:56:30 +0100 Subject: [PATCH 128/136] Add SoftEtherVPN jail --- ChangeLog | 1 + config/filter.d/softethervpn.conf | 9 +++++++++ config/jail.conf | 5 +++++ fail2ban/tests/files/logs/softethervpn | 7 +++++++ 4 files changed, 22 insertions(+) create mode 100644 config/filter.d/softethervpn.conf create mode 100644 fail2ban/tests/files/logs/softethervpn diff --git a/ChangeLog b/ChangeLog index c3e2c6d4..d1aa66c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * parsing of action in jail-configs considers space between action-names as separator also (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf new file mode 100644 index 00000000..0cbf5c94 --- /dev/null +++ b/config/filter.d/softethervpn.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for SoftEtherVPN +# Detecting unauthorized access to SoftEtherVPN +# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration + +[INCLUDES] +before = common.conf + +[Definition] +failregex = ^%(__prefix_line)s(?:\([0-9 :.-]{23}\) :)? Connection ".+": User authentication failed. The user name that has been provided was ".+", from .$ diff --git a/config/jail.conf b/config/jail.conf index 8fbd23a1..67f39e40 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -820,6 +820,11 @@ udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010 action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + [gitlab] port = http,https logpath = /var/log/gitlab/gitlab-rails/application.log diff --git a/fail2ban/tests/files/logs/softethervpn b/fail2ban/tests/files/logs/softethervpn new file mode 100644 index 00000000..dd2a798b --- /dev/null +++ b/fail2ban/tests/files/logs/softethervpn @@ -0,0 +1,7 @@ +# Access of unauthorized host in /usr/local/vpnserver/security_log/*/sec.log +# failJSON: { "time": "2020-05-12T10:53:19", "match": true , "host": "80.10.11.12" } +2020-05-12 10:53:19.781 Connection "CID-72": User authentication failed. The user name that has been provided was "bob", from 80.10.11.12. + +# Access of unauthorized host in syslog +# failJSON: { "time": "2020-05-13T10:53:19", "match": true , "host": "80.10.11.13" } +2020-05-13T10:53:19 localhost [myserver.com/VPN/defaultvpn] (2020-05-13 10:53:19.591) : Connection "CID-594": User authentication failed. The user name that has been provided was "alice", from 80.10.11.13. From 5430091acb5ba0ec7cc9cbb50a4346ec9e5c896a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2020 12:43:34 +0100 Subject: [PATCH 129/136] jail `counter-strike`: removed link to site with redirect to malicious page (gh-2868) --- config/jail.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index fb76926d..b0a67720 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -789,7 +789,6 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] From 010e76406fd2aac83b8cf6da27e9d380cc75dad4 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 9 Nov 2020 13:19:25 +0100 Subject: [PATCH 130/136] small tweaks (both 2nd time and facility are optional, avoid catch-all, etc) --- config/filter.d/softethervpn.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf index 0cbf5c94..f7e7c0c3 100644 --- a/config/filter.d/softethervpn.conf +++ b/config/filter.d/softethervpn.conf @@ -6,4 +6,4 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(?:\([0-9 :.-]{23}\) :)? Connection ".+": User authentication failed. The user name that has been provided was ".+", from .$ +failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "(?:[^"]+|.+)", from \.$ From df659a0cbc68ad7f8233f16edf64ddddec6dd1d7 Mon Sep 17 00:00:00 2001 From: Mart124 <37041094+Mart124@users.noreply.github.com> Date: Sun, 18 Oct 2020 19:56:30 +0200 Subject: [PATCH 131/136] Add Bitwarden syslog support --- ChangeLog | 1 + config/filter.d/bitwarden.conf | 8 +++++++- fail2ban/tests/files/logs/bitwarden | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d1aa66c5..96c58bb5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -73,6 +73,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * new filter and jail for GitLab recognizing failed application logins (gh-2689) * new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) +* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf index 29bd4be8..4a836cbb 100644 --- a/config/filter.d/bitwarden.conf +++ b/config/filter.d/bitwarden.conf @@ -2,5 +2,11 @@ # Detecting failed login attempts # Logged in bwdata/logs/identity/Identity/log.txt +[INCLUDES] +before = common.conf + [Definition] -failregex = ^\s*\[WRN\]\s+Failed login attempt(?:, 2FA invalid)?\. $ +failregex = ^%(__prefix_line)s\s*\[[^\s]+\]\s+Failed login attempt(?:, 2FA invalid)?\. $ + +# DEV Notes: +# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 3642b3bf..9deb2c07 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -3,3 +3,6 @@ # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } 2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } +2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 From 2a18b82f5f92ca50b63dcb01b6f4231cd4220f9f Mon Sep 17 00:00:00 2001 From: Mart124 <37041094+Mart124@users.noreply.github.com> Date: Tue, 20 Oct 2020 18:18:03 +0200 Subject: [PATCH 132/136] Support alternative Bitwarden log format --- fail2ban/tests/files/logs/bitwarden | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 9deb2c07..27a22854 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -2,7 +2,7 @@ 2019-11-26 01:04:49.008 +08:00 [WRN] Failed login attempt. 192.168.0.16 # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } -2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 +2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 # failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } 2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 From 25e006e137172c96c25864f8050b191efaaba3d8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2020 13:43:59 +0100 Subject: [PATCH 133/136] review and small tweaks (more precise and safe RE) --- config/filter.d/bitwarden.conf | 3 ++- fail2ban/tests/files/logs/bitwarden | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf index 4a836cbb..b0651c8e 100644 --- a/config/filter.d/bitwarden.conf +++ b/config/filter.d/bitwarden.conf @@ -6,7 +6,8 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s\s*\[[^\s]+\]\s+Failed login attempt(?:, 2FA invalid)?\. $ +_daemon = Bitwarden-Identity +failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. $ # DEV Notes: # __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 27a22854..0fede6c6 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -1,6 +1,9 @@ # failJSON: { "time": "2019-11-25T18:04:49", "match": true , "host": "192.168.0.16" } 2019-11-26 01:04:49.008 +08:00 [WRN] Failed login attempt. 192.168.0.16 +# failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } +2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } 2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 From 840f0ff10a5edb14afaba8b2c13bc18d2514715d Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Mon, 9 Nov 2020 15:31:06 +0100 Subject: [PATCH 134/136] Add Grafana jail --- ChangeLog | 1 + config/filter.d/grafana.conf | 9 +++++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/grafana | 5 +++++ 4 files changed, 19 insertions(+) create mode 100644 config/filter.d/grafana.conf create mode 100644 fail2ban/tests/files/logs/grafana diff --git a/ChangeLog b/ChangeLog index 96c58bb5..51ba4f90 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition * parsing of action in jail-configs considers space between action-names as separator also (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` * new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for Grafana recognizing failed application logins (gh-2855) * new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) * `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) * `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf new file mode 100644 index 00000000..78ded075 --- /dev/null +++ b/config/filter.d/grafana.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for Grafana +# Detecting unauthorized access +# Typically logged in /var/log/grafana/grafana.log + +[Init] +datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z + +[Definition] +failregex = ^.*msg="Invalid username or password".* remote_addr=$ diff --git a/config/jail.conf b/config/jail.conf index 5ca67749..ddbcf61e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -828,6 +828,10 @@ logpath = /usr/local/vpnserver/security_log/*/sec.log port = http,https logpath = /var/log/gitlab/gitlab-rails/application.log +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + [bitwarden] port = http,https logpath = /home/*/bwdata/logs/identity/Identity/log.txt diff --git a/fail2ban/tests/files/logs/grafana b/fail2ban/tests/files/logs/grafana new file mode 100644 index 00000000..aac86ebc --- /dev/null +++ b/fail2ban/tests/files/logs/grafana @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/grafana/grafana.log +# failJSON: { "time": "2020-10-19T17:44:33", "match": true , "host": "182.56.23.12" } +t=2020-10-19T17:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="Invalid Username or Password" remote_addr=182.56.23.12 +# failJSON: { "time": "2020-10-19T18:44:33", "match": true , "host": "182.56.23.13" } +t=2020-10-19T18:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="User not found" remote_addr=182.56.23.13 From 1c1a9b868c6c7677c929ede5e8c1cb8f16d3be41 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 9 Nov 2020 15:36:30 +0100 Subject: [PATCH 135/136] no catch-alls, user name and error message stored in ticket --- config/filter.d/grafana.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf index 78ded075..e7f0f420 100644 --- a/config/filter.d/grafana.conf +++ b/config/filter.d/grafana.conf @@ -6,4 +6,4 @@ datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z [Definition] -failregex = ^.*msg="Invalid username or password".* remote_addr=$ +failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"[^"]+"|\S+)| error="[^"]+"| \S+=(?:\S*|"[^"]+"))* remote_addr=$ From ca4da9d1d3f792562500182711abc149b2c741a1 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 11 Nov 2020 11:08:23 +0100 Subject: [PATCH 136/136] actions: extend tags replacement in non ticket-based commands (actionstart, actionstop, etc); fixes regression by interpolation of tag `` introduced in 0.11 with dynamic bantime (due to `bantime.increment`, see #2869) --- fail2ban/server/action.py | 13 ++++++++++++- fail2ban/server/actions.py | 12 +++++++----- fail2ban/tests/servertestcase.py | 3 ++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 1c313cf0..f52e0878 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -455,7 +455,18 @@ class CommandAction(ActionBase): ret = True # avoid double execution of same command for both families: if cmd and cmd not in self._operationExecuted(tag, lambda f: f != famoper): - ret = self.executeCmd(cmd, self.timeout) + realCmd = cmd + if self._jail: + # simulate action info with "empty" ticket: + aInfo = getattr(self._jail.actions, 'actionInfo', None) + if not aInfo: + aInfo = self._jail.actions._getActionInfo(None) + setattr(self._jail.actions, 'actionInfo', aInfo) + aInfo['time'] = MyTime.time() + aInfo['family'] = famoper + # replace dynamical tags, important - don't cache, no recursion and auto-escape here + realCmd = self.replaceDynamicTags(cmd, aInfo) + ret = self.executeCmd(realCmd, self.timeout) res &= ret if afterExec: afterExec(famoper, ret) self._operationExecuted(tag, famoper, cmd if ret else None) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 49db5df7..967908af 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -457,7 +457,9 @@ class Actions(JailThread, Mapping): return mi[idx] if mi[idx] is not None else self.__ticket - def __getActionInfo(self, ticket): + def _getActionInfo(self, ticket): + if not ticket: + ticket = BanTicket("", MyTime.time()) aInfo = Actions.ActionInfo(ticket, self._jail) return aInfo @@ -491,7 +493,7 @@ class Actions(JailThread, Mapping): bTicket = BanTicket.wrap(ticket) btime = ticket.getBanTime(self.__banManager.getBanTime()) ip = bTicket.getIP() - aInfo = self.__getActionInfo(bTicket) + aInfo = self._getActionInfo(bTicket) reason = {} if self.__banManager.addBanTicket(bTicket, reason=reason): cnt += 1 @@ -568,7 +570,7 @@ class Actions(JailThread, Mapping): """ actions = actions or self._actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Reban %s%s", self._jail.name, aInfo["ip"], (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) for name, action in actions.iteritems(): @@ -602,7 +604,7 @@ class Actions(JailThread, Mapping): if not action._prolongable: continue if aInfo is None: - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if not aInfo.immutable: aInfo.reset() action.prolong(aInfo) except Exception as e: @@ -696,7 +698,7 @@ class Actions(JailThread, Mapping): else: unbactions = actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) for name, action in unbactions.iteritems(): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 3bd4501b..d2bf8bdc 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1448,9 +1448,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # dummy -- - ('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', { + ('j-dummy', '''dummy[name=%(__name__)s, init="=='/'==bt:==bc:==", target="/tmp/fail2ban.dummy"]''', { 'ip4': ('family: inet4',), 'ip6': ('family: inet6',), 'start': ( + '''`printf %b "=='/'==bt:600==bc:0==\\n"''', ## empty family (independent in this action, same for both), no ip on start, initial bantime and bancount '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`', ), 'flush': (