From 6f1e789f31edcc303944efe1b8bc683f0eeabf02 Mon Sep 17 00:00:00 2001 From: Dobrica Pavlinusic Date: Sat, 30 Jun 2018 16:16:03 +0200 Subject: [PATCH 1/8] dovecot: collect F-USER and variants We are prefering ruser= if availble because this are credentials presented to dovecot from remote client. --- config/filter.d/dovecot.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index cfb38ea4..daea2ff3 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -12,8 +12,8 @@ _daemon = (?:dovecot(?:-auth)?|auth) prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?.+$ -failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ - ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ +failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ + ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ ^pam\(\S+,(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$ ^[a-z\-]{3,15}\(\S*,(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$ > From 00a0e980417de8524d0fef4210e840ecaae5bbc0 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 5 Jul 2018 12:29:41 -0400 Subject: [PATCH 2/8] Load logpath only after findtime is configured When new log paths are configured, their start offset is immediately determined by a filter searching for (now - findTime). But, since findTime is configured *after* the log is loaded and searched, logs are only searched back by the default 10 minute findTime, regardless of user configuration of jail settings. So, findTime must be configured before logpath or else the default findtime is used, which ignores any findtime time defined by the user. This fixes new reads on startup for actual log files. The systemd filter always performed as expected due to being setup after the jail's findtime config submission. --- fail2ban/client/jailreader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 651bbc37..3ec68cbb 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -90,12 +90,12 @@ class JailReader(ConfigReader): opts1st = [["bool", "enabled", False], ["string", "filter", ""]] opts = [["bool", "enabled", False], - ["string", "logpath", None], - ["string", "logtimezone", None], - ["string", "logencoding", None], ["string", "backend", "auto"], ["int", "maxretry", None], ["string", "findtime", None], + ["string", "logpath", None], + ["string", "logtimezone", None], + ["string", "logencoding", None], ["string", "bantime", None], ["string", "usedns", None], # be sure usedns is before all regex(s) in stream ["string", "failregex", None], From 1eb93e2556d9b720885ae16179bbed5ee8659538 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 6 Jul 2018 17:04:10 +0200 Subject: [PATCH 3/8] filter.py: repair start-time of initial seek to time (regardless the position of `findtime` option in config); jailreader.py: additionally relocate the option `logpath` after all log-related data (backend, date-pattern, etc) that may be needed by the first usage (gh-2173). Thanks to Matt Stancliff (mattsta) --- fail2ban/client/jailreader.py | 6 +++--- fail2ban/server/filter.py | 30 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 3ec68cbb..72bf780d 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -93,9 +93,6 @@ class JailReader(ConfigReader): ["string", "backend", "auto"], ["int", "maxretry", None], ["string", "findtime", None], - ["string", "logpath", None], - ["string", "logtimezone", None], - ["string", "logencoding", None], ["string", "bantime", None], ["string", "usedns", None], # be sure usedns is before all regex(s) in stream ["string", "failregex", None], @@ -105,6 +102,9 @@ class JailReader(ConfigReader): ["string", "ignoreip", None], ["string", "filter", ""], ["string", "datepattern", None], + ["string", "logtimezone", None], + ["string", "logencoding", None], + ["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc) ["string", "action", ""]] # Before interpolation (substitution) add static options always available as default: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9ffc9177..2e4f896b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -887,9 +887,6 @@ class FileFilter(Filter): self.__logs[path] = log logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) if autoSeek: - # if default, seek to "current time" - "find time": - if isinstance(autoSeek, bool): - autoSeek = MyTime.time() - self.getFindTime() self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific @@ -999,18 +996,21 @@ class FileFilter(Filter): return False # seek to find time for first usage only (prevent performance decline with polling of big files) - if self.__autoSeek.get(filename): - startTime = self.__autoSeek[filename] - del self.__autoSeek[filename] - # prevent completely read of big files first time (after start of service), - # initial seek to start time using half-interval search algorithm: - try: - self.seekToTime(log, startTime) - except Exception as e: # pragma: no cover - logSys.error("Error during seek to start time in \"%s\"", filename) - raise - logSys.exception(e) - return False + if self.__autoSeek: + startTime = self.__autoSeek.pop(filename, None) + if startTime: + # if default, seek to "current time" - "find time": + if isinstance(startTime, bool): + startTime = MyTime.time() - self.getFindTime() + # prevent completely read of big files first time (after start of service), + # initial seek to start time using half-interval search algorithm: + try: + self.seekToTime(log, startTime) + except Exception as e: # pragma: no cover + logSys.error("Error during seek to start time in \"%s\"", filename) + raise + logSys.exception(e) + return False if has_content: while not self.idle: From df33322f9fb3fdf8d2fc42f635ad64f65a70f458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Su=C5=A1nik?= Date: Tue, 8 May 2018 14:07:57 +0200 Subject: [PATCH 4/8] Added test to prove bug when specifying journal flags to systemd backend --- fail2ban/tests/servertestcase.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 59dfef50..295b2d31 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -803,6 +803,28 @@ class Transmitter(TransmitterBase): result = self.transm.proceed( ["set", jailName, "deljournalmatch", value]) self.assertTrue(isinstance(result[1], ValueError)) + + def testJournalFlagsMatch(self): + if not filtersystemd: # pragma: no cover + raise unittest.SkipTest("systemd python interface not available") + self.assertTrue(True) + jailName = "TestJail3" + self.server.addJail(jailName, "systemd[journalflags=2]") + values = [ + "_SYSTEMD_UNIT=sshd.service", + "TEST_FIELD1=ABC", + "_HOSTNAME=example.com", + ] + for n, value in enumerate(values): + self.assertEqual( + self.transm.proceed( + ["set", jailName, "addjournalmatch", value]), + (0, [[val] for val in values[:n+1]])) + for n, value in enumerate(values): + self.assertEqual( + self.transm.proceed( + ["set", jailName, "deljournalmatch", value]), + (0, [[val] for val in values[n+1:]])) class TransmitterLogging(TransmitterBase): From 54a04b3a6af62ce93ff6bfe62bf55d3faf431135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Su=C5=A1nik?= Date: Tue, 8 May 2018 14:08:14 +0200 Subject: [PATCH 5/8] Fixed data type of journal flags from str to int --- fail2ban/server/filtersystemd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index f83e7264..f87fdb4e 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -87,7 +87,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover args['files'] = list(set(files)) try: - args['flags'] = kwargs.pop('journalflags') + args['flags'] = int(kwargs.pop('journalflags')) except KeyError: pass From 1e44b3f085cd7b4d8e372c308135fd3a107a6d0d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 6 Jul 2018 17:42:28 +0200 Subject: [PATCH 6/8] systemd no cover (currently unsupported by travis) --- fail2ban/tests/servertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 295b2d31..5d52fa09 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -733,7 +733,7 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) - def testJournalMatch(self): + def testJournalMatch(self): # pragma: systemd no cover if not filtersystemd: # pragma: no cover raise unittest.SkipTest("systemd python interface not available") jailName = "TestJail2" @@ -804,7 +804,7 @@ class Transmitter(TransmitterBase): ["set", jailName, "deljournalmatch", value]) self.assertTrue(isinstance(result[1], ValueError)) - def testJournalFlagsMatch(self): + def testJournalFlagsMatch(self): # pragma: systemd no cover if not filtersystemd: # pragma: no cover raise unittest.SkipTest("systemd python interface not available") self.assertTrue(True) From a719ba81e9f1bdb9efc955bb8075c7e5bb9d95cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=C6=B0=C6=A1ng?= Date: Sat, 28 Apr 2018 07:28:03 +0700 Subject: [PATCH 7/8] Fix cymru reference link --- fail2ban/server/banmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index a341a7af..141c398e 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -156,7 +156,7 @@ class BanManager: # get cymru info: try: for ip in banIPs: - # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # Reference: https://www.team-cymru.com/IP-ASN-mapping.html#dns question = ip.getPTR( "origin.asn.cymru.com" if ip.isIPv4 else "origin6.asn.cymru.com" From 11c1bf0149790f7a2099c63e682221ee93115903 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 6 Jul 2018 18:05:59 +0200 Subject: [PATCH 8/8] Update ChangeLog --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index fefa7d5c..ac26ee02 100644 --- a/ChangeLog +++ b/ChangeLog @@ -45,10 +45,14 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition - database: improve adapter/converter handlers working on invalid characters in sense of json and/or sqlite-database; additionally both are exception-safe now, so avoid possible locking of database (closes gh-2137); - logging in fail2ban is process-wide exception-safe now. +* repaired start-time of initial seek to time (as well as other log-parsing related data), + if parameter `logpath` specified before `findtime`, `backend`, `datepattern`, etc (gh-2173) +* systemd: fixed type error on option `journalflags`: an integer is required (gh-2125); ### New Features ### Enhancements +* `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168) * since v.0.10.4, fail2ban-client, fail2ban-server and fail2ban-regex will return version without logo info, additionally option `-V` can be used to get version in normalized machine-readable short format.