From 9b6d17d07e2f00dd287a7ccb5f30b95685d6f935 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Jul 2018 13:01:16 +0200 Subject: [PATCH 1/2] extend `ignorecommand` to use actions-similar replacement (ticket-based now, so capable to interpolate all possible tags) --- ChangeLog | 2 ++ fail2ban/server/filter.py | 28 ++++++++++++++++++++-------- fail2ban/tests/filtertestcase.py | 20 +++++++++++++++++++- fail2ban/tests/utils.py | 1 + 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index ac26ee02..782c557d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -50,6 +50,8 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition * systemd: fixed type error on option `journalflags`: an integer is required (gh-2125); ### New Features +* `ignorecommand` extended to use actions-similar replacement (capable to interpolate + all possible tags like ``, ``, ``, `F-USER` etc.) ### Enhancements * `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 2e4f896b..00c1dc54 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -30,6 +30,7 @@ import re import sys import time +from .actions import Actions from .failmanager import FailManagerEmpty, FailManager from .ipdns import DNSUtils, IPAddr from .ticket import FailTicket @@ -418,11 +419,12 @@ class Filter(JailThread): def addBannedIP(self, ip): if not isinstance(ip, IPAddr): ip = IPAddr(ip) - if self.inIgnoreIPList(ip, log_ignore=False): - logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip) unixTime = MyTime.time() - self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry()) + ticket = FailTicket(ip, unixTime) + if self._inIgnoreIPList(ip, ticket, log_ignore=False): + logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip) + self.failManager.addFailure(ticket, self.failManager.getMaxRetry()) # Perform the banning of the IP now. try: # pragma: no branch - exception is the only way out @@ -487,13 +489,19 @@ class Filter(JailThread): # # Check if the given IP address matches an IP address/DNS or a CIDR # mask in the ignore list. - # @param ip IP address object + # @param ip IP address object or ticket # @return True if IP address is in ignore list def inIgnoreIPList(self, ip, log_ignore=True): - if not isinstance(ip, IPAddr): + ticket = None + if isinstance(ip, FailTicket): + ticket = ip + ip = ticket.getIP() + elif not isinstance(ip, IPAddr): ip = IPAddr(ip) + return self._inIgnoreIPList(ip, ticket, log_ignore) + def _inIgnoreIPList(self, ip, ticket, log_ignore=True): # check own IPs should be ignored and 'ip' is self IP: if self.__ignoreSelf and ip in DNSUtils.getSelfIPs(): self.logIgnoreIp(ip, log_ignore, ignore_source="ignoreself rule") @@ -506,7 +514,11 @@ class Filter(JailThread): return True if self.__ignoreCommand: - command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } ) + if ticket: + aInfo = Actions.ActionInfo(ticket, self.jail) + command = CommandAction.replaceDynamicTags(self.__ignoreCommand, aInfo) + else: + command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip }) logSys.debug('ignore command: %s', command) ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1)) ret_ignore = ret and ret_ignore == 0 @@ -549,12 +561,12 @@ class Filter(JailThread): fail = element[3] logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) - if self.inIgnoreIPList(ip): + tick = FailTicket(ip, unixTime, data=fail) + if self._inIgnoreIPList(ip, tick): continue logSys.info( "[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - tick = FailTicket(ip, unixTime, data=fail) self.failManager.addFailure(tick) # reset (halve) error counter (successfully processed line): if self._errors: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2c1e42be..3ea61b26 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -38,7 +38,7 @@ except ImportError: from ..server.jail import Jail from ..server.filterpoll import FilterPoll -from ..server.filter import Filter, FileFilter, FileContainer +from ..server.filter import FailTicket, Filter, FileFilter, FileContainer from ..server.failmanager import FailManagerEmpty from ..server.ipdns import DNSUtils, IPAddr from ..server.mytime import MyTime @@ -408,6 +408,24 @@ class IgnoreIP(LogCaptureTestCase): self.pruneLog() self.assertFalse(self.filter.inIgnoreIPList("")) self.assertLogged("usage: ignorecommand IP", "returned 10", all=True) + + def testIgnoreCommandForTicket(self): + # by host of IP (2001:db8::1 and 2001:db8::ffff map to "test-host" and "test-other" in the test-suite): + self.filter.setIgnoreCommand('if [ "" = "test-host" ]; then exit 0; fi; exit 1') + self.pruneLog() + self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1"))) + self.assertLogged("returned successfully 0") + self.pruneLog() + self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff"))) + self.assertLogged("returned successfully 1") + # by user-name (ignore tester): + self.filter.setIgnoreCommand('if [ "" = "tester" ]; then exit 0; fi; exit 1') + self.pruneLog() + self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'}))) + self.assertLogged("returned successfully 0") + self.pruneLog() + self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'}))) + self.assertLogged("returned successfully 1", all=True) def testIgnoreCauseOK(self): ip = "93.184.216.34" diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 55617874..8bf42994 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -316,6 +316,7 @@ def initTests(opts): c.set('203.0.113.%s' % i, None) c.set('2001:db8::%s' %i, 'test-host') # some legal ips used in our test cases (prevent slow dns-resolving and failures if will be changed later): + c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover # precache all wrong dns to ip's used in test cases: From f8f01d5ab771d9eff0f8f6a391b28456f419bf05 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Jul 2018 14:58:39 +0200 Subject: [PATCH 2/2] introduced new option `ignorecache` to improve performance of ignore failure check (using caching of `ignoreip`, `ignoreself` and `ignorecommand`) --- ChangeLog | 2 ++ fail2ban/client/jailreader.py | 1 + fail2ban/protocol.py | 3 +- fail2ban/server/filter.py | 54 ++++++++++++++++++++++++++------ fail2ban/server/server.py | 11 +++++-- fail2ban/server/transmitter.py | 6 ++++ fail2ban/tests/filtertestcase.py | 46 ++++++++++++++++++++++++--- fail2ban/tests/servertestcase.py | 9 +++++- man/jail.conf.5 | 14 ++++++++- 9 files changed, 127 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 782c557d..f54b4a71 100644 --- a/ChangeLog +++ b/ChangeLog @@ -50,6 +50,8 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition * systemd: fixed type error on option `journalflags`: an integer is required (gh-2125); ### New Features +* new option `ignorecache` to improve performance of ignore failure check (using caching of `ignoreip`, + `ignoreself` and `ignorecommand`), see `man jail.conf` for syntax-example; * `ignorecommand` extended to use actions-similar replacement (capable to interpolate all possible tags like ``, ``, ``, `F-USER` etc.) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 72bf780d..b06ba72d 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -100,6 +100,7 @@ class JailReader(ConfigReader): ["string", "ignorecommand", None], ["bool", "ignoreself", None], ["string", "ignoreip", None], + ["string", "ignorecache", None], ["string", "filter", ""], ["string", "datepattern", None], ["string", "logtimezone", None], diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 3625ec01..b21ab848 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -84,6 +84,8 @@ protocol = [ ["set ignoreself true|false", "allows the ignoring of own IP addresses"], ["set addignoreip ", "adds to the ignore list of "], ["set delignoreip ", "removes from the ignore list of "], +["set ignorecommand ", "sets ignorecommand of "], +["set ignorecache ", "sets ignorecache of "], ["set addlogpath ['tail']", "adds to the monitoring list of , optionally starting at the 'tail' of the file (default 'head')."], ["set dellogpath ", "removes from the monitoring list of "], ["set logencoding ", "sets the of the log files for "], @@ -91,7 +93,6 @@ protocol = [ ["set deljournalmatch ", "removes from the journal filter of "], ["set addfailregex ", "adds the regular expression which must match failures for "], ["set delfailregex ", "removes the regular expression at for failregex"], -["set ignorecommand ", "sets ignorecommand of "], ["set addignoreregex ", "adds the regular expression which should match pattern to exclude for "], ["set delignoreregex ", "removes the regular expression at for ignoreregex"], ["set findtime