Merge pull request #2176 from sebres/ignore-cache

Introduces cache for ignore-facilities (for `ignoreip`, `ignoreself` and `ignorecommand`)

ChangeLog

@@ -50,6 +50,10 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition
 * systemd: fixed type error on option `journalflags`: an integer is required (gh-2125);
 
 ### New Features
+* new option `ignorecache` to improve performance of ignore failure check (using caching of `ignoreip`, 
+  `ignoreself` and `ignorecommand`), see `man jail.conf` for syntax-example;
+* `ignorecommand` extended to use actions-similar replacement (capable to interpolate 
+  all possible tags like `<ip-host>`, `<family>`, `<fid>`, `F-USER` etc.)
 
 ### Enhancements
 * `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168)

fail2ban/client/jailreader.py

@@ -100,6 +100,7 @@ class JailReader(ConfigReader):
 				["string", "ignorecommand", None],
 				["bool",   "ignoreself", None],
 				["string", "ignoreip", None],
+				["string", "ignorecache", None],
 				["string", "filter", ""],
 				["string", "datepattern", None],
 				["string", "logtimezone", None],

fail2ban/protocol.py

@@ -84,6 +84,8 @@ protocol = [
 ["set <JAIL> ignoreself true|false", "allows the ignoring of own IP addresses"], 
 ["set <JAIL> addignoreip <IP>", "adds <IP> to the ignore list of <JAIL>"], 
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
+["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
+["set <JAIL> ignorecache <VALUE>", "sets ignorecache of <JAIL>"],
 ["set <JAIL> addlogpath <FILE> ['tail']", "adds <FILE> to the monitoring list of <JAIL>, optionally starting at the 'tail' of the file (default 'head')."], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
 ["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
@@ -91,7 +93,6 @@ protocol = [
 ["set <JAIL> deljournalmatch <MATCH>", "removes <MATCH> from the journal filter of <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
-["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
 ["set <JAIL> delignoreregex <INDEX>", "removes the regular expression at <INDEX> for ignoreregex"], 
 ["set <JAIL> findtime <TIME>", "sets the number of seconds <TIME> for which the filter will look back for <JAIL>"], 

fail2ban/server/filter.py

@@ -30,6 +30,7 @@ import re
 import sys
 import time
 
+from .actions import Actions
 from .failmanager import FailManagerEmpty, FailManager
 from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
@@ -80,6 +81,10 @@ class Filter(JailThread):
 		self.__ignoreSelf = True
 		## The ignore IP list.
 		self.__ignoreIpList = []
+		## External command
+		self.__ignoreCommand = False
+		## Cache for ignoreip:
+		self.__ignoreCache = None
 		## Size of line buffer
 		self.__lineBufferSize = 1
 		## Line buffer
@@ -89,8 +94,6 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## if set, treat log lines without explicit time zone to be in this time zone
 		self.__logtimezone = None
-		## External command
-		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
 		self.__encoding = PREFER_ENC
 		## Cache temporary holds failures info (used by multi-line for wrapping e. g. conn-id to host):
@@ -396,19 +399,34 @@ class Filter(JailThread):
 		raise Exception("run() is abstract")
 
 	##
-	# Set external command, for ignoredips
+	# External command, for ignoredips
 	#
 
-	def setIgnoreCommand(self, command):
+	@property
+	def ignoreCommand(self):
+		return self.__ignoreCommand
+
+	@ignoreCommand.setter
+	def ignoreCommand(self, command):
 		self.__ignoreCommand = command
 
 	##
-	# Get external command, for ignoredips
+	# Cache parameters for ignoredips
 	#
 
-	def getIgnoreCommand(self):
+	@property
-		return self.__ignoreCommand
+	def ignoreCache(self):
-
+		return [self.__ignoreCache[0], self.__ignoreCache[1].maxCount, self.__ignoreCache[1].maxTime] \
+			if self.__ignoreCache else None
+
+	@ignoreCache.setter
+	def ignoreCache(self, command):
+		if command:
+			self.__ignoreCache = command['key'], Utils.Cache(
+				maxCount=int(command.get('max-count', 100)), maxTime=MyTime.str2seconds(command.get('max-time', 5*60))
+			)
+		else:
+			self.__ignoreCache = None
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
@@ -418,11 +436,12 @@ class Filter(JailThread):
 	def addBannedIP(self, ip):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
-		if self.inIgnoreIPList(ip, log_ignore=False):
-			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
 
 		unixTime = MyTime.time()
-		self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry())
+		ticket = FailTicket(ip, unixTime)
+		if self._inIgnoreIPList(ip, ticket, log_ignore=False):
+			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
+		self.failManager.addFailure(ticket, self.failManager.getMaxRetry())
 
 		# Perform the banning of the IP now.
 		try: # pragma: no branch - exception is the only way out
@@ -487,32 +506,61 @@ class Filter(JailThread):
 	#
 	# Check if the given IP address matches an IP address/DNS or a CIDR
 	# mask in the ignore list.
-	# @param ip IP address object
+	# @param ip IP address object or ticket
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=True):
-		if not isinstance(ip, IPAddr):
+		ticket = None
+		if isinstance(ip, FailTicket):
+			ticket = ip
+			ip = ticket.getIP()
+		elif not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
+		return self._inIgnoreIPList(ip, ticket, log_ignore)
+
+	def _inIgnoreIPList(self, ip, ticket, log_ignore=True):
+		aInfo = None
+		# cached ?
+		if self.__ignoreCache:
+			key, c = self.__ignoreCache
+			if ticket:
+				aInfo = Actions.ActionInfo(ticket, self.jail)
+				key = CommandAction.replaceDynamicTags(key, aInfo)
+			else:
+				aInfo = { 'ip': ip }
+				key = CommandAction.replaceTag(key, aInfo)
+			v = c.get(key)
+			if v is not None:
+				return v
 
 		# check own IPs should be ignored and 'ip' is self IP:
 		if self.__ignoreSelf and ip in DNSUtils.getSelfIPs():
 			self.logIgnoreIp(ip, log_ignore, ignore_source="ignoreself rule")
+			if self.__ignoreCache: c.set(key, True)
 			return True
 
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
+				if self.__ignoreCache: c.set(key, True)
 				return True
 
 		if self.__ignoreCommand:
-			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
+			if ticket:
+				if not aInfo: aInfo = Actions.ActionInfo(ticket, self.jail)
+				command = CommandAction.replaceDynamicTags(self.__ignoreCommand, aInfo)
+			else:
+				if not aInfo: aInfo = { 'ip': ip }
+				command = CommandAction.replaceTag(self.__ignoreCommand, aInfo)
 			logSys.debug('ignore command: %s', command)
 			ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1))
 			ret_ignore = ret and ret_ignore == 0
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
+			if self.__ignoreCache: c.set(key, ret_ignore)
 			return ret_ignore
 
+		if self.__ignoreCache: c.set(key, False)
 		return False
 
 	def processLine(self, line, date=None):
@@ -549,12 +597,12 @@ class Filter(JailThread):
 				fail = element[3]
 				logSys.debug("Processing line with time:%s and ip:%s", 
 						unixTime, ip)
-				if self.inIgnoreIPList(ip):
+				tick = FailTicket(ip, unixTime, data=fail)
+				if self._inIgnoreIPList(ip, tick):
 					continue
 				logSys.info(
 					"[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 				)
-				tick = FailTicket(ip, unixTime, data=fail)
 				self.failManager.addFailure(tick)
 			# reset (halve) error counter (successfully processed line):
 			if self._errors:

fail2ban/server/server.py

@@ -391,10 +391,17 @@ class Server:
 		return self.__jails[name].filter.getLogTimeZone()
 
 	def setIgnoreCommand(self, name, value):
-		self.__jails[name].filter.setIgnoreCommand(value)
+		self.__jails[name].filter.ignoreCommand = value
 
 	def getIgnoreCommand(self, name):
-		return self.__jails[name].filter.getIgnoreCommand()
+		return self.__jails[name].filter.ignoreCommand
+
+	def setIgnoreCache(self, name, value):
+		value, options = extractOptions("cache["+value+"]")
+		self.__jails[name].filter.ignoreCache = options
+
+	def getIgnoreCache(self, name):
+		return self.__jails[name].filter.ignoreCache
 
 	def setPrefRegex(self, name, value):
 		flt = self.__jails[name].filter

fail2ban/server/transmitter.py

@@ -200,6 +200,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setIgnoreCommand(name, value)
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			value = command[2]
+			self.__server.setIgnoreCache(name, value)
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "addlogpath":
 			value = command[2]
 			tail = False
@@ -358,6 +362,8 @@ class Transmitter:
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "ignorecommand":
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "prefregex":
 			return self.__server.getPrefRegex(name)
 		elif command[1] == "failregex":

fail2ban/tests/filtertestcase.py

@@ -38,7 +38,7 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer
+from ..server.filter import FailTicket, Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
 from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
@@ -401,7 +401,7 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')
 
 	def testIgnoreCommand(self):
-		self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>"))
+		self.filter.ignoreCommand = sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>")
 		self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
 		self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
 		self.assertLogged("returned successfully 0", "returned successfully 1", all=True)
@@ -409,6 +409,60 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList(""))
 		self.assertLogged("usage: ignorecommand IP", "returned 10", all=True)
 	
+	def testIgnoreCommandForTicket(self):
+		# by host of IP (2001:db8::1 and 2001:db8::ffff map to "test-host" and "test-other" in the test-suite):
+		self.filter.ignoreCommand = 'if [ "<ip-host>" = "test-host" ]; then exit 0; fi; exit 1'
+		self.pruneLog()
+		self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1")))
+		self.assertLogged("returned successfully 0")
+		self.pruneLog()
+		self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff")))
+		self.assertLogged("returned successfully 1")
+		# by user-name (ignore tester):
+		self.filter.ignoreCommand = 'if [ "<F-USER>" = "tester" ]; then exit 0; fi; exit 1'
+		self.pruneLog()
+		self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'})))
+		self.assertLogged("returned successfully 0")
+		self.pruneLog()
+		self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'})))
+		self.assertLogged("returned successfully 1", all=True)
+
+	def testIgnoreCache(self):
+		# like both test-cases above, just cached (so once per key)...
+		self.filter.ignoreCache = {"key":"<ip>"}
+		self.filter.ignoreCommand = 'if [ "<ip>" = "10.0.0.1" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
+			self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
+			if not i:
+				self.assertLogged("returned successfully 0", "returned successfully 1", all=True)
+			else:
+				self.assertNotLogged("returned successfully 0", "returned successfully 1", all=True)
+		# by host of IP:
+		self.filter.ignoreCache = {"key":"<ip-host>"}
+		self.filter.ignoreCommand = 'if [ "<ip-host>" = "test-host" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1")))
+			self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff")))
+			if not i:
+				self.assertLogged("returned successfully")
+			else:
+				self.assertNotLogged("returned successfully")
+		# by user-name:
+		self.filter.ignoreCache = {"key":"<F-USER>", "max-count":"10", "max-time":"1h"}
+		self.assertEqual(self.filter.ignoreCache, ["<F-USER>", 10, 60*60])
+		self.filter.ignoreCommand = 'if [ "<F-USER>" = "tester" ]; then exit 0; fi; exit 1'
+		for i in xrange(5):
+			self.pruneLog()
+			self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'})))
+			self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'})))
+			if not i:
+				self.assertLogged("returned successfully")
+			else:
+				self.assertNotLogged("returned successfully")
+
 	def testIgnoreCauseOK(self):
 		ip = "93.184.216.34"
 		for ignore_source in ["dns", "ip", "command"]:
@@ -472,7 +526,7 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd])))
 		self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"])))
 		## via command:
-		self.filter.setIgnoreCommand(cmd + " <ip>")
+		self.filter.ignoreCommand = cmd + " <ip>"
 		for ip in bot_ips:
 			self.assertTrue(self.filter.inIgnoreIPList(str(ip)), "test of googlebot ip %s failed" % ip)
 			self.assertLogged('-- returned successfully')
@@ -480,7 +534,7 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
 		self.assertLogged('Argument must be a single valid IP.')
 		self.pruneLog()
-		self.filter.setIgnoreCommand(cmd + " bad arguments <ip>")
+		self.filter.ignoreCommand = cmd + " bad arguments <ip>"
 		self.assertFalse(self.filter.inIgnoreIPList("192.0"))
 		self.assertLogged('Please provide a single IP as an argument.')
 

fail2ban/tests/servertestcase.py

@@ -463,7 +463,14 @@ class Transmitter(TransmitterBase):
 			(0, False))
 
 	def testJailIgnoreCommand(self):
-		self.setGetTest("ignorecommand", "bin ", jail=self.jailName)
+		self.setGetTest("ignorecommand", "bin/ignore-command <ip>", jail=self.jailName)
+
+	def testJailIgnoreCache(self):
+		self.setGetTest("ignorecache", 
+			'key="<ip>",max-time=1d,max-count=9999', 
+			["<ip>", 9999, 24*60*60],
+			jail=self.jailName)
+		self.setGetTest("ignorecache", '', None, jail=self.jailName)
 
 	def testJailRegex(self):
 		self.jailAddDelRegexTest("failregex",

fail2ban/tests/utils.py

@@ -316,6 +316,7 @@ def initTests(opts):
 		c.set('203.0.113.%s' % i, None)
 		c.set('2001:db8::%s' %i, 'test-host')
 	# some legal ips used in our test cases (prevent slow dns-resolving and failures if will be changed later):
+	c.set('2001:db8::ffff', 'test-other')
 	c.set('87.142.124.10', 'test-host')
 	if unittest.F2B.no_network: # pragma: no cover
 		# precache all wrong dns to ip's used in test cases:

man/jail.conf.5

@@ -233,7 +233,19 @@ list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option a
 command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit.
 .br
 IP will not be banned if command returns successfully (exit code 0).
-Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution. Currently only <ip> is supported however more will be added later.
+Like ACTION FILES, tags like <ip> are can be included in the ignorecommand value and will be substituted before execution.
+.TP
+.B ignorecache
+provide cache parameters (default disabled) for ignore failure check (caching of the result from `ignoreip`, `ignoreself` and `ignorecommand`), syntax:
+
+.RS
+.nf
+        ignorecache = key="<F-USER>@<ip-host>", max-count=100, max-time=5m
+        ignorecommand = if [ "<F-USER>" = "technical" ] && [ "<ip-host>" = "my-host.example.com" ]; then exit 0; fi;
+                        exit 1
+.fi
+This will cache the result of \fBignorecommand\fR (does not call it repeatedly) for 5 minutes (cache time) for maximal 100 entries (cache size), using values substituted like "user@host" as cache-keys.  Set option \fBignorecache\fR to empty value disables the cache.
+.RE
 .TP
 .B bantime
 effective ban duration (in seconds or time abbreviation format).