Merge pull request #491 from kwirk/ipmatches

ENH: Add <ipmatches> and <ipjailmatches> tags + sendmail implementations

MANIFEST

@@ -228,6 +228,7 @@ config/action.d/mynetwatchman.conf
 config/action.d/pf.conf
 config/action.d/sendmail.conf
 config/action.d/sendmail-buffered.conf
+config/action.d/sendmail-whois-ipmatches.conf
 config/action.d/sendmail-whois.conf
 config/action.d/sendmail-whois-lines.conf
 config/action.d/shorewall.conf

config/action.d/sendmail-common.conf

@@ -8,6 +8,56 @@
 
 after = sendmail-common.local
 
+[Definition]
+
+# Option:  actionstart
+# Notes.:  command executed once at the start of Fail2Ban.
+# Values:  CMD
+#
+actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
+              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
+              From: <sendername> <<sender>>
+              To: <dest>\n
+              Hi,\n
+              The jail <name> has been started successfully.\n
+              Regards,\n
+              Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+
+# Option:  actionstop
+# Notes.:  command executed once at the end of Fail2Ban
+# Values:  CMD
+#
+actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on `uname -n`
+             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
+             From: <sendername> <<sender>>
+             To: <dest>\n
+             Hi,\n
+             The jail <name> has been stopped.\n
+             Regards,\n
+             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+
+# Option:  actioncheck
+# Notes.:  command executed once before each actionban command
+# Values:  CMD
+#
+actioncheck = 
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = 
+
+# Option:  actionunban
+# Notes.:  command executed when unbanning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionunban = 
+
 [Init]
 
 # Recipient mail address

config/action.d/sendmail-whois-ipjailmatches.conf

@@ -0,0 +1,37 @@
+# Fail2Ban configuration file
+#
+# Author: Cyril Jaquier
+#
+#
+
+[INCLUDES]
+
+before = sendmail-common.conf
+
+[Definition]
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
+            From: <sendername> <<sender>>
+            To: <dest>\n
+            Hi,\n
+            The IP <ip> has just been banned by Fail2Ban after
+            <failures> attempts against <name>.\n\n
+            Here are more information about <ip>:\n
+            `/usr/bin/whois <ip>`\n\n
+            Matches for <name> with <ipjailfailures> failures IP:<ip>\n
+            <ipjailmatches>\n\n
+            Regards,\n
+            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+
+[Init]
+
+# Default name of the chain
+#
+name = default

config/action.d/sendmail-whois-ipmatches.conf

@@ -0,0 +1,37 @@
+# Fail2Ban configuration file
+#
+# Author: Cyril Jaquier
+#
+#
+
+[INCLUDES]
+
+before = sendmail-common.conf
+
+[Definition]
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
+            From: <sendername> <<sender>>
+            To: <dest>\n
+            Hi,\n
+            The IP <ip> has just been banned by Fail2Ban after
+            <failures> attempts against <name>.\n\n
+            Here are more information about <ip>:\n
+            `/usr/bin/whois <ip>`\n\n
+            Matches with <ipfailures> failures IP:<ip>\n
+            <ipmatches>\n\n
+            Regards,\n
+            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+
+[Init]
+
+# Default name of the chain
+#
+name = default

config/action.d/sendmail-whois-lines.conf

@@ -10,38 +10,6 @@ before = sendmail-common.conf
 
 [Definition]
 
-# Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
-# Values:  CMD
-#
-actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
-              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: <sendername> <<sender>>
-              To: <dest>\n
-              Hi,\n
-              The jail <name> has been started successfully.\n
-              Regards,\n
-              Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
-# Values:  CMD
-#
-actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on `uname -n`
-             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: <sendername> <<sender>>
-             To: <dest>\n
-             Hi,\n
-             The jail <name> has been stopped.\n
-             Regards,\n
-             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actioncheck
-# Notes.:  command executed once before each actionban command
-# Values:  CMD
-#
-actioncheck = 
-
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -62,14 +30,6 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Regards,\n
             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    See jail.conf(5) man page
-# Values:  CMD
-#
-actionunban = 
-
 [Init]
 
 # Default name of the chain

config/action.d/sendmail-whois-matches.conf

@@ -0,0 +1,37 @@
+# Fail2Ban configuration file
+#
+# Author: Cyril Jaquier
+#
+#
+
+[INCLUDES]
+
+before = sendmail-common.conf
+
+[Definition]
+
+# Option:  actionban
+# Notes.:  command executed when banning an IP. Take care that the
+#          command is executed with Fail2Ban user rights.
+# Tags:    See jail.conf(5) man page
+# Values:  CMD
+#
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
+            From: <sendername> <<sender>>
+            To: <dest>\n
+            Hi,\n
+            The IP <ip> has just been banned by Fail2Ban after
+            <failures> attempts against <name>.\n\n
+            Here are more information about <ip>:\n
+            `/usr/bin/whois <ip>`\n\n
+            Matches:\n
+            <matches>\n\n
+            Regards,\n
+            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+
+[Init]
+
+# Default name of the chain
+#
+name = default

config/action.d/sendmail-whois.conf

@@ -10,38 +10,6 @@ before = sendmail-common.conf
 
 [Definition]
 
-# Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
-# Values:  CMD
-#
-actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
-              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: <sendername> <<sender>>
-              To: <dest>\n
-              Hi,\n
-              The jail <name> has been started successfully.\n
-              Regards,\n
-              Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
-# Values:  CMD
-#
-actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on `uname -n`
-             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: <sendername> <<sender>>
-             To: <dest>\n
-             Hi,\n
-             The jail <name> has been stopped.\n
-             Regards,\n
-             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actioncheck
-# Notes.:  command executed once before each actionban command
-# Values:  CMD
-#
-actioncheck = 
-
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -60,14 +28,6 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Regards,\n
             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    See jail.conf(5) man page
-# Values:  CMD
-#
-actionunban = 
-
 [Init]
 
 # Default name of the chain

config/action.d/sendmail.conf

@@ -10,38 +10,6 @@ before = sendmail-common.conf
 
 [Definition]
 
-# Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
-# Values:  CMD
-#
-actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on `uname -n`
-              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: <sendername> <<sender>>
-              To: <dest>\n
-              Hi,\n
-              The jail <name> has been started successfully.\n
-              Regards,\n
-              Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
-# Values:  CMD
-#
-actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on `uname -n`
-             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: <sendername> <<sender>>
-             To: <dest>\n
-             Hi,\n
-             The jail <name> has been stopped.\n
-             Regards,\n
-             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
-
-# Option:  actioncheck
-# Notes.:  command executed once before each actionban command
-# Values:  CMD
-#
-actioncheck = 
-
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -58,14 +26,6 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Regards,\n
             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 
-# Option:  actionunban
-# Notes.:  command executed when unbanning an IP. Take care that the
-#          command is executed with Fail2Ban user rights.
-# Tags:    See jail.conf(5) man page
-# Values:  CMD
-#
-actionunban = 
-
 [Init]
 
 # Default name of the chain

fail2ban/server/action.py

@@ -295,7 +295,7 @@ class Action:
 
 	#@staticmethod
 	def escapeTag(tag):
-		for c in '\\#&;`|*?~<>^()[]{}$\n\'"':
+		for c in '\\#&;`|*?~<>^()[]{}$\'"':
 			if c in tag:
 				tag = tag.replace(c, '\\' + c)
 		return tag
@@ -314,12 +314,15 @@ class Action:
 		"""
 		string = query
 		for tag, value in aInfo.iteritems():
-			value = str(value)			  # assure string
+			if "<%s>" % tag in query:
-			if tag == 'matches':
+				if callable(value):
-				# That one needs to be escaped since its content is
+					value = value()
-				# out of our control
+				value = str(value)			  # assure string
-				value = Action.escapeTag(value)
+				if tag.endswith('matches'):
-			string = string.replace('<' + tag + '>', value)
+					# That one needs to be escaped since its content is
+					# out of our control
+					value = Action.escapeTag(value)
+				string = string.replace('<' + tag + '>', value)
 		# New line
 		string = string.replace("<br>", '\n')
 		return string

fail2ban/server/actions.py

@@ -183,7 +183,20 @@ class Actions(JailThread):
 			aInfo["ip"] = bTicket.getIP()
 			aInfo["failures"] = bTicket.getAttempt()
 			aInfo["time"] = bTicket.getTime()
-			aInfo["matches"] = "".join(bTicket.getMatches())
+			aInfo["matches"] = "\n".join(bTicket.getMatches())
+			if self.jail.getDatabase() is not None:
+				aInfo["ipmatches"] = lambda: "\n".join(
+					self.jail.getDatabase().getBansMerged(
+						ip=bTicket.getIP()).getMatches())
+				aInfo["ipjailmatches"] = lambda: "\n".join(
+					self.jail.getDatabase().getBansMerged(
+						ip=bTicket.getIP(), jail=self.jail).getMatches())
+				aInfo["ipfailures"] = lambda: "\n".join(
+					self.jail.getDatabase().getBansMerged(
+						ip=bTicket.getIP()).getAttempt())
+				aInfo["ipjailfailures"] = lambda: "\n".join(
+					self.jail.getDatabase().getBansMerged(
+						ip=bTicket.getIP(), jail=self.jail).getAttempt())
 			if self.__banManager.addBanTicket(bTicket):
 				logSys.warning("[%s] Ban %s" % (self.jail.getName(), aInfo["ip"]))
 				for action in self.__actions:

fail2ban/server/database.py

@@ -26,6 +26,7 @@ import sys
 import sqlite3
 import json
 import locale
+from functools import wraps
 
 from fail2ban.server.mytime import MyTime
 from fail2ban.server.ticket import FailTicket
@@ -46,13 +47,12 @@ else:
 	sqlite3.register_adapter(dict, json.dumps)
 	sqlite3.register_converter("JSON", json.loads)
 
-def commitandrollback():
+def commitandrollback(f):
-	def wrap(f):
+	@wraps(f)
-		def func(self, *args, **kw):
+	def wrapper(self, *args, **kwargs):
-			with self._db: # Auto commit and rollback on exception
+		with self._db: # Auto commit and rollback on exception
-				return f(self, self._db.cursor(), *args, **kw)
+			return f(self, self._db.cursor(), *args, **kwargs)
-		return func
+	return wrapper
-	return wrap
 
 class Fail2BanDb(object):
 	__version__ = 1
@@ -64,6 +64,8 @@ class Fail2BanDb(object):
 			self._dbFilename = filename
 			self._purgeAge = purgeAge
 
+			self._bansMergedCache = {}
+
 			logSys.info(
 				"Connected to fail2ban persistent database '%s'", filename)
 		except sqlite3.OperationalError, e:
@@ -97,7 +99,7 @@ class Fail2BanDb(object):
 	def setPurgeAge(self, value):
 		self._purgeAge = int(value)
 
-	@commitandrollback()
+	@commitandrollback
 	def createDb(self, cur):
 		# Version info
 		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
@@ -143,14 +145,14 @@ class Fail2BanDb(object):
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
-	@commitandrollback()
+	@commitandrollback
 	def updateDb(self, cur, version):
 		raise NotImplementedError(
 			"Only single version of database exists...how did you get here??")
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
-	@commitandrollback()
+	@commitandrollback
 	def addJail(self, cur, jail):
 		cur.execute(
 			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
@@ -159,23 +161,23 @@ class Fail2BanDb(object):
 	def delJail(self, jail):
 		return self.delJailName(jail.getName())
 
-	@commitandrollback()
+	@commitandrollback
 	def delJailName(self, cur, name):
 		# Will be deleted by purge as appropriate
 		cur.execute(
 			"UPDATE jails SET enabled=0 WHERE name=?", (name, ))
 
-	@commitandrollback()
+	@commitandrollback
 	def delAllJails(self, cur):
 		# Will be deleted by purge as appropriate
 		cur.execute("UPDATE jails SET enabled=0")
 
-	@commitandrollback()
+	@commitandrollback
 	def getJailNames(self, cur):
 		cur.execute("SELECT name FROM jails")
 		return set(row[0] for row in cur.fetchmany())
 
-	@commitandrollback()
+	@commitandrollback
 	def addLog(self, cur, jail, container):
 		lastLinePos = None
 		cur.execute(
@@ -196,7 +198,7 @@ class Fail2BanDb(object):
 				lastLinePos = None
 		return lastLinePos
 
-	@commitandrollback()
+	@commitandrollback
 	def getLogPaths(self, cur, jail=None):
 		query = "SELECT path FROM logs"
 		queryArgs = []
@@ -206,7 +208,7 @@ class Fail2BanDb(object):
 		cur.execute(query, queryArgs)
 		return set(row[0] for row in cur.fetchmany())
 
-	@commitandrollback()
+	@commitandrollback
 	def updateLog(self, cur, *args, **kwargs):
 		self._updateLog(cur, *args, **kwargs)
 
@@ -217,8 +219,9 @@ class Fail2BanDb(object):
 			(container.getHash(), container.getPos(),
 				jail.getName(), container.getFileName()))
 
-	@commitandrollback()
+	@commitandrollback
 	def addBan(self, cur, jail, ticket):
+		self._bansMergedCache = {}
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
@@ -226,7 +229,7 @@ class Fail2BanDb(object):
 				{"matches": ticket.getMatches(),
 					"failures": ticket.getAttempt()}))
 
-	@commitandrollback()
+	@commitandrollback
 	def _getBans(self, cur, jail=None, bantime=None, ip=None):
 		query = "SELECT ip, timeofban, data FROM bans WHERE 1"
 		queryArgs = []
@@ -244,27 +247,32 @@ class Fail2BanDb(object):
 
 		return cur.execute(query, queryArgs)
 
-	def getBans(self, *args, **kwargs):
+	def getBans(self, **kwargs):
 		tickets = []
-		for ip, timeofban, data in self._getBans(*args, **kwargs):
+		for ip, timeofban, data in self._getBans(**kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban, data['matches']))
 			tickets[-1].setAttempt(data['failures'])
 		return tickets
 
-	def getBansMerged(self, ip, *args, **kwargs):
+	def getBansMerged(self, ip, jail=None, **kwargs):
+		cacheKey = ip if jail is None else "%s|%s" % (ip, jail.getName())
+		if cacheKey in self._bansMergedCache:
+			return self._bansMergedCache[cacheKey]
 		matches = []
 		failures = 0
-		for ip, timeofban, data in self._getBans(*args, ip=ip, **kwargs):
+		for ip, timeofban, data in self._getBans(ip=ip, jail=jail, **kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
 			matches.extend(data['matches'])
 			failures += data['failures']
 		ticket = FailTicket(ip, timeofban, matches)
 		ticket.setAttempt(failures)
+		self._bansMergedCache[cacheKey] = ticket
 		return ticket
 
-	@commitandrollback()
+	@commitandrollback
 	def purge(self, cur):
+		self._bansMergedCache = {}
 		cur.execute(
 			"DELETE FROM bans WHERE timeofban < ?",
 			(MyTime.time() - self._purgeAge, ))

fail2ban/tests/actiontestcase.py

@@ -102,8 +102,28 @@ class ExecuteAction(unittest.TestCase):
 			"Text 890 text 123 ABC")
 		self.assertEqual(
 			self.__action.replaceTag("<matches>",
-				{'matches': "some >char< should \< be[ escap}ed&"}),
+				{'matches': "some >char< should \< be[ escap}ed&\n"}),
-			r"some \>char\< should \\\< be\[ escap\}ed\&")
+			"some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n")
+		self.assertEqual(
+			self.__action.replaceTag("<ipmatches>",
+				{'ipmatches': "some >char< should \< be[ escap}ed&\n"}),
+			"some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n")
+		self.assertEqual(
+			self.__action.replaceTag("<ipjailmatches>",
+				{'ipjailmatches': "some >char< should \< be[ escap}ed&\n"}),
+			"some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n")
+
+		# Callable
+		self.assertEqual(
+			self.__action.replaceTag("09 <callable> 11",
+				{'callable': lambda: str(10)}),
+			"09 10 11")
+
+		# As tag not present, therefore callable should not be called
+		# Will raise ValueError if it is
+		self.assertEqual(
+			self.__action.replaceTag("abc",
+				{'callable': lambda: int("a")}), "abc")
 
 	def testExecuteActionBan(self):
 		self.__action.setActionStart("touch /tmp/fail2ban.test")

fail2ban/tests/databasetestcase.py

@@ -127,7 +127,7 @@ class DatabaseTest(unittest.TestCase):
 		ticket = FailTicket("127.0.0.1", 0, ["abc\n"])
 		self.db.addBan(self.jail, ticket)
 
-		self.assertEquals(len(self.db.getBans(self.jail)), 1)
+		self.assertEquals(len(self.db.getBans(jail=self.jail)), 1)
 		self.assertTrue(
 			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
 
@@ -162,6 +162,19 @@ class DatabaseTest(unittest.TestCase):
 		self.assertEqual(ticket.getAttempt(), 30)
 		self.assertEqual(ticket.getMatches(), ["abc\n", "123\n"])
 
+		# Should cache result if no extra bans added
+		self.assertEqual(
+			id(ticket),
+			id(self.db.getBansMerged("127.0.0.1", jail=self.jail)))
+
+		newTicket = FailTicket("127.0.0.1", 40, ["ABC\n"])
+		ticket.setAttempt(40)
+		self.db.addBan(self.jail, newTicket)
+		# Added ticket, so cache should have been cleared
+		self.assertNotEqual(
+			id(ticket),
+			id(self.db.getBansMerged("127.0.0.1", jail=self.jail)))
+
 	def testPurge(self):
 		self.testAddJail() # Add jail