Merge pull request #1622 from sebres/_0.10/configreader-and-more

0.10/configreader and more:  substitution `%(param)s` from init block

config/action.d/complain.conf

@@ -28,6 +28,10 @@
 #
 
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -54,10 +58,16 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
+actionban = oifs=${IFS}; 
+              IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
+              IFS=,; ADDRESSES=$(echo $ADDRESSES)
+            IFS=${oifs}
             IP=<ip>
             if [ ! -z "$ADDRESSES" ]; then
-                (printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])<ip>([^0-9]|$)' <logpath>) | <mailcmd> "Abuse from <ip>" <mailargs> ${ADDRESSES//,/\" \"}
+                ( printf %%b "<message>\n"; date '+Note: Local timezone is %%z (%%Z)'; 
+                  printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
+                  %(_grep_logs)s;
+                ) | <mailcmd> "Abuse from <ip>" <mailargs> $ADDRESSES
             fi
 
 # Option:  actionunban
@@ -92,3 +102,7 @@ mailcmd = mail -s
 #
 mailargs =
 
+# Number of log lines to include in the email
+#
+#grepmax = 1000
+#grepopts = -m <grepmax>

config/action.d/helpers-common.conf

@@ -0,0 +1,13 @@
+[DEFAULT]
+
+# Usage:
+#   _grep_logs_args = 'test'
+#   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
+#
+_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
+_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
+
+[Init]
+greplimit = tail -n <grepmax>
+grepmax = 1000
+grepopts = -m <grepmax>

config/action.d/mail-whois-lines.conf

@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -17,7 +18,7 @@ before = mail-whois-common.conf
 actionstart = printf %%b "Hi,\n
               The jail <name> has been started successfully.\n
               Regards,\n
-              Fail2Ban"|mail -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
+              Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: started on `uname -n`" <dest>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -26,7 +27,7 @@ actionstart = printf %%b "Hi,\n
 actionstop = printf %%b "Hi,\n
              The jail <name> has been stopped.\n
              Regards,\n
-             Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
+             Fail2Ban" | <mailcmd> -s "[Fail2Ban] <name>: stopped on `uname -n`" <dest>
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -40,15 +41,18 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+
+_ban_mail_content = ( printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
-            Here is more information about <ip> :\n
+            Here is more information about <ip> :\n"
-            `%(_whois_command)s`\n\n
+            %(_whois_command)s;
-            Lines containing IP:<ip> in <logpath>\n
+            printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n";
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+            Fail2Ban" )
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -60,6 +64,12 @@ actionunban =
 
 [Init]
 
+# Option:  mailcmd
+# Notes.:  Your system mail command. Is passed 2 args: subject and recipient
+# Values:  CMD
+#
+mailcmd = mail -s
+
 # Default name of the chain
 #
 name = default
@@ -74,4 +84,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>

config/action.d/sendmail-geoip-lines.conf

@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -19,7 +20,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -33,10 +34,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "<ip>" | cut -d':' -f2-`
             AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "<ip>" | cut -d':' -f2-`
             hostname: `host -t A <ip> 2>&1`\n\n
-            Lines containing IP:<ip> in <logpath>\n
+            Lines containing failures of <ip>\n";
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -50,4 +52,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>

config/action.d/sendmail-whois-lines.conf

@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
@@ -25,10 +26,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
             `/usr/bin/whois <ip> || echo missing whois program`\n\n
-            Lines containing IP:<ip> in <logpath>\n
+            Lines containing failures of <ip>\n";
-            `grep -E <grepopts> '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
+            %(_grep_logs)s;
+            printf %%b "\n
             Regards,\n
-            Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
+            Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest>
 
 [Init]
 
@@ -42,4 +44,5 @@ logpath = /dev/null
 
 # Number of log lines to include in the email
 #
-grepopts = -m 1000
+#grepmax = 1000
+#grepopts = -m <grepmax>

fail2ban/client/configparserinc.py

@@ -29,7 +29,7 @@ import re
 import sys
 from ..helpers import getLogger
 
-if sys.version_info >= (3,2): # pragma: no cover
+if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, \

fail2ban/client/configreader.py

@@ -28,13 +28,25 @@ import glob
 import os
 from ConfigParser import NoOptionError, NoSectionError
 
-from .configparserinc import SafeConfigParserWithIncludes, logLevel
+from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
+# if sys.version_info >= (3,5):
+# 	def _merge_dicts(x, y):
+# 		return {**x, **y}
+# else:
+def _merge_dicts(x, y):
+	r = x
+	if y:
+		r = x.copy()
+		r.update(y)
+	return r
+
+
 class ConfigReader():
 	"""Generic config reader class.
 
@@ -127,9 +139,9 @@ class ConfigReader():
 			return self._cfg.options(*args)
 		return {}
 
-	def get(self, sec, opt):
+	def get(self, sec, opt, raw=False, vars={}):
 		if self._cfg is not None:
-			return self._cfg.get(sec, opt)
+			return self._cfg.get(sec, opt, raw=raw, vars=vars)
 		return None
 
 	def getOptions(self, *args, **kwargs):
@@ -210,6 +222,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	
 	def getOptions(self, sec, options, pOptions=None, shouldExist=False):
 		values = dict()
+		if pOptions is None:
+			pOptions = {}
 		for optname in options:
 			if isinstance(options, (list,tuple)):
 				if len(optname) > 2:
@@ -218,15 +232,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 					(opttype, optname), optvalue = optname, None
 			else:
 				opttype, optvalue = options[optname]
+			if optname in pOptions:
+				continue
 			try:
 				if opttype == "bool":
 					v = self.getboolean(sec, optname)
 				elif opttype == "int":
 					v = self.getint(sec, optname)
 				else:
-					v = self.get(sec, optname)
+					v = self.get(sec, optname, vars=pOptions)
-				if not pOptions is None and optname in pOptions:
-					continue
 				values[optname] = v
 			except NoSectionError as e:
 				if shouldExist:
@@ -289,6 +303,12 @@ class DefinitionInitConfigReader(ConfigReader):
 		return SafeConfigParserWithIncludes.read(self._cfg, self._file)
 	
 	def getOptions(self, pOpts):
+		# overwrite static definition options with init values, supplied as
+		# direct parameters from jail-config via action[xtra1="...", xtra2=...]:
+		if self._initOpts:
+			if not pOpts:
+				pOpts = dict()
+			pOpts = _merge_dicts(pOpts, self._initOpts)
 		self._opts = ConfigReader.getOptions(
 			self, "Definition", self._configOpts, pOpts)
 		

fail2ban/client/filterreader.py

@@ -27,7 +27,7 @@ __license__ = "GPL"
 import os
 import shlex
 
-from .configreader import DefinitionInitConfigReader
+from .configreader import DefinitionInitConfigReader, _merge_dicts
 from ..server.action import CommandAction
 from ..helpers import getLogger
 
@@ -50,7 +50,9 @@ class FilterReader(DefinitionInitConfigReader):
 		return self.__file
 
 	def getCombined(self):
-		combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items()))
+		combinedopts = self._opts
+		if self._initOpts:
+			combinedopts = _merge_dicts(self._opts, self._initOpts)
 		if not len(combinedopts):
 			return {}
 		opts = CommandAction.substituteRecursiveTags(combinedopts)

fail2ban/client/jailreader.py

@@ -43,13 +43,13 @@ logSys = getLogger(__name__)
 class JailReader(ConfigReader):
 	
 	# regex, to extract list of options:
-	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
+	optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL)
 	# regex, to iterate over single option in option list, syntax:
 	# `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
 	# since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
 	# `action = act[p1=...][p2=...]`
 	optionExtractRE = re.compile(
-		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)')
+		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL)
 	
 	def __init__(self, name, force_enable=False, **kwargs):
 		ConfigReader.__init__(self, **kwargs)

fail2ban/tests/files/testcase01a.log

@@ -0,0 +1,4 @@
+Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:02 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:03 [sshd] error: PAM: Authentication failure for test from 87.142.124.10
+Dec 31 11:55:04 [sshd] error: PAM: Authentication failure for test from 87.142.124.10

fail2ban/tests/servertestcase.py

@@ -28,6 +28,7 @@ import unittest
 import time
 import tempfile
 import os
+import re
 import sys
 import platform
 
@@ -1609,31 +1610,114 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					# wrap default command processor:
 					action.executeCmd = self._executeCmd
 					# test start :
-					logSys.debug('# === start ==='); self.pruneLog()
+					self.pruneLog('# === start ===')
 					action.start()
 					self.assertLogged(*tests['start'], all=True)
 					# test ban ip4 :
-					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+					self.pruneLog('# === ban-ipv4 ===')
 					action.ban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test unban ip4 :
-					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+					self.pruneLog('# === unban ipv4 ===')
 					action.unban({'ip': IPAddr('192.0.2.1')})
 					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
 					self.assertNotLogged(*tests['ip6'], all=True)
 					# test ban ip6 :
-					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+					self.pruneLog('# === ban ipv6 ===')
 					action.ban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test unban ip6 :
-					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+					self.pruneLog('# === unban ipv6 ===')
 					action.unban({'ip': IPAddr('2001:DB8::')})
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
 					# test stop :
-					logSys.debug('# === stop ==='); self.pruneLog()
+					self.pruneLog('# === stop ===')
 					action.stop()
 					self.assertLogged(*tests['stop'], all=True)
 
+		def _executeMailCmd(self, realCmd, timeout=60):
+			# replace pipe to mail with pipe to cat:
+			realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)',
+				r' echo mail \1 ) | cat', realCmd)
+			# replace abuse retrieving (possible no-network):
+			realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+',
+				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
+			# execute action:
+			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
+
+		def testComplexMailActionMultiLog(self):
+			testJailsActions = (
+				# mail-whois-lines --
+				('j-mail-whois-lines', 
+					'mail-whois-lines['
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s", ' +
+						# 2 logs to test grep from multiple logs:
+					  'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' +
+				    '         ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", '
+					  '_whois_command="echo \'-- information about <ip> --\'"'
+					  ']',
+				{
+					'ip4-ban': (
+						'The IP 87.142.124.10 has just been banned by Fail2Ban after',
+						'100 attempts against j-mail-whois-lines.',
+						'Here is more information about 87.142.124.10 :',
+						'-- information about 87.142.124.10 --',
+						'Lines containing failures of 87.142.124.10 (max 2)',
+						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
+						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
+					),
+				}),
+				# complain --
+				('j-complain-abuse', 
+					'complain['
+					  'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' +
+						# 2 logs to test grep from multiple logs:
+					  'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' +
+				    '         ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", '
+					  ']',
+				{
+					'ip4-ban': (
+						'Lines containing failures of 87.142.124.10 (max 2)',
+						'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10',
+						'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10',
+						# both abuse mails should be separated with space:
+						'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server',
+					),
+				}),
+			)
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			for jail, act, tests in testJailsActions:
+				stream = self.getDefaultJailStream(jail, act)
+
+				# for cmd in stream:
+				# 	print(cmd)
+
+				# transmit jail to the server:
+				for cmd in stream:
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
+
+			jails = server._Server__jails
+
+			for jail, act, tests in testJailsActions:
+				# print(jail, jails[jail])
+				for a in jails[jail].actions:
+					action = jails[jail].actions[a]
+					logSys.debug('# ' + ('=' * 50))
+					logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+					logSys.debug('# ' + ('=' * 50))
+					# wrap default command processor:
+					action.executeCmd = self._executeMailCmd
+					# test ban :
+					self.pruneLog('# === ban ===')
+					action.ban({'ip': IPAddr('87.142.124.10'), 
+						'failures': 100,
+					})
+					self.assertLogged(*tests['ip4-ban'], all=True)