From 097970781cbe61160824c834ca5cb4a065c2a8f8 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 16:53:46 +0100 Subject: [PATCH 1/4] filter/action (and its includes): substitution `%(param)s` may be used now (instead of ``) for init-values specified in jail-configs via `action[param1="...", param2=...]`; substitution `` should be used for dynamic interpolation only (todo: review configurations to replace it); --- fail2ban/client/configparserinc.py | 2 +- fail2ban/client/configreader.py | 32 ++++++++++++++++++++++++------ fail2ban/client/filterreader.py | 6 ++++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index d6cda7f7..35fa7498 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): # pragma: no cover +if sys.version_info >= (3,2): # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, \ diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 643cdf3a..caab67aa 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -28,13 +28,25 @@ import glob import os from ConfigParser import NoOptionError, NoSectionError -from .configparserinc import SafeConfigParserWithIncludes, logLevel +from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +# if sys.version_info >= (3,5): +# def _merge_dicts(x, y): +# return {**x, **y} +# else: +def _merge_dicts(x, y): + r = x + if y: + r = x.copy() + r.update(y) + return r + + class ConfigReader(): """Generic config reader class. @@ -127,9 +139,9 @@ class ConfigReader(): return self._cfg.options(*args) return {} - def get(self, sec, opt): + def get(self, sec, opt, raw=False, vars={}): if self._cfg is not None: - return self._cfg.get(sec, opt) + return self._cfg.get(sec, opt, raw, vars) return None def getOptions(self, *args, **kwargs): @@ -210,6 +222,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): def getOptions(self, sec, options, pOptions=None, shouldExist=False): values = dict() + if pOptions is None: + pOptions = {} for optname in options: if isinstance(options, (list,tuple)): if len(optname) > 2: @@ -218,15 +232,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): (opttype, optname), optvalue = optname, None else: opttype, optvalue = options[optname] + if optname in pOptions: + continue try: if opttype == "bool": v = self.getboolean(sec, optname) elif opttype == "int": v = self.getint(sec, optname) else: - v = self.get(sec, optname) - if not pOptions is None and optname in pOptions: - continue + v = self.get(sec, optname, vars=pOptions) values[optname] = v except NoSectionError as e: if shouldExist: @@ -289,6 +303,12 @@ class DefinitionInitConfigReader(ConfigReader): return SafeConfigParserWithIncludes.read(self._cfg, self._file) def getOptions(self, pOpts): + # overwrite static definition options with init values, supplied as + # direct parameters from jail-config via action[xtra1="...", xtra2=...]: + if self._initOpts: + if not pOpts: + pOpts = dict() + pOpts = _merge_dicts(pOpts, self._initOpts) self._opts = ConfigReader.getOptions( self, "Definition", self._configOpts, pOpts) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 8b30f914..5e6b2b74 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import os import shlex -from .configreader import DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader, _merge_dicts from ..server.action import CommandAction from ..helpers import getLogger @@ -50,7 +50,9 @@ class FilterReader(DefinitionInitConfigReader): return self.__file def getCombined(self): - combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) + combinedopts = self._opts + if self._initOpts: + combinedopts = _merge_dicts(self._opts, self._initOpts) if not len(combinedopts): return {} opts = CommandAction.substituteRecursiveTags(combinedopts) From 65abc639cc7838beaf1f587eae74e98afc4a1473 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 16:56:46 +0100 Subject: [PATCH 2/4] allow newline in extra init-parameters of action/filter (or interpolation of it), e. g. action[..., logpath="%(logpath)s"] --- fail2ban/client/jailreader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index b63df5f1..d01064b2 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -43,13 +43,13 @@ logSys = getLogger(__name__) class JailReader(ConfigReader): # regex, to extract list of options: - optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") + optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL) # regex, to iterate over single option in option list, syntax: # `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']' # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` optionExtractRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)') + r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) def __init__(self, name, force_enable=False, **kwargs): ConfigReader.__init__(self, **kwargs) From a2af19c9f0c7ad42e733693f1cbf3d247cded582 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 19:06:34 +0100 Subject: [PATCH 3/4] fixed several actions, that could not work with jails using multiple logpath; additionally repaired execution in default shell (bad substitution by `${x//...}` executing in `/bin/sh`); added helper "action.d/helpers-common.conf", and `_grep_logs` part-command for actions needed grep logs from multiple log-files test cases: executing of some complex actions covered --- config/action.d/complain.conf | 20 ++++- config/action.d/helpers-common.conf | 13 +++ config/action.d/mail-whois-lines.conf | 29 ++++--- config/action.d/sendmail-geoip-lines.conf | 13 +-- config/action.d/sendmail-whois-lines.conf | 13 +-- fail2ban/tests/files/testcase01a.log | 4 + fail2ban/tests/servertestcase.py | 96 +++++++++++++++++++++-- 7 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 config/action.d/helpers-common.conf create mode 100644 fail2ban/tests/files/testcase01a.log diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 9247803e..e4ceb35f 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -28,6 +28,10 @@ # +[INCLUDES] + +before = helpers-common.conf + [Definition] # Option: actionstart @@ -54,10 +58,16 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = oifs=${IFS}; IFS=.;SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs} - IP= +actionban = oifs=${IFS}; + IFS=.; SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); + IFS=,; ADDRESSES=$(echo $ADDRESSES) + IFS=${oifs} + IP= if [ ! -z "$ADDRESSES" ]; then - (printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; grep -E '(^|[^0-9])([^0-9]|$)' ) | "Abuse from " ${ADDRESSES//,/\" \"} + ( printf %%b "\n"; date '+Note: Local timezone is %%z (%%Z)'; + printf %%b "\nLines containing failures of (max )\n"; + %(_grep_logs)s; + ) | "Abuse from " $ADDRESSES fi # Option: actionunban @@ -92,3 +102,7 @@ mailcmd = mail -s # mailargs = +# Number of log lines to include in the email +# +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf new file mode 100644 index 00000000..7fa8e9e4 --- /dev/null +++ b/config/action.d/helpers-common.conf @@ -0,0 +1,13 @@ +[DEFAULT] + +# Usage: +# _grep_logs_args = 'test' +# (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... +# +_grep_logs = logpath=""; grep -E %(_grep_logs_args)s $logpath | +_grep_logs_args = '(^|[^0-9])([^0-9]|$)' + +[Init] +greplimit = tail -n +grepmax = 1000 +grepopts = -m \ No newline at end of file diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 6e39c605..cbd970c9 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = mail-whois-common.conf + helpers-common.conf [Definition] @@ -17,7 +18,7 @@ before = mail-whois-common.conf actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on `uname -n`" + Fail2Ban" | -s "[Fail2Ban] : started on `uname -n`" # Option: actionstop # Notes.: command executed once at the end of Fail2Ban @@ -26,7 +27,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on `uname -n`" + Fail2Ban" | -s "[Fail2Ban] : stopped on `uname -n`" # Option: actioncheck # Notes.: command executed once before each actionban command @@ -40,15 +41,18 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Hi,\n + +_ban_mail_content = ( printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n - Here is more information about :\n - `%(_whois_command)s`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Here is more information about :\n" + %(_whois_command)s; + printf %%b "\nLines containing failures of (max )\n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" + Fail2Ban" ) +actionban = %(_ban_mail_content)s | "[Fail2Ban] : banned from `uname -n`" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -60,6 +64,12 @@ actionunban = [Init] +# Option: mailcmd +# Notes.: Your system mail command. Is passed 2 args: subject and recipient +# Values: CMD +# +mailcmd = mail -s + # Default name of the chain # name = default @@ -74,4 +84,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index 2232642c..a5616e9f 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + helpers-common.conf [Definition] @@ -19,7 +20,7 @@ before = sendmail-common.conf # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n @@ -33,10 +34,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "" | cut -d':' -f2-` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "" | cut -d':' -f2-` hostname: `host -t A 2>&1`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Lines containing failures of \n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" ) | /usr/sbin/sendmail -f [Init] @@ -50,4 +52,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4156c947..e1c85928 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + helpers-common.conf [Definition] @@ -16,7 +17,7 @@ before = sendmail-common.conf # Tags: See jail.conf(5) man page # Values: CMD # -actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` +actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n` Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"` From: <> To: \n @@ -25,10 +26,11 @@ actionban = printf %%b "Subject: [Fail2Ban] : banned from `uname -n` attempts against .\n\n Here is more information about :\n `/usr/bin/whois || echo missing whois program`\n\n - Lines containing IP: in \n - `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n + Lines containing failures of \n"; + %(_grep_logs)s; + printf %%b "\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f + Fail2Ban" ) | /usr/sbin/sendmail -f [Init] @@ -42,4 +44,5 @@ logpath = /dev/null # Number of log lines to include in the email # -grepopts = -m 1000 +#grepmax = 1000 +#grepopts = -m diff --git a/fail2ban/tests/files/testcase01a.log b/fail2ban/tests/files/testcase01a.log new file mode 100644 index 00000000..203f0517 --- /dev/null +++ b/fail2ban/tests/files/testcase01a.log @@ -0,0 +1,4 @@ +Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:02 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:03 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 +Dec 31 11:55:04 [sshd] error: PAM: Authentication failure for test from 87.142.124.10 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 56c85e94..f8915b59 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -28,6 +28,7 @@ import unittest import time import tempfile import os +import re import sys import platform @@ -1609,31 +1610,114 @@ class ServerConfigReaderTests(LogCaptureTestCase): # wrap default command processor: action.executeCmd = self._executeCmd # test start : - logSys.debug('# === start ==='); self.pruneLog() + self.pruneLog('# === start ===') action.start() self.assertLogged(*tests['start'], all=True) # test ban ip4 : - logSys.debug('# === ban-ipv4 ==='); self.pruneLog() + self.pruneLog('# === ban-ipv4 ===') action.ban({'ip': IPAddr('192.0.2.1')}) self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test unban ip4 : - logSys.debug('# === unban ipv4 ==='); self.pruneLog() + self.pruneLog('# === unban ipv4 ===') action.unban({'ip': IPAddr('192.0.2.1')}) self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test ban ip6 : - logSys.debug('# === ban ipv6 ==='); self.pruneLog() + self.pruneLog('# === ban ipv6 ===') action.ban({'ip': IPAddr('2001:DB8::')}) self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test unban ip6 : - logSys.debug('# === unban ipv6 ==='); self.pruneLog() + self.pruneLog('# === unban ipv6 ===') action.unban({'ip': IPAddr('2001:DB8::')}) self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test stop : - logSys.debug('# === stop ==='); self.pruneLog() + self.pruneLog('# === stop ===') action.stop() self.assertLogged(*tests['stop'], all=True) + def _executeMailCmd(self, realCmd, timeout=60): + # replace pipe to mail with pipe to cat: + realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)', + r' echo mail \1 ) | cat', realCmd) + # replace abuse retrieving (possible no-network): + realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+', + 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd) + # execute action: + return _actions.CommandAction.executeCmd(realCmd, timeout=timeout) + + def testComplexMailActionMultiLog(self): + testJailsActions = ( + # mail-whois-lines -- + ('j-mail-whois-lines', + 'mail-whois-lines[' + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s", ' + + # 2 logs to test grep from multiple logs: + 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' + '_whois_command="echo \'-- information about --\'"' + ']', + { + 'ip4-ban': ( + 'The IP 87.142.124.10 has just been banned by Fail2Ban after', + '100 attempts against j-mail-whois-lines.', + 'Here is more information about 87.142.124.10 :', + '-- information about 87.142.124.10 --', + 'Lines containing failures of 87.142.124.10 (max 2)', + 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + ), + }), + # complain -- + ('j-complain-abuse', + 'complain[' + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' + + # 2 logs to test grep from multiple logs: + 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' + ']', + { + 'ip4-ban': ( + 'Lines containing failures of 87.142.124.10 (max 2)', + 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + # both abuse mails should be separated with space: + 'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', + ), + }), + ) + server = TestServer() + transm = server._Server__transm + cmdHandler = transm._Transmitter__commandHandler + + for jail, act, tests in testJailsActions: + stream = self.getDefaultJailStream(jail, act) + + # for cmd in stream: + # print(cmd) + + # transmit jail to the server: + for cmd in stream: + # command to server: + ret, res = transm.proceed(cmd) + self.assertEqual(ret, 0) + + jails = server._Server__jails + + for jail, act, tests in testJailsActions: + # print(jail, jails[jail]) + for a in jails[jail].actions: + action = jails[jail].actions[a] + logSys.debug('# ' + ('=' * 50)) + logSys.debug('# == %-44s ==', jail + ' - ' + action._name) + logSys.debug('# ' + ('=' * 50)) + # wrap default command processor: + action.executeCmd = self._executeMailCmd + # test ban : + self.pruneLog('# === ban ===') + action.ban({'ip': IPAddr('87.142.124.10'), + 'failures': 100, + }) + self.assertLogged(*tests['ip4-ban'], all=True) From ec7bb0d6c9c06dcd9a09d8bd7338d9cfc4359a0d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 25 Nov 2016 20:12:49 +0100 Subject: [PATCH 4/4] python 3x compatibility fix (positional arguments vs named arguments) --- fail2ban/client/configreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index caab67aa..a72ca1e9 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -141,7 +141,7 @@ class ConfigReader(): def get(self, sec, opt, raw=False, vars={}): if self._cfg is not None: - return self._cfg.get(sec, opt, raw, vars) + return self._cfg.get(sec, opt, raw=raw, vars=vars) return None def getOptions(self, *args, **kwargs):