From 901eeff53df0b3b73d607212aa9b1fd526617176 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Mon, 6 Feb 2017 22:04:36 +0100 Subject: [PATCH 01/23] Make Abusix lookup compatible with Dash --- config/action.d/complain.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 84dbaf39..9934bd94 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -62,8 +62,9 @@ actioncheck = # Values: CMD # actionban = oifs=${IFS}; - IFS=.; SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); - IFS=,; ADDRESSES=$(echo $ADDRESSES) + REV_IP=$(printf %%s ${IP}. | tac -s.) + ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org) + IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} IP= if [ ! -z "$ADDRESSES" ]; then From 1bcf0de7c168da9211982e64b28a839a6078dcf6 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Tue, 7 Feb 2017 21:39:46 +0100 Subject: [PATCH 02/23] Update complain.conf --- config/action.d/complain.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 9934bd94..16117db8 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -62,7 +62,7 @@ actioncheck = # Values: CMD # actionban = oifs=${IFS}; - REV_IP=$(printf %%s ${IP}. | tac -s.) + REV_IP=$(printf %%s . | tac -s.) ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org) IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} From 58c68b75f0b2d879f3fca03cd6e14e66cf5f5804 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Wed, 8 Feb 2017 14:16:13 +0100 Subject: [PATCH 03/23] Remove double-quotes from email addresses --- config/action.d/complain.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 16117db8..3f97d188 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -63,7 +63,7 @@ actioncheck = # actionban = oifs=${IFS}; REV_IP=$(printf %%s . | tac -s.) - ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org) + ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org | tr -d '"') IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} IP= From 2d12349eaccb221e6ffc8aea80a366b5ebaea579 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 15 Feb 2017 20:30:48 +0100 Subject: [PATCH 04/23] Update servertestcase.py Make the test-case gh-1685 compliant --- fail2ban/tests/servertestcase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index aada699c..c5b997f3 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1644,9 +1644,9 @@ class ServerConfigReaderTests(LogCaptureTestCase): # replace pipe to mail with pipe to cat: realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)', r' echo mail \1 ) | cat', realCmd) - # replace abuse retrieving (possible no-network): - realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+', - 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd) + # replace abuse retrieving (possible no-network), just replace first occurrence of 'dig...': + realCmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+', + 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd, 1) # execute action: return _actions.CommandAction.executeCmd(realCmd, timeout=timeout) From fc315be4ea88c3619f984542b21c95820f53d87b Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 13:32:56 +0100 Subject: [PATCH 05/23] try to parse and interpolate all options in section "Definition" (section "Init" no more needed) --- fail2ban/client/configreader.py | 22 ++++++++++++++++++---- fail2ban/server/action.py | 6 +++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 7840cd12..b44a8c57 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -221,10 +221,14 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, + allOpts=None, shouldExist=False + ): values = dict() if pOptions is None: pOptions = {} + + # Get only specified options: for optname in options: if isinstance(options, (list,tuple)): if len(optname) > 2: @@ -261,6 +265,15 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue + + # Fill all option of the section (used for replacement): + if allOpts is not None and self.has_section(sec): + for optname in self.options(sec): + v = values.get(optname) + if v is None: + v = self.get(sec, optname, vars=pOptions) + allOpts[optname] = v + return values @@ -310,9 +323,9 @@ class DefinitionInitConfigReader(ConfigReader): if not pOpts: pOpts = dict() pOpts = _merge_dicts(pOpts, self._initOpts) + self._allOpts = dict() self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) - + self, "Definition", self._configOpts, pOpts, allOpts=self._allOpts) if self.has_section("Init"): for opt in self.options("Init"): v = self.get("Init", opt) @@ -338,7 +351,8 @@ class DefinitionInitConfigReader(ConfigReader): n, cond = cond.groups() ignore.add(n) # substiture options already specified direct: - opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore) + opts = CommandAction.substituteRecursiveTags(combinedopts, + ignore=ignore, addtags=self._allOpts) if not opts: raise ValueError('recursive tag definitions unable to be resolved') return opts diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 46a19cd1..69cd84a3 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -365,7 +365,9 @@ class CommandAction(ActionBase): return self._executeOperation('', 'reloading') @classmethod - def substituteRecursiveTags(cls, inptags, conditional='', ignore=()): + def substituteRecursiveTags(cls, inptags, conditional='', + ignore=(), addtags={} + ): """Sort out tag definitions within other tags. Since v.0.9.2 supports embedded interpolation (see test cases for examples). @@ -420,6 +422,8 @@ class CommandAction(ActionBase): repl = tags.get(found_tag + '?' + conditional) if repl is None: repl = tags.get(found_tag) + if repl is None: + repl = addtags.get(found_tag) if repl is None: # Escaped or missing tags - just continue on searching after end of match # Missing tags are ok - cInfo can contain aInfo elements like and valid shell From 4bf09bf2972df852133f64a3645eb391c16f51c9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 13:38:20 +0100 Subject: [PATCH 06/23] provides new tag `` for PTR reversed representation of IP address; [action.d/complain.conf] fixed using this new tag; --- config/action.d/complain.conf | 17 +++++++++++++---- fail2ban/server/actions.py | 16 +++++++++------- fail2ban/server/ipdns.py | 6 +++--- fail2ban/tests/servertestcase.py | 25 ++++++++++++++++++++----- 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 3f97d188..1f74d635 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -34,6 +34,9 @@ before = helpers-common.conf [Definition] +# Used in test cases for coverage internal transformations +debug = 0 + # bypass ban/unban for restored tickets norestored = 1 @@ -61,9 +64,10 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = oifs=${IFS}; - REV_IP=$(printf %%s . | tac -s.) - ADDRESSES=$(dig +short -t txt -q ${REV_IP}abuse-contacts.abusix.org | tr -d '"') +actionban = oifs=${IFS}; + RESOLVER_ADDR="%(addr_resolver)s" + if [ "" -gt 0 ]; then echo "try to resolve $RESOLVER_ADDR"; fi + ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | tr -d '"') IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} IP= @@ -82,7 +86,12 @@ actionban = oifs=${IFS}; # actionunban = -[Init] +# Server as resolver used in dig command +# +addr_resolver = abuse-contacts.abusix.org + +# Default message used for abuse content +# message = Dear Sir/Madam,\n\nWe have detected abuse from the IP address $IP, which according to a abusix.com is on your network. We would appreciate if you would investigate and take action as appropriate.\n\nLog lines are given below, but please ask if you require any further information.\n\n(If you are not the correct person to contact about this please accept our apologies - your e-mail address was extracted from the whois record by an automated process.)\n\n This mail was generated by Fail2Ban.\nThe recipient address of this report was provided by the Abuse Contact DB by abusix.com. abusix.com does not maintain the content of the database. All information which we pass out, derives from the RIR databases and is processed for ease of use. If you want to change or report non working abuse contacts please contact the appropriate RIR. If you have any further question, contact abusix.com directly via email (info@abusix.com). Information about the Abuse Contact Database can be found here: https://abusix.com/global-reporting/abuse-contact-db\nabusix.com is neither responsible nor liable for the content or accuracy of this message.\n # Path to the log files which contain relevant lines for the abuser IP diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 28c23abd..e2046e15 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -341,15 +341,17 @@ class Actions(JailThread, Mapping): ticket = self._jail.getFailTicket() if not ticket: break - aInfo = CallingMap() bTicket = BanManager.createBanTicket(ticket) ip = bTicket.getIP() - aInfo["ip"] = ip - aInfo["failures"] = bTicket.getAttempt() - aInfo["time"] = bTicket.getTime() - aInfo["matches"] = "\n".join(bTicket.getMatches()) - # to bypass actions, that should not be executed for restored tickets - aInfo["restored"] = 1 if ticket.restored else 0 + aInfo = CallingMap({ + "ip" : ip, + "ip-rev" : lambda: ip.getPTR(''), + "failures": bTicket.getAttempt(), + "time" : bTicket.getTime(), + "matches" : "\n".join(bTicket.getMatches()), + # to bypass actions, that should not be executed for restored tickets + "restored": (1 if ticket.restored else 0) + }) if self._jail.database is not None: mi4ip = lambda overalljails=False, self=self, \ mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index f8db6a04..a5625629 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -346,7 +346,7 @@ class IPAddr(object): return socket.inet_ntop(self._family, binary) + add - def getPTR(self, suffix=""): + def getPTR(self, suffix=None): """ return the DNS PTR string of the provided IP address object If "suffix" is provided it will be appended as the second and top @@ -356,11 +356,11 @@ class IPAddr(object): """ if self.isIPv4: exploded_ip = self.ntoa.split(".") - if not suffix: + if suffix is None: suffix = "in-addr.arpa." elif self.isIPv6: exploded_ip = self.hexdump - if not suffix: + if suffix is None: suffix = "ip6.arpa." else: return "" diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index f5541ed0..fce03573 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1678,18 +1678,29 @@ class ServerConfigReaderTests(LogCaptureTestCase): ('j-complain-abuse', 'complain[' 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' + + # test reverse ip: + 'debug=1,' + # 2 logs to test grep from multiple logs: 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' ']', { 'ip4-ban': ( + # test reverse ip: + 'try to resolve 10.124.142.87.abuse-contacts.abusix.org', 'Lines containing failures of 87.142.124.10 (max 2)', 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', # both abuse mails should be separated with space: 'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), + 'ip6-ban': ( + # test reverse ip: + 'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org', + 'Lines containing failures of 2001:db8::1 (max 2)', + # both abuse mails should be separated with space: + 'mail -s Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', + ), }), ) server = TestServer() @@ -1710,6 +1721,8 @@ class ServerConfigReaderTests(LogCaptureTestCase): jails = server._Server__jails + ipv4 = IPAddr('87.142.124.10') + ipv6 = IPAddr('2001:db8::1'); for jail, act, tests in testJailsActions: # print(jail, jails[jail]) for a in jails[jail].actions: @@ -1720,8 +1733,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): # wrap default command processor: action.executeCmd = self._executeMailCmd # test ban : - self.pruneLog('# === ban ===') - action.ban({'ip': IPAddr('87.142.124.10'), - 'failures': 100, - }) - self.assertLogged(*tests['ip4-ban'], all=True) + for (test, ip) in (('ip4-ban', ipv4), ('ip6-ban', ipv6)): + if not tests.get(test): continue + self.pruneLog('# === %s ===' % test) + ticket = _actions.CallingMap({ + 'ip': ip, 'ip-rev': lambda: ip.getPTR(''), 'failures': 100,}) + action.ban(ticket) + self.assertLogged(*tests[test], all=True) From 3fae8a7e432217c5ff650ef156a35663e4ccc78d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 14:53:57 +0100 Subject: [PATCH 07/23] amend to fc315be4ea88c3619f984542b21c95820f53d87b: parse and interpolate all options in section "Definition" (section "Init" no more needed), because of better performance with this solution; --- fail2ban/client/configreader.py | 33 +++++++++++++++++---------------- fail2ban/server/action.py | 9 +++++---- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b44a8c57..8098433e 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -221,13 +221,10 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, - allOpts=None, shouldExist=False - ): + def getOptions(self, sec, options, pOptions=None, shouldExist=False): values = dict() if pOptions is None: pOptions = {} - # Get only specified options: for optname in options: if isinstance(options, (list,tuple)): @@ -265,15 +262,6 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") values[optname] = optvalue - - # Fill all option of the section (used for replacement): - if allOpts is not None and self.has_section(sec): - for optname in self.options(sec): - v = values.get(optname) - if v is None: - v = self.get(sec, optname, vars=pOptions) - allOpts[optname] = v - return values @@ -293,6 +281,8 @@ class DefinitionInitConfigReader(ConfigReader): self.setFile(file_) self.setJailName(jailName) self._initOpts = initOpts + self._pOpts = dict() + self._defCache = dict() def setFile(self, fileName): self._file = fileName @@ -323,9 +313,9 @@ class DefinitionInitConfigReader(ConfigReader): if not pOpts: pOpts = dict() pOpts = _merge_dicts(pOpts, self._initOpts) - self._allOpts = dict() self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts, allOpts=self._allOpts) + self, "Definition", self._configOpts, pOpts) + self._pOpts = pOpts if self.has_section("Init"): for opt in self.options("Init"): v = self.get("Init", opt) @@ -337,6 +327,17 @@ class DefinitionInitConfigReader(ConfigReader): def _convert_to_boolean(self, value): return value.lower() in ("1", "yes", "true", "on") + def getCombOption(self, optname): + try: + return self._defCache[optname] + except KeyError: + try: + v = self.get("Definition", optname, vars=self._pOpts) + except (NoSectionError, NoOptionError, ValueError): + v = None + self._defCache[optname] = v + return v + def getCombined(self, ignore=()): combinedopts = self._opts ignore = set(ignore).copy() @@ -352,7 +353,7 @@ class DefinitionInitConfigReader(ConfigReader): ignore.add(n) # substiture options already specified direct: opts = CommandAction.substituteRecursiveTags(combinedopts, - ignore=ignore, addtags=self._allOpts) + ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') return opts diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 69cd84a3..d14c9aab 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -366,7 +366,7 @@ class CommandAction(ActionBase): @classmethod def substituteRecursiveTags(cls, inptags, conditional='', - ignore=(), addtags={} + ignore=(), addrepl=None ): """Sort out tag definitions within other tags. Since v.0.9.2 supports embedded interpolation (see test cases for examples). @@ -419,11 +419,12 @@ class CommandAction(ActionBase): (tag, found_tag, refCounts, value)) repl = None if found_tag not in cls._escapedTags: - repl = tags.get(found_tag + '?' + conditional) + if conditional: + repl = tags.get(found_tag + '?' + conditional) if repl is None: repl = tags.get(found_tag) - if repl is None: - repl = addtags.get(found_tag) + if repl is None and addrepl is not None: + repl = addrepl(found_tag) if repl is None: # Escaped or missing tags - just continue on searching after end of match # Missing tags are ok - cInfo can contain aInfo elements like and valid shell From a6318b159b0c2db5405aa525882e208aac240ead Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 16:00:24 +0100 Subject: [PATCH 08/23] substituteRecursiveTags optimization + moved in helpers facilities (because currently used commonly in server and in client) --- fail2ban/client/actionreader.py | 4 +- fail2ban/client/configreader.py | 10 +++- fail2ban/helpers.py | 99 ++++++++++++++++++++++++++++++++ fail2ban/server/action.py | 98 +------------------------------ fail2ban/tests/actiontestcase.py | 40 ++++++------- 5 files changed, 131 insertions(+), 120 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 55ceda93..0fd55f41 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -28,6 +28,7 @@ import os from .configreader import DefinitionInitConfigReader from ..helpers import getLogger +from ..server.action import CommandAction # Gets the instance of the logger. logSys = getLogger(__name__) @@ -69,7 +70,8 @@ class ActionReader(DefinitionInitConfigReader): return self._name def convert(self): - opts = self.getCombined(ignore=('timeout', 'bantime')) + opts = self.getCombined( + ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) # type-convert only after combined (otherwise boolean converting prevents substitution): if opts.get('norestored'): opts['norestored'] = self._convert_to_boolean(opts['norestored']) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 8098433e..2caa97ce 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -29,8 +29,7 @@ import os from ConfigParser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel -from ..helpers import getLogger -from ..server.action import CommandAction +from ..helpers import getLogger, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) @@ -328,6 +327,11 @@ class DefinitionInitConfigReader(ConfigReader): return value.lower() in ("1", "yes", "true", "on") def getCombOption(self, optname): + """Get combined definition option (as string) using pre-set and init + options as preselection (values with higher precedence as specified in section). + + Can be used only after calling of getOptions. + """ try: return self._defCache[optname] except KeyError: @@ -352,7 +356,7 @@ class DefinitionInitConfigReader(ConfigReader): n, cond = cond.groups() ignore.add(n) # substiture options already specified direct: - opts = CommandAction.substituteRecursiveTags(combinedopts, + opts = substituteRecursiveTags(combinedopts, ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 4f4426b0..1e5a053b 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -200,6 +200,105 @@ else: raise return uni_decode(x, enc, 'replace') + +# +# Following facilities used for safe recursive interpolation of +# tags () in tagged options. +# + +# max tag replacement count: +MAX_TAG_REPLACE_COUNT = 10 + +# compiled RE for tag name (replacement name) +TAG_CRE = re.compile(r'<([^ <>]+)>') + +def substituteRecursiveTags(inptags, conditional='', + ignore=(), addrepl=None +): + """Sort out tag definitions within other tags. + Since v.0.9.2 supports embedded interpolation (see test cases for examples). + + so: becomes: + a = 3 a = 3 + b = _3 b = 3_3 + + Parameters + ---------- + inptags : dict + Dictionary of tags(keys) and their values. + + Returns + ------- + dict + Dictionary of tags(keys) and their values, with tags + within the values recursively replaced. + """ + # copy return tags dict to prevent modifying of inptags: + tags = inptags.copy() + t = TAG_CRE + ignore = set(ignore) + done = set() + # repeat substitution while embedded-recursive (repFlag is True) + while True: + repFlag = False + # substitute each value: + for tag in tags.iterkeys(): + # ignore escaped or already done (or in ignore list): + if tag in ignore or tag in done: continue + value = orgval = str(tags[tag]) + # search and replace all tags within value, that can be interpolated using other tags: + m = t.search(value) + refCounts = {} + #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) + while m: + found_tag = m.group(1) + # don't replace tags that should be currently ignored (pre-replacement): + if found_tag in ignore: + m = t.search(value, m.end()) + continue + #logSys.log(5, 'found: %s' % found_tag) + if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT: + # recursive definitions are bad + #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) + raise ValueError( + "properties contain self referencing definitions " + "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % + (tag, found_tag, refCounts, value)) + repl = None + if conditional: + repl = tags.get(found_tag + '?' + conditional) + if repl is None: + repl = tags.get(found_tag) + # try to find tag using additional replacement (callable): + if repl is None and addrepl is not None: + repl = addrepl(found_tag) + if repl is None: + # Missing tags - just continue on searching after end of match + # Missing tags are ok - cInfo can contain aInfo elements like and valid shell + # constructs like . + m = t.search(value, m.end()) + continue + value = value.replace('<%s>' % found_tag, repl) + #logSys.log(5, 'value now: %s' % value) + # increment reference count: + refCounts[found_tag] = refCounts.get(found_tag, 0) + 1 + # the next match for replace: + m = t.search(value, m.start()) + #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) + # was substituted? + if orgval != value: + # check still contains any tag - should be repeated (possible embedded-recursive substitution): + if t.search(value): + repFlag = True + tags[tag] = value + # no more sub tags (and no possible composite), add this tag to done set (just to be faster): + if '<' not in value: done.add(tag) + # stop interpolation, if no replacements anymore: + if not repFlag: + break + return tags + + class BgService(object): """Background servicing diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d14c9aab..2f7f423c 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -23,7 +23,6 @@ __license__ = "GPL" import logging import os -import re import signal import subprocess import tempfile @@ -35,7 +34,7 @@ from collections import MutableMapping from .ipdns import asip from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger +from ..helpers import getLogger, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) @@ -46,12 +45,6 @@ _cmd_lock = threading.Lock() # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`: allowed_ipv6 = True -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 - -# compiled RE for tag name (replacement name) -TAG_CRE = re.compile(r'<([^ <>]+)>') - class CallingMap(MutableMapping): """A Mapping type which returns the result of callable values. @@ -364,93 +357,6 @@ class CommandAction(ActionBase): """ return self._executeOperation('', 'reloading') - @classmethod - def substituteRecursiveTags(cls, inptags, conditional='', - ignore=(), addrepl=None - ): - """Sort out tag definitions within other tags. - Since v.0.9.2 supports embedded interpolation (see test cases for examples). - - so: becomes: - a = 3 a = 3 - b = _3 b = 3_3 - - Parameters - ---------- - inptags : dict - Dictionary of tags(keys) and their values. - - Returns - ------- - dict - Dictionary of tags(keys) and their values, with tags - within the values recursively replaced. - """ - # copy return tags dict to prevent modifying of inptags: - tags = inptags.copy() - t = TAG_CRE - ignore = set(ignore) - done = cls._escapedTags.copy() | ignore - # repeat substitution while embedded-recursive (repFlag is True) - while True: - repFlag = False - # substitute each value: - for tag in tags.iterkeys(): - # ignore escaped or already done (or in ignore list): - if tag in done: continue - value = orgval = str(tags[tag]) - # search and replace all tags within value, that can be interpolated using other tags: - m = t.search(value) - refCounts = {} - #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) - while m: - found_tag = m.group(1) - # don't replace tags that should be currently ignored (pre-replacement): - if found_tag in ignore: - m = t.search(value, m.end()) - continue - #logSys.log(5, 'found: %s' % found_tag) - if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT: - # recursive definitions are bad - #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) - raise ValueError( - "properties contain self referencing definitions " - "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, found_tag, refCounts, value)) - repl = None - if found_tag not in cls._escapedTags: - if conditional: - repl = tags.get(found_tag + '?' + conditional) - if repl is None: - repl = tags.get(found_tag) - if repl is None and addrepl is not None: - repl = addrepl(found_tag) - if repl is None: - # Escaped or missing tags - just continue on searching after end of match - # Missing tags are ok - cInfo can contain aInfo elements like and valid shell - # constructs like . - m = t.search(value, m.end()) - continue - value = value.replace('<%s>' % found_tag, repl) - #logSys.log(5, 'value now: %s' % value) - # increment reference count: - refCounts[found_tag] = refCounts.get(found_tag, 0) + 1 - # the next match for replace: - m = t.search(value, m.start()) - #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) - # was substituted? - if orgval != value: - # check still contains any tag - should be repeated (possible embedded-recursive substitution): - if t.search(value): - repFlag = True - tags[tag] = value - # no more sub tags (and no possible composite), add this tag to done set (just to be faster): - if '<' not in value: done.add(tag) - # stop interpolation, if no replacements anymore: - if not repFlag: - break - return tags - @staticmethod def escapeTag(value): """Escape characters which may be used for command injection. @@ -501,7 +407,7 @@ class CommandAction(ActionBase): return string # replace: string = query - aInfo = cls.substituteRecursiveTags(aInfo, conditional) + aInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) for tag in aInfo: if "<%s>" % tag in query: value = aInfo.get(tag + '?' + conditional) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 403f7ea6..8a74201c 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -29,7 +29,7 @@ import tempfile import time import unittest -from ..server.action import CommandAction, CallingMap +from ..server.action import CommandAction, CallingMap, substituteRecursiveTags from ..server.actions import OrderedDict from ..server.utils import Utils @@ -56,30 +56,30 @@ class CommandActionTest(LogCaptureTestCase): } # Recursion is bad self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': ''})) + lambda: substituteRecursiveTags({'A': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': '', 'B': ''})) + lambda: substituteRecursiveTags({'A': '', 'B': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) + lambda: substituteRecursiveTags({'A': '', 'B': '', 'C': ''})) # Unresolveable substition self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'A': 'to= fromip=', 'C': '', 'B': '', 'D': ''})) + lambda: substituteRecursiveTags({'A': 'to= fromip=', 'C': '', 'B': '', 'D': ''})) self.assertRaises(ValueError, - lambda: CommandAction.substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) + lambda: substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) # We need here an ordered, because the sequence of iteration is very important for this test if OrderedDict: # No cyclic recursion, just multiple replacement of tag , should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict( + self.assertEqual(substituteRecursiveTags( OrderedDict( (('X', 'x=x'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) ), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'} ) # No cyclic recursion, just multiple replacement of tag in composite tags, should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict( + self.assertEqual(substituteRecursiveTags( OrderedDict( (('X', 'x=x <> <>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) ), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'} ) # No cyclic recursion, just multiple replacement of same tags, should be successful: - self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertEqual(substituteRecursiveTags( OrderedDict(( ('actionstart', 'ipset create hash:ip timeout family \n -I '), ('ipmset', 'f2b-'), ('name', 'any'), @@ -111,42 +111,42 @@ class CommandActionTest(LogCaptureTestCase): )) ) # Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle: - self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( ('A', '<>'), ('B', 'D'), ('C', 'E'), ('DE', 'cycle '), )) )) - self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags( OrderedDict(( + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( ('DE', 'cycle '), ('A', '<>'), ('B', 'D'), ('C', 'E'), )) )) # missing tags are ok - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ''}), {'A': ''}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(substituteRecursiveTags({'A': ''}), {'A': ''}) + self.assertEqual(substituteRecursiveTags({'A': ' ','X':'fun'}), {'A': ' fun', 'X':'fun'}) + self.assertEqual(substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Escaped tags should be ignored - self.assertEqual(CommandAction.substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) + self.assertEqual(substituteRecursiveTags({'A': ' ', 'B': 'cool'}), {'A': ' cool', 'B': 'cool'}) # Multiple stuff on same line is ok - self.assertEqual(CommandAction.substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), + self.assertEqual(substituteRecursiveTags({'failregex': 'to= fromip= evilperson=', 'honeypot': 'pokie', 'ignoreregex': ''}), { 'failregex': "to=pokie fromip= evilperson=pokie", 'honeypot': 'pokie', 'ignoreregex': '', }) # rest is just cool - self.assertEqual(CommandAction.substituteRecursiveTags(aInfo), + self.assertEqual(substituteRecursiveTags(aInfo), { 'HOST': "192.0.2.0", 'ABC': '123 192.0.2.0', 'xyz': '890 123 192.0.2.0', }) # obscure embedded case - self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), + self.assertEqual(substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4'}), {'A': '', 'PREF': 'IPV4'}) - self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), + self.assertEqual(substituteRecursiveTags({'A': '<HOST>', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}), {'A': '1.2.3.4', 'PREF': 'IPV4', 'IPV4HOST': '1.2.3.4'}) # more embedded within a string and two interpolations - self.assertEqual(CommandAction.substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), + self.assertEqual(substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}) def testReplaceTag(self): From 9ebf70cd6a90143342733e4e8af85be3455b2795 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 18:25:34 +0100 Subject: [PATCH 09/23] Safer, more stable and faster replaceTag interpolation (switched from cycle over all tags to re.sub with callable) --- fail2ban/server/action.py | 86 +++++++++++++++++++++++--------- fail2ban/tests/actiontestcase.py | 6 +-- 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 2f7f423c..5acc98ad 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -34,7 +34,7 @@ from collections import MutableMapping from .ipdns import asip from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger, substituteRecursiveTags +from ..helpers import getLogger, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT # Gets the instance of the logger. logSys = getLogger(__name__) @@ -399,33 +399,71 @@ class CommandAction(ActionBase): str `query` string with tags replaced. """ + if '<' not in query: return query + # use cache if allowed: if cache is not None: ckey = (query, conditional) - string = cache.get(ckey) - if string is not None: - return string - # replace: - string = query - aInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) - for tag in aInfo: - if "<%s>" % tag in query: - value = aInfo.get(tag + '?' + conditional) - if value is None: - value = aInfo.get(tag) - value = str(value) # assure string - if tag in cls._escapedTags: - # That one needs to be escaped since its content is - # out of our control - value = cls.escapeTag(value) - string = string.replace('<' + tag + '>', value) - # New line, space - string = reduce(lambda s, kv: s.replace(*kv), (("
", '\n'), ("", " ")), string) - # cache if properties: + value = cache.get(ckey) + if value is not None: + return value + + # first try get cached tags dictionary: + subInfo = csubkey = None if cache is not None: - cache[ckey] = string + csubkey = ('subst-tags', id(aInfo), conditional) + subInfo = cache.get(csubkey) + # interpolation of dictionary: + if subInfo is None: + subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) + # New line, space + for (tag, value) in (("br", '\n'), ("sp", " ")): + if subInfo.get(tag) is None: subInfo[tag] = value + # cache if possible: + if csubkey is not None: + cache[csubkey] = subInfo + + # substitution callable, used by interpolation of each tag + repeatSubst = {} + def substVal(m): + tag = m.group(1) # tagname from match + value = None + if conditional: + value = subInfo.get(tag + '?' + conditional) + if value is None: + value = subInfo.get(tag) + if value is None: + return m.group() # fallback (no replacement) + value = str(value) # assure string + if tag in cls._escapedTags: + # That one needs to be escaped since its content is + # out of our control + value = cls.escapeTag(value) + # possible contains tags: + if '<' in value: + repeatSubst[1] = True + return value + + # interpolation of query: + count = MAX_TAG_REPLACE_COUNT + 1 + while True: + repeatSubst = {} + value = TAG_CRE.sub(substVal, query) + # possible recursion ? + if not repeatSubst or value == query: break + query = value + count -= 1 + if count <= 0: # pragma: no cover - almost impossible (because resolved above) + raise ValueError( + "unexpected too long replacement interpolation, " + "possible self referencing definitions in query: %s" % (query,)) + + + # cache if possible: + if cache is not None: + cache[ckey] = value # - return string + return value def _processCmd(self, cmd, aInfo=None, conditional=''): """Executes a command with preliminary checks and substitutions. @@ -491,7 +529,7 @@ class CommandAction(ActionBase): realCmd = self.replaceTag(cmd, self._properties, conditional=conditional, cache=self.__substCache) - # Replace tags + # Replace dynamical tags (don't use cache here) if aInfo is not None: realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional) else: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 8a74201c..38f576b1 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -217,10 +217,10 @@ class CommandActionTest(LogCaptureTestCase): self.__action.replaceTag(" ''", self.__action._properties, conditional="family=inet6", cache=cache), "Text 890-567 text 567 '567'") - self.assertEqual(len(cache) if cache is not None else -1, 3) + self.assertTrue(len(cache) >= 3) # set one parameter - internal properties and cache should be reseted: setattr(self.__action, 'xyz', "000-") - self.assertEqual(len(cache) if cache is not None else -1, 0) + self.assertEqual(len(cache), 0) # test againg, should have 000 instead of 890: for i in range(2): self.assertEqual( @@ -235,7 +235,7 @@ class CommandActionTest(LogCaptureTestCase): self.__action.replaceTag(" ''", self.__action._properties, conditional="family=inet6", cache=cache), "Text 000-567 text 567 '567'") - self.assertEqual(len(cache), 3) + self.assertTrue(len(cache) >= 3) def testExecuteActionBan(self): From a8c0cec4ac1c91b821d3fc80e0aac91a7b12902a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 21:00:48 +0100 Subject: [PATCH 10/23] small amend with several fixes and test coverage --- fail2ban/helpers.py | 35 +++++++++++++++++--------------- fail2ban/server/action.py | 30 +++++++++++++++++++-------- fail2ban/tests/actiontestcase.py | 32 +++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 1e5a053b..2407df1f 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -233,9 +233,11 @@ def substituteRecursiveTags(inptags, conditional='', Dictionary of tags(keys) and their values, with tags within the values recursively replaced. """ + #logSys = getLogger("fail2ban") + tre_search = TAG_CRE.search # copy return tags dict to prevent modifying of inptags: tags = inptags.copy() - t = TAG_CRE + # init: ignore = set(ignore) done = set() # repeat substitution while embedded-recursive (repFlag is True) @@ -247,48 +249,49 @@ def substituteRecursiveTags(inptags, conditional='', if tag in ignore or tag in done: continue value = orgval = str(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: - m = t.search(value) + m = tre_search(value) refCounts = {} #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: - found_tag = m.group(1) + # found replacement tag: + rtag = m.group(1) # don't replace tags that should be currently ignored (pre-replacement): - if found_tag in ignore: - m = t.search(value, m.end()) + if rtag in ignore: + m = tre_search(value, m.end()) continue - #logSys.log(5, 'found: %s' % found_tag) - if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT: + #logSys.log(5, 'found: %s' % rtag) + if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, found_tag, refCounts, value)) + (tag, rtag, refCounts, value)) repl = None if conditional: - repl = tags.get(found_tag + '?' + conditional) + repl = tags.get(rtag + '?' + conditional) if repl is None: - repl = tags.get(found_tag) + repl = tags.get(rtag) # try to find tag using additional replacement (callable): if repl is None and addrepl is not None: - repl = addrepl(found_tag) + repl = addrepl(rtag) if repl is None: # Missing tags - just continue on searching after end of match # Missing tags are ok - cInfo can contain aInfo elements like and valid shell # constructs like . - m = t.search(value, m.end()) + m = tre_search(value, m.end()) continue - value = value.replace('<%s>' % found_tag, repl) + value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[found_tag] = refCounts.get(found_tag, 0) + 1 + refCounts[rtag] = refCounts.get(rtag, 0) + 1 # the next match for replace: - m = t.search(value, m.start()) + m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) # was substituted? if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): - if t.search(value): + if tre_search(value): repFlag = True tags[tag] = value # no more sub tags (and no possible composite), add this tag to done set (just to be faster): diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 5acc98ad..b4616a15 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -252,6 +252,16 @@ class CommandAction(ActionBase): # set: self.__dict__[name] = value + def __delattr__(self, name): + if not name.startswith('_'): + # parameters changed - clear properties and substitution cache: + self.__properties = None + self.__substCache.clear() + #self._logSys.debug("Unset action %r %s", self._name, name) + self._logSys.debug(" Unset %s", name) + # del: + del self.__dict__[name] + @property def _properties(self): """A dictionary of the actions properties. @@ -404,15 +414,19 @@ class CommandAction(ActionBase): # use cache if allowed: if cache is not None: ckey = (query, conditional) - value = cache.get(ckey) - if value is not None: - return value + try: + return cache[ckey] + except KeyError: + pass # first try get cached tags dictionary: subInfo = csubkey = None if cache is not None: csubkey = ('subst-tags', id(aInfo), conditional) - subInfo = cache.get(csubkey) + try: + subInfo = cache[csubkey] + except KeyError: + pass # interpolation of dictionary: if subInfo is None: subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) @@ -424,7 +438,7 @@ class CommandAction(ActionBase): cache[csubkey] = subInfo # substitution callable, used by interpolation of each tag - repeatSubst = {} + repeatSubst = {0: 0} def substVal(m): tag = m.group(1) # tagname from match value = None @@ -441,19 +455,19 @@ class CommandAction(ActionBase): value = cls.escapeTag(value) # possible contains tags: if '<' in value: - repeatSubst[1] = True + repeatSubst[0] = 1 return value # interpolation of query: count = MAX_TAG_REPLACE_COUNT + 1 while True: - repeatSubst = {} + repeatSubst[0] = 0 value = TAG_CRE.sub(substVal, query) # possible recursion ? if not repeatSubst or value == query: break query = value count -= 1 - if count <= 0: # pragma: no cover - almost impossible (because resolved above) + if count <= 0: raise ValueError( "unexpected too long replacement interpolation, " "possible self referencing definitions in query: %s" % (query,)) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 38f576b1..435dc1da 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -40,12 +40,20 @@ class CommandActionTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" - self.__action = CommandAction(None, "Test") LogCaptureTestCase.setUp(self) + self.__action = CommandAction(None, "Test") + # prevent execute stop if start fails (or event not started at all): + self.__action_started = False + orgstart = self.__action.start + def _action_start(): + self.__action_started = True + return orgstart() + self.__action.start = _action_start def tearDown(self): """Call after every test case.""" - self.__action.stop() + if self.__action_started: + self.__action.stop() LogCaptureTestCase.tearDown(self) def testSubstituteRecursiveTags(self): @@ -196,6 +204,26 @@ class CommandActionTest(LogCaptureTestCase): self.__action.replaceTag("abc", CallingMap(matches=lambda: int("a"))), "abc") + def testReplaceTagSelfRecursion(self): + setattr(self.__action, 'a', "") + setattr(self.__action, 'b?family=inet6', "b>") + setattr(self.__action, 'ac', "") + setattr(self.__action, 'ab', "") + setattr(self.__action, 'x?family=inet6', "") + # produce self-referencing properties except: + self.assertRaisesRegexp(ValueError, r"properties contain self referencing definitions", + lambda: self.__action.replaceTag("", + self.__action._properties, conditional="family=inet4") + ) + # remore self-referencing in props: + delattr(self.__action, 'ac') + # produce self-referencing query except: + self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", + lambda: self.__action.replaceTag(">>>>>>>>>>>>>>>>>>>>", + self.__action._properties, conditional="family=inet6") + ) + def testReplaceTagConditionalCached(self): setattr(self.__action, 'abc', "123") setattr(self.__action, 'abc?family=inet4', "345") From 6d878f3a43854ac30fe6759208cd6b0c7e1c30af Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 16 Feb 2017 22:50:36 +0100 Subject: [PATCH 11/23] try to provide filter captures (already in ticket data) to the actions as interpolation options (closes gh-1110) --- fail2ban/server/actions.py | 4 +- fail2ban/server/failregex.py | 126 ++++++++++++++++++++++++++--------- 2 files changed, 98 insertions(+), 32 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index e2046e15..3a5eada9 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -350,7 +350,9 @@ class Actions(JailThread, Mapping): "time" : bTicket.getTime(), "matches" : "\n".join(bTicket.getMatches()), # to bypass actions, that should not be executed for restored tickets - "restored": (1 if ticket.restored else 0) + "restored": (1 if ticket.restored else 0), + # extra-interpolation - all match-tags (captured from the filter): + "F-*": lambda tag=None: bTicket.getData(tag) }) if self._jail.database is not None: mi4ip = lambda overalljails=False, self=self, \ diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 7c51ddb8..096cb57a 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -27,6 +27,62 @@ import sys from .ipdns import IPAddr + +FTAG_CRE = re.compile(r'') + +FCUSTAG_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only + +R_HOST = [ + # separated ipv4: + r"""(?:::f{4,6}:)?(?P%s)""" % (IPAddr.IP_4_RE,), + # separated ipv6: + r"""(?P%s)""" % (IPAddr.IP_6_RE,), + # place-holder for ipv6 enclosed in optional [] (used in addr-, host-regex) + "", + # separated dns: + r"""(?P[\w\-.^_]*\w)""", + # place-holder for ADDR tag-replacement (joined): + "", + # place-holder for HOST tag replacement (joined): + "" +] +RI_IPV4 = 0 +RI_IPV6 = 1 +RI_IPV6BR = 2 +RI_DNS = 3 +RI_ADDR = 4 +RI_HOST = 5 + +R_HOST[RI_IPV6BR] = r"""\[?%s\]?""" % (R_HOST[RI_IPV6],) +R_HOST[RI_ADDR] = "(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR])),) +R_HOST[RI_HOST] = "(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR], R_HOST[RI_DNS])),) + +RH4TAG = { + # separated ipv4 (self closed, closed): + "IP4": R_HOST[RI_IPV4], + "F-IP4/": R_HOST[RI_IPV4], + # separated ipv6 (self closed, closed): + "IP6": R_HOST[RI_IPV6], + "F-IP6/": R_HOST[RI_IPV6], + # 2 address groups instead of - in opposition to ``, + # for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together + "ADDR": R_HOST[RI_ADDR], + "F-ADDR/": R_HOST[RI_ADDR], + # separated dns (self closed, closed): + "DNS": R_HOST[RI_DNS], + "F-DNS/": R_HOST[RI_DNS], + # default failure-id as no space tag: + "F-ID/": r"""(?P\S+)""", + # default failure port, like 80 or http : + "F-PORT/": r"""(?P\w+)""", +} + +# default failure groups map for customizable expressions (with different group-id): +R_MAP = { + "ID": "fid", + "PORT": "fport", +} + ## # Regular expression class. # @@ -71,38 +127,46 @@ class Regex: @staticmethod def _resolveHostTag(regex, useDns="yes"): - # separated ipv4: - r_host = [] - r = r"""(?:::f{4,6}:)?(?P%s)""" % (IPAddr.IP_4_RE,) - regex = regex.replace("", r); # self closed - regex = regex.replace("", r); # closed - r_host.append(r) - # separated ipv6: - r = r"""(?P%s)""" % (IPAddr.IP_6_RE,) - regex = regex.replace("", r); # self closed - regex = regex.replace("", r); # closed - r_host.append(r"""\[?%s\]?""" % (r,)); # enclose ipv6 in optional [] in host-regex - # 2 address groups instead of - in opposition to ``, - # for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together - regex = regex.replace("", "(?:%s)" % ("|".join(r_host),)) - # separated dns: - r = r"""(?P[\w\-.^_]*\w)""" - regex = regex.replace("", r); # self closed - regex = regex.replace("", r); # closed - if useDns not in ("no",): - r_host.append(r) - # 3 groups instead of - separated ipv4, ipv6 and host (dns) - regex = regex.replace("", "(?:%s)" % ("|".join(r_host),)) - # default failure-id as no space tag: - regex = regex.replace("", r"""(?P\S+)"""); # closed - # default failure port, like 80 or http : - regex = regex.replace("", r"""(?P\w+)"""); # closed - # default failure groups (begin / end tag) for customizable expressions: - for o,r in (('IP4', 'ip4'), ('IP6', 'ip6'), ('DNS', 'dns'), ('ID', 'fid'), ('PORT', 'fport')): - regex = regex.replace("" % o, "(?P<%s>" % r); # open tag - regex = regex.replace("" % o, ")"); # close tag - return regex + openTags = dict() + # tag interpolation callable: + def substTag(m): + tag = m.group() + tn = tag[1:-1] + # 3 groups instead of - separated ipv4, ipv6 and host (dns) + if tn == "HOST": + return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR] + # static replacement from RH4TAG: + try: + return RH4TAG[tn] + except KeyError: + pass + + # (begin / end tag) for customizable expressions, additionally used as + # user custom tags (match will be stored in ticket data, can be used in actions): + m = FCUSTAG_CRE.match(tn) + if m: # match F-... + m = m.groups() + tn = m[1] + # close tag: + if m[0]: + # check it was already open: + if openTags.get(tn): + return ")" + return tag; # tag not opened, use original + # open tag: + openTags[tn] = 1 + try: # if should be mapped: + tn = R_MAP[tn] + except KeyError: + tn = tn.lower() + return "(?P<%s>" % (tn,) + + # original, no replacement: + return tag + + # substitute tags: + return FTAG_CRE.sub(substTag, regex) ## # Gets the regular expression. From 61c8cd11b8f4b553ab014b5581bcb11ddb451e57 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 18 Feb 2017 00:08:35 +0100 Subject: [PATCH 12/23] Exposes filter group captures in actions (non-recursive interpolation of tags ``); Closes gh-1110 --- fail2ban/server/action.py | 24 +++++++++++++++++++++--- fail2ban/server/failregex.py | 16 ++++++++++------ fail2ban/server/ticket.py | 4 +++- fail2ban/tests/actiontestcase.py | 19 +++++++++++++++---- fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++--------- 5 files changed, 59 insertions(+), 23 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index b4616a15..bf291690 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -23,6 +23,7 @@ __license__ = "GPL" import logging import os +import re import signal import subprocess import tempfile @@ -31,6 +32,7 @@ import time from abc import ABCMeta from collections import MutableMapping +from .failregex import mapTag2Opt from .ipdns import asip from .mytime import MyTime from .utils import Utils @@ -45,6 +47,10 @@ _cmd_lock = threading.Lock() # Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`: allowed_ipv6 = True +# capture groups from filter for map to ticket data: +FCUSTAG_CRE = re.compile(r''); # currently uppercase only + + class CallingMap(MutableMapping): """A Mapping type which returns the result of callable values. @@ -72,9 +78,9 @@ class CallingMap(MutableMapping): def __getitem__(self, key): value = self.data[key] if callable(value): - return value() - else: - return value + value = value() + self.data[key] = value + return value def __setitem__(self, key, value): self.data[key] = value @@ -546,6 +552,18 @@ class CommandAction(ActionBase): # Replace dynamical tags (don't use cache here) if aInfo is not None: realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional) + # Replace ticket options (filter capture groups) non-recursive: + if '<' in realCmd: + tickData = aInfo.get("F-*") + if not tickData: tickData = {} + def substTag(m): + tn = mapTag2Opt(m.groups()[0]) + try: + return str(tickData[tn]) + except KeyError: + return "" + + realCmd = FCUSTAG_CRE.sub(substTag, realCmd) else: realCmd = cmd diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 096cb57a..cbd0cef0 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -30,7 +30,7 @@ from .ipdns import IPAddr FTAG_CRE = re.compile(r'') -FCUSTAG_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only +FCUSTNAME_CRE = re.compile(r'^(/?)F-([A-Z0-9_\-]+)$'); # currently uppercase only R_HOST = [ # separated ipv4: @@ -83,6 +83,12 @@ R_MAP = { "PORT": "fport", } +def mapTag2Opt(tag): + try: # if should be mapped: + return R_MAP[tag] + except KeyError: + return tag.lower() + ## # Regular expression class. # @@ -144,7 +150,7 @@ class Regex: # (begin / end tag) for customizable expressions, additionally used as # user custom tags (match will be stored in ticket data, can be used in actions): - m = FCUSTAG_CRE.match(tn) + m = FCUSTNAME_CRE.match(tn) if m: # match F-... m = m.groups() tn = m[1] @@ -156,10 +162,8 @@ class Regex: return tag; # tag not opened, use original # open tag: openTags[tn] = 1 - try: # if should be mapped: - tn = R_MAP[tn] - except KeyError: - tn = tn.lower() + # if should be mapped: + tn = mapTag2Opt(tn) return "(?P<%s>" % (tn,) # original, no replacement: diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index c7bb4d47..a66b64ac 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -56,7 +56,9 @@ class Ticket(object): self._time = time if time is not None else MyTime.time() self._data = {'matches': matches or [], 'failures': 0} if data is not None: - self._data.update(data) + for k,v in data.iteritems(): + if v is not None: + self._data[k] = v if ticket: # ticket available - copy whole information from ticket: self.__dict__.update(i for i in ticket.__dict__.iteritems() if i[0] in self.__dict__) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 435dc1da..ca908b65 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -329,13 +329,24 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(self.__action.ROST,"192.0.2.0") def testExecuteActionUnbanAinfo(self): - aInfo = { + aInfo = CallingMap({ 'ABC': "123", - } - self.__action.actionban = "touch /tmp/fail2ban.test.123" - self.__action.actionunban = "rm /tmp/fail2ban.test." + 'ip': '192.0.2.1', + 'F-*': lambda: { + 'fid': 111, + 'fport': 222, + 'user': "tester" + } + }) + self.__action.actionban = "touch /tmp/fail2ban.test.123; echo 'failure of -- from :'" + self.__action.actionunban = "rm /tmp/fail2ban.test.; echo 'user unbanned'" self.__action.ban(aInfo) self.__action.unban(aInfo) + self.assertLogged( + " -- stdout: 'failure 111 of tester -- from 192.0.2.1:222'", + " -- stdout: 'user tester unbanned'", + all=True + ) def testExecuteActionStartEmpty(self): self.__action.actionstart = "" diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index b8417be5..adaf4719 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -761,9 +761,10 @@ class Fail2banServerTest(Fail2banClientServerBase): "[Definition]", "norestored = %(_exec_once)s", "restore = ", + "info = ", "actionstart = echo '[%(name)s] %(actname)s: ** start'", start, "actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload, - "actionban = echo '[%(name)s] %(actname)s: ++ ban %(restore)s'", ban, + "actionban = echo '[%(name)s] %(actname)s: ++ ban %(restore)s%(info)s'", ban, "actionunban = echo '[%(name)s] %(actname)s: -- unban '", unban, "actionstop = echo '[%(name)s] %(actname)s: __ stop'", stop, ) @@ -777,28 +778,28 @@ class Fail2banServerTest(Fail2banClientServerBase): "usedns = no", "maxretry = 3", "findtime = 10m", - "failregex = ^\s*failure (401|403) from ", + "failregex = ^\s*failure 401|403 from ", "datepattern = {^LN-BEG}EPOCH", "", "[test-jail1]", "backend = " + backend, "filter =", "action = ", " test-action1[name='%(__name__)s']" \ if 1 in actions else "", - " test-action2[name='%(__name__)s', restore='restored: ']" \ + " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" \ if 3 in actions else "", "logpath = " + test1log, " " + test2log if 2 in enabled else "", " " + test3log if 2 in enabled else "", - "failregex = ^\s*failure (401|403) from ", - " ^\s*error (401|403) from " \ + "failregex = ^\s*failure 401|403 from ", + " ^\s*error 401|403 from " \ if 2 in enabled else "", "enabled = true" if 1 in enabled else "", "", "[test-jail2]", "backend = " + backend, "filter =", "action = ", - " test-action2[name='%(__name__)s', restore='restored: ']" \ + " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" \ if 3 in actions else "", @@ -837,7 +838,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "stdout: '[test-jail1] test-action2: ** start'", all=True) # test restored is 0 (both actions available): self.assertLogged( - "stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'", + "stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0, err-code: 401'", "stdout: '[test-jail1] test-action3: ++ ban 192.0.2.1 restored: 0'", all=True, wait=MID_WAITTIME) @@ -958,8 +959,8 @@ class Fail2banServerTest(Fail2banClientServerBase): ) # test restored is 1 (only test-action2): self.assertLogged( - "stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'", - "stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'", + "stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1, err-code: 401'", + "stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1, err-code: 401'", all=True, wait=MID_WAITTIME) # test test-action3 not executed at all (norestored check): self.assertNotLogged( From fe06ffca71e9054b21b93237c40c0c53478a19df Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Feb 2017 11:45:03 +0100 Subject: [PATCH 13/23] Fix retrieving of IPv6 address with dnsToIp on some systems (default returns AF_INET family only), fix network test-cases. --- fail2ban/server/ipdns.py | 31 ++++++++++++++++++++----------- fail2ban/tests/filtertestcase.py | 5 +++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index a5625629..757cceba 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -64,16 +64,19 @@ class DNSUtils: if ips is not None: return ips # retrieve ips - try: - ips = list() - for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP): - ip = IPAddr(result[4][0]) - if ip.isValid: - ips.append(ip) - except socket.error as e: - # todo: make configurable the expired time of cache entry: - logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e) - ips = list() + ips = list() + saveerr = None + for fam, ipfam in ((socket.AF_INET, IPAddr.FAM_IPv4), (socket.AF_INET6, IPAddr.FAM_IPv6)): + try: + for result in socket.getaddrinfo(dns, None, fam, 0, socket.IPPROTO_TCP): + ip = IPAddr(result[4][0], ipfam) + if ip.isValid: + ips.append(ip) + except socket.error as e: + saveerr = e + if not ips and saveerr: + logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, saveerr) + DNSUtils.CACHE_nameToIp.set(dns, ips) return ips @@ -140,6 +143,8 @@ class IPAddr(object): CIDR_RAW = -2 CIDR_UNSPEC = -1 + FAM_IPv4 = CIDR_RAW - socket.AF_INET + FAM_IPv6 = CIDR_RAW - socket.AF_INET6 def __new__(cls, ipstr, cidr=CIDR_UNSPEC): # check already cached as IPAddr @@ -191,7 +196,11 @@ class IPAddr(object): self._raw = ipstr # if not raw - recognize family, set addr, etc.: if cidr != IPAddr.CIDR_RAW: - for family in [socket.AF_INET, socket.AF_INET6]: + if cidr is not None and cidr < IPAddr.CIDR_RAW: + family = [IPAddr.CIDR_RAW - cidr] + else: + family = [socket.AF_INET, socket.AF_INET6] + for family in family: try: binary = socket.inet_pton(family, ipstr) self._family = family diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 986cf1f0..174152b9 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -337,6 +337,11 @@ class IgnoreIP(LogCaptureTestCase): for ip in ipList: self.filter.addIgnoreIP(ip) self.assertFalse(self.filter.inIgnoreIPList(ip)) + if not unittest.F2B.no_network: # pragma: no cover + self.assertLogged( + 'Unable to find a corresponding IP address for 999.999.999.999', + 'Unable to find a corresponding IP address for abcdef.abcdef', + 'Unable to find a corresponding IP address for 192.168.0.', all=True) def testIgnoreIPCIDR(self): self.filter.addIgnoreIP('192.168.1.0/25') From 9d15a792a5f0fd818b2723733071bff4e3ab6d90 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Feb 2017 16:46:02 +0100 Subject: [PATCH 14/23] amend to fe06ffca71e9054b21b93237c40c0c53478a19df: small optimization using already known IP family --- fail2ban/server/filter.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 0540dedc..b7c78b9b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -564,7 +564,7 @@ class Filter(JailThread): dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) if dateTimeMatch is None: - logSys.error("findFailure failed to parse timeText: " + timeText) + logSys.error("findFailure failed to parse timeText: %s", timeText) date = self.__lastDate else: @@ -621,10 +621,16 @@ class Filter(JailThread): # failure-id: fid = fail.get('fid') # ip-address or host: - host = fail.get('ip4') or fail.get('ip6') + host = fail.get('ip4') if host is not None: + cidr = IPAddr.FAM_IPv4 raw = True else: + host = fail.get('ip6') + if host is not None: + cidr = IPAddr.FAM_IPv6 + raw = True + if host is None: host = fail.get('dns') if host is None: # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: @@ -639,18 +645,14 @@ class Filter(JailThread): # check host equal failure-id, if not - failure with complex id: if fid is not None and fid != host: ip = IPAddr(fid, IPAddr.CIDR_RAW) - failList.append([failRegexIndex, ip, date, - failRegex.getMatchedLines(), fail]) - if not self.checkAllRegex: - break + ips = [ip] else: ips = DNSUtils.textToIp(host, self.__useDns) - if ips: - for ip in ips: - failList.append([failRegexIndex, ip, date, - failRegex.getMatchedLines(), fail]) - if not self.checkAllRegex: - break + for ip in ips: + failList.append([failRegexIndex, ip, date, + failRegex.getMatchedLines(), fail]) + if not self.checkAllRegex: + break except RegexException as e: # pragma: no cover - unsure if reachable logSys.error(e) return failList From 4ff8d051f49808ac769709c5aff8591fcd79040a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 20 Feb 2017 16:42:51 +0100 Subject: [PATCH 15/23] Introduced new filter option `prefregex` for pre-filtering using single regular expression; Some filters extended with user name; [filter.d/pam-generic.conf]: grave fix injection on user name to host fixed; test-cases in testSampleRegexsFactory can now check the captured groups (using additionally fields in failJSON structure) --- config/filter.d/pam-generic.conf | 7 ++- config/filter.d/sshd.conf | 30 +++++----- fail2ban/client/fail2banregex.py | 74 +++++++++++++----------- fail2ban/client/filterreader.py | 5 +- fail2ban/server/failregex.py | 81 ++++++++++++++------------- fail2ban/server/filter.py | 43 ++++++++++++-- fail2ban/server/server.py | 8 +++ fail2ban/server/transmitter.py | 6 ++ fail2ban/tests/files/logs/pam-generic | 14 +++-- fail2ban/tests/samplestestcase.py | 18 ++++-- 10 files changed, 183 insertions(+), 103 deletions(-) diff --git a/config/filter.d/pam-generic.conf b/config/filter.d/pam-generic.conf index e0d4e9c1..ff4ea802 100644 --- a/config/filter.d/pam-generic.conf +++ b/config/filter.d/pam-generic.conf @@ -16,7 +16,12 @@ _ttys_re=\S* __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:? _daemon = \S+ -failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s ruser=\S* rhost=(?:\s+user=.*)?\s*$ +prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s .+$ + +failregex = ^ruser=\S* rhost=\s*$ + ^ruser= rhost=\s+user=\S*\s*$ + ^ruser= rhost=\s+user=.*?\s*$ + ^ruser=.*? rhost=\s*$ ignoreregex = diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index ebdc06ec..872a73e4 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -32,23 +32,23 @@ __prefix_line_ml2 = %(__suff)s$^(?P=__prefix)%(__pref)s mode = %(normal)s -normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ - ^%(__prefix_line_sl)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ - ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ - ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ +normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ + ^%(__prefix_line_sl)sFailed \S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ + ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ + ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ ^%(__prefix_line_sl)srefused connect from \S+ \(\)\s*%(__suff)s$ ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ - ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ - ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ - ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from : 11: .+%(__suff)s$ - ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ + ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ + ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from : 11: .+%(__suff)s$ + ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index d111e09c..84106d02 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -271,7 +271,7 @@ class Fail2banRegex(object): def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) regex = regextype + 'regex' - if os.path.isfile(value) or os.path.isfile(value + '.conf'): + if regextype == 'fail' and (os.path.isfile(value) or os.path.isfile(value + '.conf')): if os.path.basename(os.path.dirname(value)) == 'filter.d': ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): basedir = os.path.dirname(os.path.dirname(value)) @@ -291,43 +291,51 @@ class Fail2banRegex(object): return False reader.getOptions(None) readercommands = reader.convert() - regex_values = [ - RegexStat(m[3]) - for m in filter( - lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype, - readercommands) - ] + [ - RegexStat(m) - for mm in filter( - lambda x: x[0] == 'multi-set' and x[2] == "add%sregex" % regextype, - readercommands) - for m in mm[3] - ] - # Read out and set possible value of maxlines - for command in readercommands: - if command[2] == "maxlines": - maxlines = int(command[3]) + + regex_values = {} + for opt in readercommands: + if opt[0] == 'multi-set': + optval = opt[3] + elif opt[0] == 'set': + optval = [opt[3]] + else: + continue + for optval in optval: try: - self.setMaxLines(maxlines) - except ValueError: - output( "ERROR: Invalid value for maxlines (%(maxlines)r) " \ - "read from %(value)s" % locals() ) + if opt[2] == "prefregex": + self._filter.prefRegex = optval + elif opt[2] == "addfailregex": + stor = regex_values.get('fail') + if not stor: stor = regex_values['fail'] = list() + stor.append(RegexStat(optval)) + #self._filter.addFailRegex(optval) + elif opt[2] == "addignoreregex": + stor = regex_values.get('ignore') + if not stor: stor = regex_values['ignore'] = list() + stor.append(RegexStat(optval)) + #self._filter.addIgnoreRegex(optval) + elif opt[2] == "maxlines": + self.setMaxLines(optval) + elif opt[2] == "datepattern": + self.setDatePattern(optval) + elif opt[2] == "addjournalmatch": + self.setJournalMatch(optval) + except ValueError as e: # pragma: no cover + output( "ERROR: Invalid value for %s (%r) " \ + "read from %s: %s" % (opt[2], optval, value, e) ) return False - elif command[2] == 'addjournalmatch': - journalmatch = command[3:] - self.setJournalMatch(journalmatch) - elif command[2] == 'datepattern': - datepattern = command[3] - self.setDatePattern(datepattern) + else: output( "Use %11s line : %s" % (regex, shortstr(value)) ) - regex_values = [RegexStat(value)] + regex_values = {regextype: [RegexStat(value)]} - setattr(self, "_" + regex, regex_values) - for regex in regex_values: - getattr( - self._filter, - 'add%sRegex' % regextype.title())(regex.getFailRegex()) + for regextype, regex_values in regex_values.iteritems(): + regex = regextype + 'regex' + setattr(self, "_" + regex, regex_values) + for regex in regex_values: + getattr( + self._filter, + 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True def testIgnoreRegex(self, line): diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index d89ef5ad..59e78307 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -37,6 +37,7 @@ logSys = getLogger(__name__) class FilterReader(DefinitionInitConfigReader): _configOpts = { + "prefregex": ["string", None], "ignoreregex": ["string", None], "failregex": ["string", ""], "maxlines": ["int", None], @@ -72,8 +73,8 @@ class FilterReader(DefinitionInitConfigReader): # We warn when multiline regex is used without maxlines > 1 # therefore keep sure we set this option first. stream.insert(0, ["set", self._jailName, "maxlines", value]) - elif opt == 'datepattern': - stream.append(["set", self._jailName, "datepattern", value]) + elif opt in ('datepattern', 'prefregex'): + stream.append(["set", self._jailName, opt, value]) # Do not send a command if the match is empty. elif opt == 'journalmatch': if value is None: continue diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index cbd0cef0..5f6ca01f 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -189,40 +189,45 @@ class Regex: # method of this object. # @param a list of tupples. The tupples are ( prematch, datematch, postdatematch ) - def search(self, tupleLines): + def search(self, tupleLines, orgLines=None): self._matchCache = self._regexObj.search( "\n".join("".join(value[::2]) for value in tupleLines) + "\n") - if self.hasMatched(): - # Find start of the first line where the match was found - try: - self._matchLineStart = self._matchCache.string.rindex( - "\n", 0, self._matchCache.start() +1 ) + 1 - except ValueError: - self._matchLineStart = 0 - # Find end of the last line where the match was found - try: - self._matchLineEnd = self._matchCache.string.index( - "\n", self._matchCache.end() - 1) + 1 - except ValueError: - self._matchLineEnd = len(self._matchCache.string) + if self._matchCache: + if orgLines is None: orgLines = tupleLines + # if single-line: + if len(orgLines) <= 1: + self._matchedTupleLines = orgLines + self._unmatchedTupleLines = [] + else: + # Find start of the first line where the match was found + try: + matchLineStart = self._matchCache.string.rindex( + "\n", 0, self._matchCache.start() +1 ) + 1 + except ValueError: + matchLineStart = 0 + # Find end of the last line where the match was found + try: + matchLineEnd = self._matchCache.string.index( + "\n", self._matchCache.end() - 1) + 1 + except ValueError: + matchLineEnd = len(self._matchCache.string) - lineCount1 = self._matchCache.string.count( - "\n", 0, self._matchLineStart) - lineCount2 = self._matchCache.string.count( - "\n", 0, self._matchLineEnd) - self._matchedTupleLines = tupleLines[lineCount1:lineCount2] - self._unmatchedTupleLines = tupleLines[:lineCount1] - - n = 0 - for skippedLine in self.getSkippedLines(): - for m, matchedTupleLine in enumerate( - self._matchedTupleLines[n:]): - if "".join(matchedTupleLine[::2]) == skippedLine: - self._unmatchedTupleLines.append( - self._matchedTupleLines.pop(n+m)) - n += m - break - self._unmatchedTupleLines.extend(tupleLines[lineCount2:]) + lineCount1 = self._matchCache.string.count( + "\n", 0, matchLineStart) + lineCount2 = self._matchCache.string.count( + "\n", 0, matchLineEnd) + self._matchedTupleLines = orgLines[lineCount1:lineCount2] + self._unmatchedTupleLines = orgLines[:lineCount1] + n = 0 + for skippedLine in self.getSkippedLines(): + for m, matchedTupleLine in enumerate( + self._matchedTupleLines[n:]): + if "".join(matchedTupleLine[::2]) == skippedLine: + self._unmatchedTupleLines.append( + self._matchedTupleLines.pop(n+m)) + n += m + break + self._unmatchedTupleLines.extend(orgLines[lineCount2:]) # Checks if the previous call to search() matched. # @@ -234,6 +239,13 @@ class Regex: else: return False + ## + # Returns all matched groups. + # + + def getGroups(self): + return self._matchCache.groupdict() + ## # Returns skipped lines. # @@ -332,13 +344,6 @@ class FailRegex(Regex): if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]: raise RegexException("No failure-id group in '%s'" % self._regex) - ## - # Returns all matched groups. - # - - def getGroups(self): - return self._matchCache.groupdict() - ## # Returns the matched failure id. # diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b7c78b9b..d25e8dc5 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -65,6 +65,8 @@ class Filter(JailThread): self.jail = jail ## The failures manager. self.failManager = FailManager() + ## Regular expression pre-filtering matching the failures. + self.__prefRegex = None ## The regular expression list matching the failures. self.__failRegex = list() ## The regular expression list with expressions to ignore. @@ -129,6 +131,16 @@ class Filter(JailThread): self.delLogPath(path) delattr(self, '_reload_logs') + @property + def prefRegex(self): + return self.__prefRegex + @prefRegex.setter + def prefRegex(self, value): + if value: + self.__prefRegex = Regex(value, useDns=self.__useDns) + else: + self.__prefRegex = None + ## # Add a regular expression which matches the failure. # @@ -582,13 +594,30 @@ class Filter(JailThread): date, MyTime.time(), self.getFindTime()) return failList - self.__lineBuffer = ( - self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] - logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer) + if self.__lineBufferSize > 1: + orgBuffer = self.__lineBuffer = ( + self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] + else: + orgBuffer = self.__lineBuffer = [tupleLine[:3]] + logSys.log(5, "Looking for failregex match of %r", self.__lineBuffer) + + # Pre-filter fail regex (if available): + preGroups = {} + if self.__prefRegex: + failRegex = self.__prefRegex.search(self.__lineBuffer) + if not self.__prefRegex.hasMatched(): + return failList + logSys.log(7, "Pre-filter matched %s", failRegex) + preGroups = self.__prefRegex.getGroups() + repl = preGroups.get('content') + # Content replacement: + if repl: + del preGroups['content'] + self.__lineBuffer = [('', '', repl)] # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): - failRegex.search(self.__lineBuffer) + failRegex.search(self.__lineBuffer, orgBuffer) if failRegex.hasMatched(): # The failregex matched. logSys.log(7, "Matched %s", failRegex) @@ -617,7 +646,11 @@ class Filter(JailThread): # retrieve failure-id, host, etc from failure match: raw = returnRawHost try: - fail = failRegex.getGroups() + if preGroups: + fail = preGroups.copy() + fail.update(failRegex.getGroups()) + else: + fail = failRegex.getGroups() # failure-id: fid = fail.get('fid') # ip-address or host: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 313b6ee5..dfab1e38 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -379,6 +379,14 @@ class Server: def getIgnoreCommand(self, name): return self.__jails[name].filter.getIgnoreCommand() + def setPrefRegex(self, name, value): + flt = self.__jails[name].filter + logSys.debug(" prefregex: %r", value) + flt.prefRegex = value + + def getPrefRegex(self, name): + return self.__jails[name].filter.prefRegex + def addFailRegex(self, name, value, multiple=False): flt = self.__jails[name].filter if not multiple: value = (value,) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index d23f12e2..265b9704 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -221,6 +221,10 @@ class Transmitter: value = command[2:] self.__server.delJournalMatch(name, value) return self.__server.getJournalMatch(name) + elif command[1] == "prefregex": + value = command[2] + self.__server.setPrefRegex(name, value) + return self.__server.getPrefRegex(name) elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) @@ -341,6 +345,8 @@ class Transmitter: return self.__server.getIgnoreIP(name) elif command[1] == "ignorecommand": return self.__server.getIgnoreCommand(name) + elif command[1] == "prefregex": + return self.__server.getPrefRegex(name) elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": diff --git a/fail2ban/tests/files/logs/pam-generic b/fail2ban/tests/files/logs/pam-generic index e562ac7f..1740f0c8 100644 --- a/fail2ban/tests/files/logs/pam-generic +++ b/fail2ban/tests/files/logs/pam-generic @@ -1,17 +1,23 @@ -# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1" } +# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1", "user": "sample-user" } Feb 7 15:10:42 example pure-ftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=pure-ftpd ruser=sample-user rhost=192.168.1.1 -# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } +# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com", "user": "root" } May 12 09:47:54 vaio sshd[16004]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com user=root # failJSON: { "time": "2005-05-12T09:48:03", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" } May 12 09:48:03 vaio sshd[16021]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com -# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62" } +# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62", "user": "mark" } May 15 18:02:12 localhost proftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=66.232.129.62 user=mark # linux-pam messages before commit f0f9c4479303b5a9c37667cf07f58426dc081676 (release 0.99.2.0 ) - nolonger supported # failJSON: { "time": "2004-11-25T17:12:13", "match": false } Nov 25 17:12:13 webmail pop(pam_unix)[4920]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.168.10.3 user=mailuser -# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com" } +# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com", "user": "an8767" } Jul 19 18:11:26 srv2 vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www.google.com # failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www.google.com" } Jul 19 18:11:26 srv2 vsftpd: pam_unix: authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www.google.com + + +# failJSON: { "time": "2005-07-19T18:11:50", "match": true , "host": "192.0.2.1", "user": "test rhost=192.0.2.151", "desc": "Injecting on username"} +Jul 19 18:11:50 srv2 daemon: pam_unix(auth): authentication failure; logname= uid=0 euid=0 tty=xxx ruser=test rhost=192.0.2.151 rhost=192.0.2.1 +# failJSON: { "time": "2005-07-19T18:11:52", "match": true , "host": "192.0.2.2", "user": "test rhost=192.0.2.152", "desc": "Injecting on username after host"} +Jul 19 18:11:52 srv2 daemon: pam_unix(auth): authentication failure; logname= uid=0 euid=0 tty=xxx ruser= rhost=192.0.2.2 user=test rhost=192.0.2.152 diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 3fdc50c6..a97e92c3 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -102,7 +102,9 @@ def testSampleRegexsFactory(name, basedir): else: continue for optval in optval: - if opt[2] == "addfailregex": + if opt[2] == "prefregex": + self.filter.prefRegex = optval + elif opt[2] == "addfailregex": self.filter.addFailRegex(optval) elif opt[2] == "addignoreregex": self.filter.addIgnoreRegex(optval) @@ -126,7 +128,7 @@ def testSampleRegexsFactory(name, basedir): # test regexp contains greedy catch-all before , that is # not hard-anchored at end or has not precise sub expression after : for fr in self.filter.getFailRegex(): - if RE_WRONG_GREED.search(fr): #pragma: no cover + if RE_WRONG_GREED.search(fr): # pragma: no cover raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before , " "that is not hard-anchored at end or has not precise sub expression after :\n%s" % (name, str(fr).replace(RE_HOST, ''))) @@ -152,12 +154,12 @@ def testSampleRegexsFactory(name, basedir): if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True), - "Line not matched when should have: %s:%i %r" % + "Line not matched when should have: %s:%i, line:\n%s" % (logFile.filename(), logFile.filelineno(), line)) elif ret: # Check line is flagged to match self.assertTrue(faildata.get('match', False), - "Line matched when shouldn't have: %s:%i %r" % + "Line matched when shouldn't have: %s:%i, line:\n%s" % (logFile.filename(), logFile.filelineno(), line)) self.assertEqual(len(ret), 1, "Multiple regexs matched %r - %s:%i" % (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno())) @@ -165,6 +167,12 @@ def testSampleRegexsFactory(name, basedir): # Verify timestamp and host as expected failregex, host, fail2banTime, lines, fail = ret[0] self.assertEqual(host, faildata.get("host", None)) + # Verify other captures: + for k, v in faildata.iteritems(): + if k not in ("time", "match", "host", "desc"): + fv = fail.get(k, None) + self.assertEqual(fv, v, "Value of %s mismatch %r != %r on: %s:%i, line:\n%s" % ( + k, fv, v, logFile.filename(), logFile.filelineno(), line)) t = faildata.get("time", None) try: @@ -177,7 +185,7 @@ def testSampleRegexsFactory(name, basedir): jsonTime += jsonTimeLocal.microsecond / 1000000 self.assertEqual(fail2banTime, jsonTime, - "UTC Time mismatch fail2ban %s (%s) != failJson %s (%s) (diff %.3f seconds) on: %s:%i %r:" % + "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds) on: %s:%i, line:\n%s" % (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) ) From 2fad50b6e8296a03fecc12b542d62350ad662ff5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Feb 2017 15:54:25 +0100 Subject: [PATCH 16/23] Precedence of `prefregex` higher as `failregex` should be in head of the convert-stream; Allow using failure-id (``) within `prefregex` (by common prefix for all expressions specified with `failregex`) --- fail2ban/client/filterreader.py | 9 ++++----- fail2ban/server/failregex.py | 7 +++++-- fail2ban/server/filter.py | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 59e78307..9edeb2f3 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -69,11 +69,10 @@ class FilterReader(DefinitionInitConfigReader): stream.append(["multi-set", self._jailName, "add" + opt, multi]) elif len(multi): stream.append(["set", self._jailName, "add" + opt, multi[0]]) - elif opt == 'maxlines': - # We warn when multiline regex is used without maxlines > 1 - # therefore keep sure we set this option first. - stream.insert(0, ["set", self._jailName, "maxlines", value]) - elif opt in ('datepattern', 'prefregex'): + elif opt in ('maxlines', 'prefregex'): + # Be sure we set this options first. + stream.insert(0, ["set", self._jailName, opt, value]) + elif opt in ('datepattern'): stream.append(["set", self._jailName, opt, value]) # Do not send a command if the match is empty. elif opt == 'journalmatch': diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 5f6ca01f..19a89303 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -337,11 +337,14 @@ class FailRegex(Regex): # avoid construction of invalid object. # @param value the regular expression - def __init__(self, regex, **kwargs): + def __init__(self, regex, prefRegex=None, **kwargs): # Initializes the parent. Regex.__init__(self, regex, **kwargs) # Check for group "dns", "ip4", "ip6", "fid" - if not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex]: + if (not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex] + and (prefRegex is None or + not [grp for grp in FAILURE_ID_GROPS if grp in prefRegex._regexObj.groupindex]) + ): raise RegexException("No failure-id group in '%s'" % self._regex) ## diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d25e8dc5..85c07103 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -150,7 +150,7 @@ class Filter(JailThread): def addFailRegex(self, value): try: - regex = FailRegex(value, useDns=self.__useDns) + regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns) self.__failRegex.append(regex) if "\n" in regex.getRegex() and not self.getMaxLines() > 1: logSys.warning( @@ -604,11 +604,11 @@ class Filter(JailThread): # Pre-filter fail regex (if available): preGroups = {} if self.__prefRegex: - failRegex = self.__prefRegex.search(self.__lineBuffer) + self.__prefRegex.search(self.__lineBuffer) if not self.__prefRegex.hasMatched(): return failList - logSys.log(7, "Pre-filter matched %s", failRegex) preGroups = self.__prefRegex.getGroups() + logSys.log(7, "Pre-filter matched %s", preGroups) repl = preGroups.get('content') # Content replacement: if repl: From 22afdbd536a193fe8ee7ff8dcc0ed7dd6cd23342 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Feb 2017 15:54:59 +0100 Subject: [PATCH 17/23] Several filters optimized with pre-filtering using new option `prefregex` --- config/filter.d/apache-auth.conf | 32 +++++++++++++++----------- config/filter.d/apache-botsearch.conf | 9 ++++---- config/filter.d/apache-shellshock.conf | 6 +++-- config/filter.d/asterisk.conf | 22 ++++++++++-------- config/filter.d/courier-smtp.conf | 6 +++-- config/filter.d/dovecot.conf | 15 +++++++----- config/filter.d/dropbear.conf | 8 ++++--- config/filter.d/exim-common.conf | 4 +++- config/filter.d/exim.conf | 7 ++++-- config/filter.d/froxlor-auth.conf | 7 ++++-- config/filter.d/murmur.conf | 6 +++-- config/filter.d/named-refused.conf | 8 ++++--- config/filter.d/postfix.conf | 16 +++++++------ config/filter.d/proftpd.conf | 12 ++++++---- config/filter.d/xinetd-fail.conf | 6 +++-- 15 files changed, 99 insertions(+), 65 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 8a63858d..c59a817c 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -9,20 +9,24 @@ before = apache-common.conf [Definition] +prefregex = ^%(_apache_error_client)s (?:AH\d+: )?.+$ -failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$ - ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH\d+: )?Authorization of user \S+ to access \S* failed, reason: .*$ - ^%(_apache_error_client)s (AH0179[24]: )?(Digest: )?user .*?: password mismatch: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH0179[01]: |Digest: )user `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01631: )?user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01775: )?(Digest: )?invalid nonce .* received - length is not \S+(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01788: )?(Digest: )?realm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01789: )?(Digest: )?unknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01793: )?invalid qop `.*?' received: \S*(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01777: )?(Digest: )?invalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$ +# auth_type = ((?:Digest|Basic): )? +auth_type = ([A-Z]\w+: )? + +failregex = ^client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$ + ^user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$ + ^user .*? not found(: )?\S*(, referer: \S+)?\s*$ + ^client used wrong authentication scheme: \S*(, referer: \S+)?\s*$ + ^Authorization of user \S+ to access \S* failed, reason: .*$ + ^%(auth_type)suser .*?: password mismatch: \S*(, referer: \S+)?\s*$ + ^%(auth_type)suser `.*?' in realm `.+' (not found|denied by provider): \S*(, referer: \S+)?\s*$ + ^user .*?: authorization failure for "\S*":(, referer: \S+)?\s*$ + ^%(auth_type)sinvalid nonce .* received - length is not \S+(, referer: \S+)?\s*$ + ^%(auth_type)srealm mismatch - got `.*?' but expected `.+'(, referer: \S+)?\s*$ + ^%(auth_type)sunknown algorithm `.*?' received: \S*(, referer: \S+)?\s*$ + ^invalid qop `.*?' received: \S*(, referer: \S+)?\s*$ + ^%(auth_type)sinvalid nonce .*? received - user attempted time travel(, referer: \S+)?\s*$ ignoreregex = @@ -53,4 +57,4 @@ ignoreregex = # referer is always in error log messages if it exists added as per the log_error_core function in server/log.c # # Author: Cyril Jaquier -# Major edits by Daniel Black +# Major edits by Daniel Black and Sergey Brester (sebres) diff --git a/config/filter.d/apache-botsearch.conf b/config/filter.d/apache-botsearch.conf index 5687d405..7def09ac 100644 --- a/config/filter.d/apache-botsearch.conf +++ b/config/filter.d/apache-botsearch.conf @@ -23,14 +23,13 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): (, referer: \S+)?\s*$ - ^%(_apache_error_client)s script '' not found or unable to stat(, referer: \S+)?\s*$ +prefregex = ^%(_apache_error_client)s (?:AH\d+: )?.+$ + +failregex = ^(?:File does not exist|script not found or unable to stat): (, referer: \S+)?\s*$ + ^script '' not found or unable to stat(, referer: \S+)?\s*$ ignoreregex = - -[Init] - # Webroot represents the webroot on which all other files are based webroot = /var/www/ diff --git a/config/filter.d/apache-shellshock.conf b/config/filter.d/apache-shellshock.conf index 39df1704..e2707dc0 100644 --- a/config/filter.d/apache-shellshock.conf +++ b/config/filter.d/apache-shellshock.conf @@ -9,8 +9,10 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: warning: HTTP_.*?: ignoring function definition attempt(, referer: \S+)?\s*$ - ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: error importing function definition for `HTTP_.*?'(, referer: \S+)?\s*$ +prefregex = ^%(_apache_error_client)s (AH01215: )?/bin/([bd]a)?sh: .+$ + +failregex = ^warning: HTTP_[^:]+: ignoring function definition attempt(, referer: \S+)?\s*$ + ^error importing function definition for `HTTP_[^']+'(, referer: \S+)?\s*$ ignoreregex = diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index df55d288..b9871b6c 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -18,16 +18,18 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} # All Asterisk log messages begin like this: log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)? -failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context - ^%(__prefix_line)s%(log_prefix)s Host failed to authenticate as '[^']*'$ - ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from \)$ - ^%(__prefix_line)s%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ - ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@\S*$ - ^%(__prefix_line)s%(log_prefix)s hacking attempt detected ''$ - ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)//\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$ - ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from "$ - ^%(__prefix_line)s%(log_prefix)s Request (?:'[^']*' )?from '[^']*' failed for '(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$ +prefregex = ^%(__prefix_line)s%(log_prefix)s .+$ + +failregex = ^Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ + ^Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context + ^Host failed to authenticate as '[^']*'$ + ^No registration for peer '[^']*' \(from \)$ + ^Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^Failed to authenticate (user|device) [^@]+@\S*$ + ^hacking attempt detected ''$ + ^SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)//\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$ + ^"Rejecting unknown SIP connection from "$ + ^Request (?:'[^']*' )?from '[^']*' failed for '(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$ ignoreregex = diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index fc0afc26..888753c4 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,8 +12,10 @@ before = common.conf _daemon = courieresmtpd -failregex = ^%(__prefix_line)serror,relay=,.*: 550 User (<.*> )?unknown\.?$ - ^%(__prefix_line)serror,relay=,msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ +prefregex = ^%(__prefix_line)serror,relay=,.+$ + +failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ + ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ ignoreregex = diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 3c45a332..966a7e86 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -7,13 +7,16 @@ before = common.conf [Definition] -_daemon = (auth|dovecot(-auth)?|auth-worker) +_auth_worker = (?:dovecot: )?auth(?:-worker)? +_daemon = (dovecot(-auth)?|auth) -failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ - ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ - ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ - ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,\): unknown user\s*$ - ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ +prefregex = ^%(__prefix_line)s(%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?.+$ + +failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ + ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ + ^(?:pam|passwd-file)\(\S+,\): unknown user\s*$ + ^ldap\(\S*,,\S*\): invalid credentials\s*$ ignoreregex = diff --git a/config/filter.d/dropbear.conf b/config/filter.d/dropbear.conf index 288b0882..930bb128 100644 --- a/config/filter.d/dropbear.conf +++ b/config/filter.d/dropbear.conf @@ -23,9 +23,11 @@ before = common.conf _daemon = dropbear -failregex = ^%(__prefix_line)s[Ll]ogin attempt for nonexistent user ('.*' )?from :\d+$ - ^%(__prefix_line)s[Bb]ad (PAM )?password attempt for .+ from (:\d+)?$ - ^%(__prefix_line)s[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from :\d+\s*$ +prefregex = ^%(__prefix_line)s(?:[Ll]ogin|[Bb]ad|[Ee]xit).+$ + +failregex = ^[Ll]ogin attempt for nonexistent user ('.*' )?from :\d+$ + ^[Bb]ad (PAM )?password attempt for .+ from (:\d+)?$ + ^[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from :\d+\s*$ ignoreregex = diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index 0e1b74fa..b3b25750 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -9,7 +9,9 @@ after = exim-common.local [Definition] -host_info = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?\[\](?::\d+)? (?:I=\[\S+\](:\d+)? )?(?:U=\S+ )?(?:P=e?smtp )? +host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)? +host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s +host_info = %(host_info_pre)s\[\]%(host_info_suf)s pid = (?: \[\d+\])? # DEV Notes: diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index a1d699c0..5d293429 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -13,14 +13,17 @@ before = exim-common.conf [Definition] +# Fre-filter via "prefregex" is currently inactive because of too different failure syntax in exim-log (testing needed): +#prefregex = ^%(pid)s \b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info_pre)s\[[^\]]+\]%(host_info_suf)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+$ + failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s \w+ authenticator failed for (\S+ )?\(\S+\) \[\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ ^%(pid)s no MAIL in SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$ - ^%(pid)s \S+ SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ + ^%(pid)s ([\w\-]+ )?SMTP connection from (?:\S* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ ignoreregex = diff --git a/config/filter.d/froxlor-auth.conf b/config/filter.d/froxlor-auth.conf index 04003263..d8f3785c 100644 --- a/config/filter.d/froxlor-auth.conf +++ b/config/filter.d/froxlor-auth.conf @@ -25,8 +25,11 @@ _daemon = Froxlor # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)s\[Login Action \] Unknown user \S* tried to login.$ - ^%(__prefix_line)s\[Login Action \] User \S* tried to login with wrong password.$ + +prefregex = ^%(__prefix_line)s\[Login Action \] .+$ + +failregex = ^Unknown user \S* tried to login.$ + ^User \S* tried to login with wrong password.$ # Option: ignoreregex diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf index 507bbd2f..f5f100a6 100644 --- a/config/filter.d/murmur.conf +++ b/config/filter.d/murmur.conf @@ -17,8 +17,10 @@ _usernameregex = [^>]+ _prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from :\d+: -failregex = ^%(_prefix)s Invalid server password$ - ^%(_prefix)s Wrong certificate or password for existing user$ +prefregex = ^%(_prefix)s .+$ + +failregex = ^Invalid server password$ + ^Wrong certificate or password for existing user$ ignoreregex = diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index eec3d667..2e14d442 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -34,9 +34,11 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re) # this can be optional (for instance if we match named native log files) __line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? -failregex = ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ - ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$ - ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ +prefregex = ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: .+$ + +failregex = ^(view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ + ^zone transfer '\S+/AXFR/\w+' denied\s*$ + ^bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ ignoreregex = diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 3051409b..005ed585 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -12,13 +12,15 @@ before = common.conf _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] -failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ - ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ - ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ - ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; - ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ - ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ - ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ +prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) .+$ + +failregex = ^RCPT from \S+\[\]: 554 5\.7\.1 + ^RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ + ^RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; + ^VRFY from \S+\[\]: 550 5\.1\.1 + ^RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ + ^after \S+ from [^[]*\[\]:?$ ignoreregex = diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 4bc0ba01..455bbdfc 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -16,10 +16,14 @@ _daemon = proftpd __suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? -failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ + +prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ (?:USER|SECURITY|Maximum).+$ + + +failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ + ^SECURITY VIOLATION: .* login attempted\. *$ + ^Maximum login attempts \(\d+\) exceeded *$ ignoreregex = diff --git a/config/filter.d/xinetd-fail.conf b/config/filter.d/xinetd-fail.conf index d75e3d66..b4093d98 100644 --- a/config/filter.d/xinetd-fail.conf +++ b/config/filter.d/xinetd-fail.conf @@ -14,8 +14,10 @@ before = common.conf _daemon = xinetd -failregex = ^%(__prefix_line)sFAIL: \S+ address from=$ - ^%(__prefix_line)sFAIL: \S+ libwrap from=$ +prefregex = ^%(__prefix_line)sFAIL: .+$ + +failregex = ^\S+ address from=$ + ^\S+ libwrap from=$ ignoreregex = From 8bcaeb9022543a31214c1d38881afc0b6f9e6783 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Feb 2017 17:03:36 +0100 Subject: [PATCH 18/23] amend to 4ff8d051f49808ac769709c5aff8591fcd79040a: fixed fail2ban-regex with journalmatch using systemd-journal --- fail2ban/client/fail2banregex.py | 49 ++++++++++++++----------- fail2ban/tests/fail2banregextestcase.py | 24 ++++++------ 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 84106d02..395fe8ef 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -235,7 +235,7 @@ class Fail2banRegex(object): else: self._maxlines = 20 if opts.journalmatch is not None: - self.setJournalMatch(opts.journalmatch.split()) + self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: @@ -243,6 +243,7 @@ class Fail2banRegex(object): self._filter.returnRawHost = opts.raw self._filter.checkFindTime = False self._filter.checkAllRegex = True + self._opts = opts def decode_line(self, line): return FileContainer.decode_line('', self._encoding, line) @@ -265,8 +266,7 @@ class Fail2banRegex(object): output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): - if self._journalmatch is None: - self._journalmatch = v + self._journalmatch = v def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) @@ -297,33 +297,38 @@ class Fail2banRegex(object): if opt[0] == 'multi-set': optval = opt[3] elif opt[0] == 'set': - optval = [opt[3]] + optval = opt[3:] else: continue - for optval in optval: - try: - if opt[2] == "prefregex": + try: + if opt[2] == "prefregex": + for optval in optval: self._filter.prefRegex = optval - elif opt[2] == "addfailregex": - stor = regex_values.get('fail') - if not stor: stor = regex_values['fail'] = list() + elif opt[2] == "addfailregex": + stor = regex_values.get('fail') + if not stor: stor = regex_values['fail'] = list() + for optval in optval: stor.append(RegexStat(optval)) #self._filter.addFailRegex(optval) - elif opt[2] == "addignoreregex": - stor = regex_values.get('ignore') - if not stor: stor = regex_values['ignore'] = list() + elif opt[2] == "addignoreregex": + stor = regex_values.get('ignore') + if not stor: stor = regex_values['ignore'] = list() + for optval in optval: stor.append(RegexStat(optval)) #self._filter.addIgnoreRegex(optval) - elif opt[2] == "maxlines": + elif opt[2] == "maxlines": + for optval in optval: self.setMaxLines(optval) - elif opt[2] == "datepattern": + elif opt[2] == "datepattern": + for optval in optval: self.setDatePattern(optval) - elif opt[2] == "addjournalmatch": + elif opt[2] == "addjournalmatch": + if self._opts.journalmatch is None: self.setJournalMatch(optval) - except ValueError as e: # pragma: no cover - output( "ERROR: Invalid value for %s (%r) " \ - "read from %s: %s" % (opt[2], optval, value, e) ) - return False + except ValueError as e: # pragma: no cover + output( "ERROR: Invalid value for %s (%r) " \ + "read from %s: %s" % (opt[2], optval, value, e) ) + return False else: output( "Use %11s line : %s" % (regex, shortstr(value)) ) @@ -510,7 +515,7 @@ class Fail2banRegex(object): for line in hdlr: yield self.decode_line(line) - def start(self, opts, args): + def start(self, args): cmd_log, cmd_regex = args[:2] @@ -603,5 +608,5 @@ def exec_command_line(*args): logSys.addHandler(stdout) fail2banRegex = Fail2banRegex(opts) - if not fail2banRegex.start(opts, args): + if not fail2banRegex.start(args): sys.exit(-1) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 5767a3f7..8e6fcfe0 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -96,7 +96,7 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "test", r".** from $" ) - self.assertFalse(fail2banRegex.start(opts, args)) + self.assertFalse(fail2banRegex.start(args)) self.assertLogged("Unable to compile regular expression") def testWrongIngnoreRE(self): @@ -104,7 +104,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--datepattern", "{^LN-BEG}EPOCH", "test", r".*? from $", r".**" ) - self.assertFalse(fail2banRegex.start(opts, args)) + self.assertFalse(fail2banRegex.start(args)) self.assertLogged("Unable to compile regular expression") def testDirectFound(self): @@ -114,7 +114,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", r"Authentication failure for .*? from $" ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') def testDirectNotFound(self): @@ -123,7 +123,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", r"XYZ from $" ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed') def testDirectIgnored(self): @@ -133,7 +133,7 @@ class Fail2banRegexTest(LogCaptureTestCase): r"Authentication failure for .*? from $", r"kevin from 192.0.2.0$" ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed') def testDirectRE_1(self): @@ -143,7 +143,7 @@ class Fail2banRegexTest(LogCaptureTestCase): Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') self.assertLogged('Error decoding line'); @@ -159,7 +159,7 @@ class Fail2banRegexTest(LogCaptureTestCase): Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') def testDirectRE_1raw_noDns(self): @@ -169,7 +169,7 @@ class Fail2banRegexTest(LogCaptureTestCase): Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') def testDirectRE_2(self): @@ -179,7 +179,7 @@ class Fail2banRegexTest(LogCaptureTestCase): Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') def testVerbose(self): @@ -189,7 +189,7 @@ class Fail2banRegexTest(LogCaptureTestCase): Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') self.assertLogged('141.3.81.106 Sun Aug 14 11:53:59 2005') @@ -200,7 +200,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') self.assertLogged('Error decoding line') @@ -215,7 +215,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--debuggex", "--print-all-matched", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) - self.assertTrue(fail2banRegex.start(opts, args)) + self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') self.assertLogged('https://') From 35efca594136695f537760e417ab4255a00692c3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 22 Feb 2017 18:39:44 +0100 Subject: [PATCH 19/23] Better multi-line handling introduced: single-line parsing with caching of needed failure information to process in further lines. Many times faster and fewer CPU-hungry because of parsing with `maxlines=1`, so without line buffering (scrolling of the buffer-window). Combination of tags `` and `` can be used now to process multi-line logs using single-line expressions: - tag ``: used to identify resp. store failure info for groups of log-lines with the same identifier (e. g. combined failure-info for the same conn-id by `(?:conn-id)`, see sshd.conf for example) - tag ``: used as mark for no-failure (helper to accumulate common failure-info); filter.d/sshd.conf: [sshd], [sshd-ddos], [sshd-aggressive] optimized with pre-filtering using new option `prefregex` and new multi-line handling. --- config/filter.d/sshd.conf | 58 ++++---- fail2ban/server/failregex.py | 8 +- fail2ban/server/filter.py | 222 ++++++++++++++++++------------ fail2ban/server/ticket.py | 9 +- fail2ban/server/utils.py | 6 + fail2ban/tests/filtertestcase.py | 1 + fail2ban/tests/samplestestcase.py | 52 ++++--- 7 files changed, 214 insertions(+), 142 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 872a73e4..922ea193 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -24,37 +24,37 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: \[preauth\])?\s* __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? -# single line prefix: -__prefix_line_sl = %(__prefix_line)s%(__pref)s -# multi line prefixes (for first and second lines): -__prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s -__prefix_line_ml2 = %(__suff)s$^(?P=__prefix)%(__pref)s +prefregex = ^%(__prefix_line)s%(__pref)s.+$ mode = %(normal)s -normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ - ^%(__prefix_line_sl)sFailed \S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ - ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ - ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ - ^%(__prefix_line_sl)srefused connect from \S+ \(\)\s*%(__suff)s$ - ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ - ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ - ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ - ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ - ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from : 11: .+%(__suff)s$ - ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ +normal = ^[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ + ^User not known to the underlying authentication module for .* from \s*%(__suff)s$ + ^Failed \S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^ROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ + ^[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ + ^User .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ + ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ + ^User .+ from not allowed because not in any group\s*%(__suff)s$ + ^refused connect from \S+ \(\)\s*%(__suff)s$ + ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ + ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ + ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ + ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ + ^User .+ not allowed because account is locked%(__suff)s + ^Disconnecting: Too many authentication failures for .+?%(__suff)s + ^Received disconnect from : 11: + ^Connection closed by %(__suff)s$ -ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ - ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ - ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$ - ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ +ddos = ^Did not receive identification string from %(__suff)s$ + ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ + ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. + ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$ + ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: + ^Read from socket failed: Connection reset by peer \[preauth\] + +common = ^Connection from aggressive = %(normal)s %(ddos)s @@ -62,11 +62,11 @@ aggressive = %(normal)s [Definition] failregex = %(mode)s + %(common)s ignoreregex = -# "maxlines" is number of log lines to buffer for multi-line regex searches -maxlines = 10 +maxlines = 1 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 19a89303..59f59978 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -323,6 +323,10 @@ class RegexException(Exception): # FAILURE_ID_GROPS = ("fid", "ip4", "ip6", "dns") +# Additionally allows multi-line failure-id (used for wrapping e. g. conn-id to host) +# +FAILURE_ID_PRESENTS = FAILURE_ID_GROPS + ("mlfid",) + ## # Regular expression class. # @@ -341,9 +345,9 @@ class FailRegex(Regex): # Initializes the parent. Regex.__init__(self, regex, **kwargs) # Check for group "dns", "ip4", "ip6", "fid" - if (not [grp for grp in FAILURE_ID_GROPS if grp in self._regexObj.groupindex] + if (not [grp for grp in FAILURE_ID_PRESENTS if grp in self._regexObj.groupindex] and (prefRegex is None or - not [grp for grp in FAILURE_ID_GROPS if grp in prefRegex._regexObj.groupindex]) + not [grp for grp in FAILURE_ID_PRESENTS if grp in prefRegex._regexObj.groupindex]) ): raise RegexException("No failure-id group in '%s'" % self._regex) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 85c07103..6b782fcc 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,6 +38,7 @@ from .datedetector import DateDetector from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction +from .utils import Utils from ..helpers import getLogger, PREFER_ENC # Gets the instance of the logger. @@ -88,6 +89,8 @@ class Filter(JailThread): self.__ignoreCommand = False ## Default or preferred encoding (to decode bytes from file or journal): self.__encoding = PREFER_ENC + ## Cache temporary holds failures info (used by multi-line for wrapping e. g. conn-id to host): + self.__mlfidCache = None ## Error counter (protected, so can be used in filter implementations) ## if it reached 100 (at once), run-cycle will go idle self._errors = 0 @@ -101,7 +104,7 @@ class Filter(JailThread): self.ticks = 0 self.dateDetector = DateDetector() - logSys.debug("Created %s" % self) + logSys.debug("Created %s", self) def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self.jail) @@ -131,6 +134,13 @@ class Filter(JailThread): self.delLogPath(path) delattr(self, '_reload_logs') + @property + def mlfidCache(self): + if self.__mlfidCache: + return self.__mlfidCache + self.__mlfidCache = Utils.Cache(maxCount=100, maxTime=5*60) + return self.__mlfidCache + @property def prefRegex(self): return self.__prefRegex @@ -170,7 +180,7 @@ class Filter(JailThread): del self.__failRegex[index] except IndexError: logSys.error("Cannot remove regular expression. Index %d is not " - "valid" % index) + "valid", index) ## # Get the regular expression which matches the failure. @@ -208,7 +218,7 @@ class Filter(JailThread): del self.__ignoreRegex[index] except IndexError: logSys.error("Cannot remove regular expression. Index %d is not " - "valid" % index) + "valid", index) ## # Get the regular expression which matches the failure. @@ -231,9 +241,9 @@ class Filter(JailThread): value = value.lower() # must be a string by now if not (value in ('yes', 'warn', 'no', 'raw')): logSys.error("Incorrect value %r specified for usedns. " - "Using safe 'no'" % (value,)) + "Using safe 'no'", value) value = 'no' - logSys.debug("Setting usedns = %s for %s" % (value, self)) + logSys.debug("Setting usedns = %s for %s", value, self) self.__useDns = value ## @@ -346,7 +356,7 @@ class Filter(JailThread): encoding = PREFER_ENC codecs.lookup(encoding) # Raise LookupError if invalid codec self.__encoding = encoding - logSys.info(" encoding: %s" % encoding) + logSys.info(" encoding: %s", encoding) return encoding ## @@ -391,7 +401,7 @@ class Filter(JailThread): if not isinstance(ip, IPAddr): ip = IPAddr(ip) if self.inIgnoreIPList(ip): - logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip) + logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip) unixTime = MyTime.time() self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry()) @@ -435,7 +445,7 @@ class Filter(JailThread): def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: - logSys.info("[%s] Ignore %s by %s" % (self.jailName, ip, ignore_source)) + logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): return self.__ignoreIpList @@ -459,7 +469,7 @@ class Filter(JailThread): if self.__ignoreCommand: command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } ) - logSys.debug('ignore command: ' + command) + logSys.debug('ignore command: %s', command) ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1)) ret_ignore = ret and ret_ignore == 0 self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command") @@ -498,10 +508,7 @@ class Filter(JailThread): for element in self.processLine(line, date): ip = element[1] unixTime = element[2] - lines = element[3] - fail = {} - if len(element) > 4: - fail = element[4] + fail = element[3] logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) if self.inIgnoreIPList(ip, log_ignore=True): @@ -509,7 +516,7 @@ class Filter(JailThread): logSys.info( "[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - tick = FailTicket(ip, unixTime, lines, data=fail) + tick = FailTicket(ip, unixTime, data=fail) self.failManager.addFailure(tick) # reset (halve) error counter (successfully processed line): if self._errors: @@ -544,6 +551,29 @@ class Filter(JailThread): return ignoreRegexIndex return None + def _mergeFailure(self, mlfid, fail, failRegex): + mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None + if mlfidFail: + mlfidGroups = mlfidFail[1] + # if current line not failure, but previous was failure: + if fail.get('nofail') and not mlfidGroups.get('nofail'): + del fail['nofail'] # remove nofail flag - was already market as failure + self.mlfidCache.unset(mlfid) # remove cache entry + # if current line is failure, but previous was not: + elif not fail.get('nofail') and mlfidGroups.get('nofail'): + del mlfidGroups['nofail'] # remove nofail flag + self.mlfidCache.unset(mlfid) # remove cache entry + fail2 = mlfidGroups.copy() + fail2.update(fail) + fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() + fail = fail2 + elif fail.get('nofail'): + fail["matches"] = failRegex.getMatchedTupleLines() + mlfidFail = [self.__lastDate, fail] + self.mlfidCache.set(mlfid, mlfidFail) + return fail + + ## # Finds the failure in a line given split into time and log parts. # @@ -618,76 +648,94 @@ class Filter(JailThread): # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): failRegex.search(self.__lineBuffer, orgBuffer) - if failRegex.hasMatched(): - # The failregex matched. - logSys.log(7, "Matched %s", failRegex) - # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: - # The ignoreregex matched. Remove ignored match. - self.__lineBuffer = failRegex.getUnmatchedTupleLines() - logSys.log(7, "Matched ignoreregex and was ignored") - if not self.checkAllRegex: - break - else: - continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format." - % ("\n".join(failRegex.getMatchedLines()), timeText)) + if not failRegex.hasMatched(): + continue + # The failregex matched. + logSys.log(7, "Matched %s", failRegex) + # Checks if we must ignore this match. + if self.ignoreLine(failRegex.getMatchedTupleLines()) \ + is not None: + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + logSys.log(7, "Matched ignoreregex and was ignored") + if not self.checkAllRegex: + break else: - self.__lineBuffer = failRegex.getUnmatchedTupleLines() - # retrieve failure-id, host, etc from failure match: - raw = returnRawHost - try: - if preGroups: - fail = preGroups.copy() - fail.update(failRegex.getGroups()) - else: - fail = failRegex.getGroups() - # failure-id: - fid = fail.get('fid') - # ip-address or host: - host = fail.get('ip4') - if host is not None: - cidr = IPAddr.FAM_IPv4 - raw = True - else: - host = fail.get('ip6') - if host is not None: - cidr = IPAddr.FAM_IPv6 - raw = True - if host is None: - host = fail.get('dns') - if host is None: + continue + if date is None: + logSys.warning( + "Found a match for %r but no valid date/time " + "found for %r. Please try setting a custom " + "date pattern (see man page jail.conf(5)). " + "If format is complex, please " + "file a detailed issue on" + " https://github.com/fail2ban/fail2ban/issues " + "in order to get support for this format.", + "\n".join(failRegex.getMatchedLines()), timeText) + continue + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + # retrieve failure-id, host, etc from failure match: + try: + raw = returnRawHost + if preGroups: + fail = preGroups.copy() + fail.update(failRegex.getGroups()) + else: + fail = failRegex.getGroups() + # first try to check we have mlfid case (caching of connection id by multi-line): + mlfid = fail.get('mlfid') + if mlfid is not None: + fail = self._mergeFailure(mlfid, fail, failRegex) + else: + # matched lines: + fail["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() + # failure-id: + fid = fail.get('fid') + # ip-address or host: + host = fail.get('ip4') + if host is not None: + cidr = IPAddr.FAM_IPv4 + raw = True + else: + host = fail.get('ip6') + if host is not None: + cidr = IPAddr.FAM_IPv6 + raw = True + if host is None: + host = fail.get('dns') + if host is None: + # first try to check we have mlfid case (cache connection id): + if fid is None: + if mlfid: + fail = self._mergeFailure(mlfid, fail, failRegex) + else: # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: - if fid is None: - fid = failRegex.getFailID() - host = fid - cidr = IPAddr.CIDR_RAW - # if raw - add single ip or failure-id, - # otherwise expand host to multiple ips using dns (or ignore it if not valid): - if raw: - ip = IPAddr(host, cidr) - # check host equal failure-id, if not - failure with complex id: - if fid is not None and fid != host: - ip = IPAddr(fid, IPAddr.CIDR_RAW) - ips = [ip] - else: - ips = DNSUtils.textToIp(host, self.__useDns) - for ip in ips: - failList.append([failRegexIndex, ip, date, - failRegex.getMatchedLines(), fail]) - if not self.checkAllRegex: - break - except RegexException as e: # pragma: no cover - unsure if reachable - logSys.error(e) + fid = failRegex.getFailID() + host = fid + cidr = IPAddr.CIDR_RAW + # if mlfid case (not failure): + if host is None: + if not self.checkAllRegex: # or fail.get('nofail'): + return failList + ips = [None] + # if raw - add single ip or failure-id, + # otherwise expand host to multiple ips using dns (or ignore it if not valid): + elif raw: + ip = IPAddr(host, cidr) + # check host equal failure-id, if not - failure with complex id: + if fid is not None and fid != host: + ip = IPAddr(fid, IPAddr.CIDR_RAW) + ips = [ip] + # otherwise, try to use dns conversion: + else: + ips = DNSUtils.textToIp(host, self.__useDns) + # append failure with match to the list: + for ip in ips: + failList.append([failRegexIndex, ip, date, fail]) + if not self.checkAllRegex: + break + except RegexException as e: # pragma: no cover - unsure if reachable + logSys.error(e) return failList def status(self, flavor="basic"): @@ -751,7 +799,7 @@ class FileFilter(Filter): db = self.jail.database if db is not None: db.updateLog(self.jail, log) - logSys.info("Removed logfile: %r" % path) + logSys.info("Removed logfile: %r", path) self._delLogPath(path) return @@ -816,7 +864,7 @@ class FileFilter(Filter): def getFailures(self, filename): log = self.getLog(filename) if log is None: - logSys.error("Unable to get failures in " + filename) + logSys.error("Unable to get failures in %s", filename) return False # We should always close log (file), otherwise may be locked (log-rotate, etc.) try: @@ -825,11 +873,11 @@ class FileFilter(Filter): has_content = log.open() # see http://python.org/dev/peps/pep-3151/ except IOError as e: - logSys.error("Unable to open %s" % filename) + logSys.error("Unable to open %s", filename) logSys.exception(e) return False except OSError as e: # pragma: no cover - requires race condition to tigger this - logSys.error("Error opening %s" % filename) + logSys.error("Error opening %s", filename) logSys.exception(e) return False except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate @@ -1050,7 +1098,7 @@ class FileContainer: ## sys.stdout.flush() # Compare hash and inode if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.info("Log rotation detected for %s" % self.__filename) + logSys.info("Log rotation detected for %s", self.__filename) self.__hash = myHash self.__ino = stats.st_ino self.__pos = 0 diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index a66b64ac..be205303 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -138,7 +138,8 @@ class Ticket(object): self._data['matches'] = matches or [] def getMatches(self): - return self._data.get('matches', []) + return [(line if isinstance(line, basestring) else "".join(line)) \ + for line in self._data.get('matches', ())] @property def restored(self): @@ -235,7 +236,11 @@ class FailTicket(Ticket): self.__retry += count self._data['failures'] += attempt if matches: - self._data['matches'] += matches + # we should duplicate "matches", because possibly referenced to multiple tickets: + if self._data['matches']: + self._data['matches'] = self._data['matches'] + matches + else: + self._data['matches'] = matches def setLastTime(self, value): if value > self._time: diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 57da495a..b258ae77 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -98,6 +98,12 @@ class Utils(): cache.popitem() cache[k] = (v, t + self.maxTime) + def unset(self, k): + try: + del self._cache[k] + except KeyError: # pragme: no cover + pass + @staticmethod def setFBlockMode(fhandle, value): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 174152b9..8558f8d3 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1431,6 +1431,7 @@ class GetFailures(LogCaptureTestCase): ('no', output_no), ('warn', output_yes) ): + self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) filter_.active = True diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index a97e92c3..b59ccd98 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -150,29 +150,34 @@ def testSampleRegexsFactory(name, basedir): else: faildata = {} - ret = self.filter.processLine(line) - if not ret: - # Check line is flagged as none match - self.assertFalse(faildata.get('match', True), - "Line not matched when should have: %s:%i, line:\n%s" % - (logFile.filename(), logFile.filelineno(), line)) - elif ret: - # Check line is flagged to match - self.assertTrue(faildata.get('match', False), - "Line matched when shouldn't have: %s:%i, line:\n%s" % - (logFile.filename(), logFile.filelineno(), line)) - self.assertEqual(len(ret), 1, "Multiple regexs matched %r - %s:%i" % - (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno())) + try: + ret = self.filter.processLine(line) + if not ret: + # Check line is flagged as none match + self.assertFalse(faildata.get('match', True), + "Line not matched when should have") + continue - # Verify timestamp and host as expected - failregex, host, fail2banTime, lines, fail = ret[0] - self.assertEqual(host, faildata.get("host", None)) - # Verify other captures: + failregex, fid, fail2banTime, fail = ret[0] + # Bypass no failure helpers-regexp: + if not faildata.get('match', False) and (fid is None or fail.get('nofail')): + regexsUsed.add(failregex) + continue + + # Check line is flagged to match + self.assertTrue(faildata.get('match', False), + "Line matched when shouldn't have") + self.assertEqual(len(ret), 1, + "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) + + # Fallback for backwards compatibility (previously no fid, was host only): + if faildata.get("host", None) is not None and fail.get("host", None) is None: + fail["host"] = fid + # Verify match captures (at least fid/host) and timestamp as expected for k, v in faildata.iteritems(): - if k not in ("time", "match", "host", "desc"): + if k not in ("time", "match", "desc"): fv = fail.get(k, None) - self.assertEqual(fv, v, "Value of %s mismatch %r != %r on: %s:%i, line:\n%s" % ( - k, fv, v, logFile.filename(), logFile.filelineno(), line)) + self.assertEqual(fv, v) t = faildata.get("time", None) try: @@ -185,12 +190,15 @@ def testSampleRegexsFactory(name, basedir): jsonTime += jsonTimeLocal.microsecond / 1000000 self.assertEqual(fail2banTime, jsonTime, - "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds) on: %s:%i, line:\n%s" % + "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), - fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) ) + fail2banTime - jsonTime) ) regexsUsed.add(failregex) + except AssertionError as e: # pragma: no cover + raise AssertionError("%s on: %s:%i, line:\n%s" % ( + e, logFile.filename(), logFile.filelineno(), line)) for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): self.assertTrue( From 4efcc293846d18bf9bfcba90b8311607f9126d0a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 22 Feb 2017 20:45:06 +0100 Subject: [PATCH 20/23] coverage of new multi-line handling within fail2ban-regex --- fail2ban/tests/fail2banregextestcase.py | 31 ++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8e6fcfe0..7b08b936 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -27,14 +27,14 @@ import os import sys from ..client import fail2banregex -from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output +from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output, str2LogLevel from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase, logSys from .utils import CONFIG_DIR fail2banregex.logSys = logSys def _test_output(*args): - logSys.info(args[0]) + logSys.notice(args[0]) fail2banregex.output = _test_output @@ -45,6 +45,9 @@ DEV_NULL = None def _Fail2banRegex(*args): parser = get_opt_parser() (opts, args) = parser.parse_args(list(args)) + # put down log-level if expected, because of too many debug-messages: + if opts.log_level in ("notice", "warning"): + logSys.setLevel(str2LogLevel(opts.log_level)) return (opts, args, Fail2banRegex(opts)) class ExitException(Exception): @@ -80,6 +83,7 @@ class Fail2banRegexTest(LogCaptureTestCase): FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") + FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') def setUp(self): @@ -195,8 +199,26 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('141.3.81.106 Sun Aug 14 11:53:59 2005') self.assertLogged('141.3.81.106 Sun Aug 14 11:54:59 2005') - def testWronChar(self): + def testVerboseFullSshd(self): (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages + "-v", "--verbose-date", "--print-all-matched", + Fail2banRegexTest.FILENAME_SSHD, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(args)) + # test failure line and not-failure lines both presents: + self.assertLogged("[29116]: User root not allowed because account is locked", + "[29116]: Received disconnect from 1.2.3.4", all=True) + + def _reset(self): + # reset global warn-counter: + from ..server.filter import _decode_line_warn + _decode_line_warn.clear() + + def testWronChar(self): + self._reset() + (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) @@ -210,12 +232,15 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') def testWronCharDebuggex(self): + self._reset() (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) self.assertTrue(fail2banRegex.start(args)) + self.assertLogged('Error decoding line') self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') self.assertLogged('https://') From d2a3d093c6208a9d71dae27beb5e052060be481b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Feb 2017 11:54:24 +0100 Subject: [PATCH 21/23] rewritten CallingMap: performance optimized, immutable, self-referencing, template possibility (used in new ActionInfo objects); new ActionInfo handling: saves content between actions, without interim copying (save original on demand, recoverable via reset); test cases extended --- config/action.d/smtp.py | 2 +- fail2ban/server/action.py | 66 +++++++++++--- fail2ban/server/actions.py | 148 ++++++++++++++++++------------- fail2ban/tests/actiontestcase.py | 48 +++++++++- fail2ban/tests/servertestcase.py | 2 +- 5 files changed, 186 insertions(+), 80 deletions(-) diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py index aa85b259..9cdfe327 100644 --- a/config/action.d/smtp.py +++ b/config/action.d/smtp.py @@ -123,7 +123,7 @@ class SMTPAction(ActionBase): self.message_values = CallingMap( jailname = self._jail.name, hostname = socket.gethostname, - bantime = self._jail.actions.getBanTime, + bantime = lambda: self._jail.actions.getBanTime(), ) # bypass ban/unban for restored tickets diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index bf291690..e7a98dbb 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -50,9 +50,14 @@ allowed_ipv6 = True # capture groups from filter for map to ticket data: FCUSTAG_CRE = re.compile(r''); # currently uppercase only +# New line, space +ADD_REPL_TAGS = { + "br": "\n", + "sp": " " +} -class CallingMap(MutableMapping): +class CallingMap(MutableMapping, object): """A Mapping type which returns the result of callable values. `CallingMap` behaves similar to a standard python dictionary, @@ -69,23 +74,64 @@ class CallingMap(MutableMapping): The dictionary data which can be accessed to obtain items uncalled """ + # immutable=True saves content between actions, without interim copying (save original on demand, recoverable via reset) + __slots__ = ('data', 'storage', 'immutable', '__org_data') def __init__(self, *args, **kwargs): + self.storage = dict() + self.immutable = True self.data = dict(*args, **kwargs) + def reset(self, immutable=True): + self.storage = dict() + try: + self.data = self.__org_data + except AttributeError: + pass + self.immutable = immutable + def __repr__(self): - return "%s(%r)" % (self.__class__.__name__, self.data) + return "%s(%r)" % (self.__class__.__name__, self._asdict()) + + def _asdict(self): + try: + return dict(self) + except: + return dict(self.data, **self.storage) def __getitem__(self, key): - value = self.data[key] + try: + value = self.storage[key] + except KeyError: + value = self.data[key] if callable(value): - value = value() - self.data[key] = value + # check arguments can be supplied to callable (for backwards compatibility): + value = value(self) if hasattr(value, '__code__') and value.__code__.co_argcount else value() + self.storage[key] = value return value def __setitem__(self, key, value): - self.data[key] = value + # mutate to copy: + if self.immutable: + self.storage = self.storage.copy() + self.__org_data = self.data + self.data = self.data.copy() + self.immutable = False + self.storage[key] = value + + def __unavailable(self, key): + raise KeyError("Key %r was deleted" % key) def __delitem__(self, key): + # mutate to copy: + if self.immutable: + self.storage = self.storage.copy() + self.__org_data = self.data + self.data = self.data.copy() + self.immutable = False + try: + del self.storage[key] + except KeyError: + pass del self.data[key] def __iter__(self): @@ -94,7 +140,7 @@ class CallingMap(MutableMapping): def __len__(self): return len(self.data) - def copy(self): + def copy(self): # pargma: no cover return self.__class__(self.data.copy()) @@ -436,9 +482,6 @@ class CommandAction(ActionBase): # interpolation of dictionary: if subInfo is None: subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) - # New line, space - for (tag, value) in (("br", '\n'), ("sp", " ")): - if subInfo.get(tag) is None: subInfo[tag] = value # cache if possible: if csubkey is not None: cache[csubkey] = subInfo @@ -453,7 +496,8 @@ class CommandAction(ActionBase): if value is None: value = subInfo.get(tag) if value is None: - return m.group() # fallback (no replacement) + # fallback (no or default replacement) + return ADD_REPL_TAGS.get(tag, m.group()) value = str(value) # assure string if tag in cls._escapedTags: # That one needs to be escaped since its content is diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3a5eada9..6b793b8f 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -286,44 +286,86 @@ class Actions(JailThread, Mapping): self.stopActions() return True - def __getBansMerged(self, mi, overalljails=False): - """Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside. + class ActionInfo(CallingMap): - This function never returns None for ainfo lambdas - always a ticket (merged or single one) - and prevents any errors through merging (to guarantee ban actions will be executed). - [TODO] move merging to observer - here we could wait for merge and read already merged info from a database + AI_DICT = { + "ip": lambda self: self.__ticket.getIP(), + "ip-rev": lambda self: self['ip'].getPTR(''), + "fid": lambda self: self.__ticket.getID(), + "failures": lambda self: self.__ticket.getAttempt(), + "time": lambda self: self.__ticket.getTime(), + "matches": lambda self: "\n".join(self.__ticket.getMatches()), + # to bypass actions, that should not be executed for restored tickets + "restored": lambda self: (1 if self.__ticket.restored else 0), + # extra-interpolation - all match-tags (captured from the filter): + "F-*": lambda self, tag=None: self.__ticket.getData(tag), + # merged info: + "ipmatches": lambda self: "\n".join(self._mi4ip(True).getMatches()), + "ipjailmatches": lambda self: "\n".join(self._mi4ip().getMatches()), + "ipfailures": lambda self: self._mi4ip(True).getAttempt(), + "ipjailfailures": lambda self: self._mi4ip().getAttempt(), + } - Parameters - ---------- - mi : dict - merge info, initial for lambda should contains {ip, ticket} - overalljails : bool - switch to get a merged bans : - False - (default) bans merged for current jail only - True - bans merged for all jails of current ip address + __slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip') + + def __init__(self, ticket, jail=None, immutable=True, data=AI_DICT): + self.__ticket = ticket + self.__jail = jail + self.storage = dict() + self.immutable = immutable + self.data = data + + def copy(self): # pargma: no cover + return self.__class__(self.__ticket, self.__jail, self.immutable, self.data.copy()) + + def _mi4ip(self, overalljails=False): + """Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside. + + This function never returns None for ainfo lambdas - always a ticket (merged or single one) + and prevents any errors through merging (to guarantee ban actions will be executed). + [TODO] move merging to observer - here we could wait for merge and read already merged info from a database + + Parameters + ---------- + overalljails : bool + switch to get a merged bans : + False - (default) bans merged for current jail only + True - bans merged for all jails of current ip address + + Returns + ------- + BanTicket + merged or self ticket only + """ + if not hasattr(self, '__mi4ip'): + self.__mi4ip = {} + mi = self.__mi4ip + idx = 'all' if overalljails else 'jail' + if idx in mi: + return mi[idx] if mi[idx] is not None else self.__ticket + try: + jail = self.__jail + ip = self['ip'] + mi[idx] = None + if not jail.database: # pragma: no cover + return self.__ticket + if overalljails: + mi[idx] = jail.database.getBansMerged(ip=ip) + else: + mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail) + except Exception as e: + logSys.error( + "Failed to get %s bans merged, jail '%s': %s", + idx, jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + return mi[idx] if mi[idx] is not None else self.__ticket + + + def __getActionInfo(self, ticket): + ip = ticket.getIP() + aInfo = Actions.ActionInfo(ticket, self._jail) + return aInfo - Returns - ------- - BanTicket - merged or self ticket only - """ - idx = 'all' if overalljails else 'jail' - if idx in mi: - return mi[idx] if mi[idx] is not None else mi['ticket'] - try: - jail=self._jail - ip=mi['ip'] - mi[idx] = None - if overalljails: - mi[idx] = jail.database.getBansMerged(ip=ip) - else: - mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail) - except Exception as e: - logSys.error( - "Failed to get %s bans merged, jail '%s': %s", - idx, jail.name, e, - exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - return mi[idx] if mi[idx] is not None else mi['ticket'] def __checkBan(self): """Check for IP address to ban. @@ -343,24 +385,7 @@ class Actions(JailThread, Mapping): break bTicket = BanManager.createBanTicket(ticket) ip = bTicket.getIP() - aInfo = CallingMap({ - "ip" : ip, - "ip-rev" : lambda: ip.getPTR(''), - "failures": bTicket.getAttempt(), - "time" : bTicket.getTime(), - "matches" : "\n".join(bTicket.getMatches()), - # to bypass actions, that should not be executed for restored tickets - "restored": (1 if ticket.restored else 0), - # extra-interpolation - all match-tags (captured from the filter): - "F-*": lambda tag=None: bTicket.getData(tag) - }) - if self._jail.database is not None: - mi4ip = lambda overalljails=False, self=self, \ - mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails) - aInfo["ipmatches"] = lambda: "\n".join(mi4ip(True).getMatches()) - aInfo["ipjailmatches"] = lambda: "\n".join(mi4ip().getMatches()) - aInfo["ipfailures"] = lambda: mi4ip(True).getAttempt() - aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt() + aInfo = self.__getActionInfo(bTicket) reason = {} if self.__banManager.addBanTicket(bTicket, reason=reason): cnt += 1 @@ -369,7 +394,8 @@ class Actions(JailThread, Mapping): try: if ticket.restored and getattr(action, 'norestored', False): continue - action.ban(aInfo.copy()) + if not aInfo.immutable: aInfo.reset() + action.ban(aInfo) except Exception as e: logSys.error( "Failed to execute ban jail '%s' action '%s' " @@ -452,21 +478,17 @@ class Actions(JailThread, Mapping): unbactions = self._actions else: unbactions = actions - aInfo = dict() - aInfo["ip"] = ticket.getIP() - aInfo["failures"] = ticket.getAttempt() - aInfo["time"] = ticket.getTime() - aInfo["matches"] = "".join(ticket.getMatches()) - # to bypass actions, that should not be executed for restored tickets - aInfo["restored"] = 1 if ticket.restored else 0 + ip = ticket.getIP() + aInfo = self.__getActionInfo(ticket) if actions is None: logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) for name, action in unbactions.iteritems(): try: if ticket.restored and getattr(action, 'norestored', False): continue - logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"]) - action.unban(aInfo.copy()) + logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip) + if not aInfo.immutable: aInfo.reset() + action.unban(aInfo) except Exception as e: logSys.error( "Failed to execute unban jail '%s' action '%s' " diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index ca908b65..8834ab8d 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -194,7 +194,7 @@ class CommandActionTest(LogCaptureTestCase): # Callable self.assertEqual( self.__action.replaceTag("09 11", - CallingMap(matches=lambda: str(10))), + CallingMap(matches=lambda self: str(10))), "09 10 11") def testReplaceNoTag(self): @@ -202,7 +202,7 @@ class CommandActionTest(LogCaptureTestCase): # Will raise ValueError if it is self.assertEqual( self.__action.replaceTag("abc", - CallingMap(matches=lambda: int("a"))), "abc") + CallingMap(matches=lambda self: int("a"))), "abc") def testReplaceTagSelfRecursion(self): setattr(self.__action, 'a', " Date: Fri, 24 Feb 2017 13:22:15 +0100 Subject: [PATCH 22/23] test coverage --- fail2ban/client/fail2banregex.py | 22 +++--- .../filter.d/zzz-sshd-obsolete-multiline.conf | 76 +++++++++++++++++++ fail2ban/tests/fail2banregextestcase.py | 39 ++++++++++ .../files/logs/zzz-sshd-obsolete-multiline | 2 + .../files/zzz-sshd-obsolete-multiline.log | 4 + 5 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf create mode 100644 fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline create mode 100644 fail2ban/tests/files/zzz-sshd-obsolete-multiline.log diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 395fe8ef..e2f1f67c 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -61,7 +61,7 @@ def debuggexURL(sample, regex): 'flavor': 'python' }) return 'https://www.debuggex.com/?' + q -def output(args): +def output(args): # pragma: no cover (overriden in test-cases) print(args) def shortstr(s, l=53): @@ -278,15 +278,15 @@ class Fail2banRegex(object): value = os.path.splitext(os.path.basename(value))[0] output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir) - if not reader.read(): + if not reader.read(): # pragma: no cover output( "ERROR: failed to load filter %s" % value ) return False - else: + else: # pragma: no cover ## foreign file - readexplicit this file and includes if possible: output( "Use %11s file : %s" % (regex, value) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config) reader.setBaseDir(None) - if not reader.readexplicit(): + if not reader.readexplicit(): output( "ERROR: failed to read %s" % value ) return False reader.getOptions(None) @@ -298,7 +298,7 @@ class Fail2banRegex(object): optval = opt[3] elif opt[0] == 'set': optval = opt[3:] - else: + else: # pragma: no cover continue try: if opt[2] == "prefregex": @@ -322,7 +322,7 @@ class Fail2banRegex(object): elif opt[2] == "datepattern": for optval in optval: self.setDatePattern(optval) - elif opt[2] == "addjournalmatch": + elif opt[2] == "addjournalmatch": # pragma: no cover if self._opts.journalmatch is None: self.setJournalMatch(optval) except ValueError as e: # pragma: no cover @@ -350,7 +350,7 @@ class Fail2banRegex(object): if ret is not None: found = True regex = self._ignoreregex[ret].inc() - except RegexException as e: + except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) return False return found @@ -368,7 +368,7 @@ class Fail2banRegex(object): regex = self._failregex[match[0]] regex.inc() regex.appendIP(match) - except RegexException as e: + except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) return False for bufLine in orgLineBuffer[int(fullBuffer):]: @@ -520,9 +520,9 @@ class Fail2banRegex(object): cmd_log, cmd_regex = args[:2] try: - if not self.readRegex(cmd_regex, 'fail'): + if not self.readRegex(cmd_regex, 'fail'): # pragma: no cover return False - if len(args) == 3 and not self.readRegex(args[2], 'ignore'): + if len(args) == 3 and not self.readRegex(args[2], 'ignore'): # pragma: no cover return False except RegexException as e: output( 'ERROR: %s' % e ) @@ -534,7 +534,7 @@ class Fail2banRegex(object): output( "Use log file : %s" % cmd_log ) output( "Use encoding : %s" % self._encoding ) test_lines = self.file_lines_gen(hdlr) - except IOError as e: + except IOError as e: # pragma: no cover output( e ) return False elif cmd_log.startswith("systemd-journal"): # pragma: no cover diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf new file mode 100644 index 00000000..729fecd7 --- /dev/null +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -0,0 +1,76 @@ +# Fail2Ban obsolete multiline example resp. test filter (previously sshd.conf) +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = ../../../../config/filter.d/common.conf + +[DEFAULT] + +_daemon = sshd + +# optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: " +__pref = (?:(?:error|fatal): (?:PAM: )?)? +# optional suffix (logged from several ssh versions) like " [preauth]" +__suff = (?: \[preauth\])?\s* +__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? + +# single line prefix: +__prefix_line_sl = %(__prefix_line)s%(__pref)s +# multi line prefixes (for first and second lines): +__prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s +__prefix_line_ml2 = %(__suff)s$^(?P=__prefix)%(__pref)s + +mode = %(normal)s + +normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ + ^%(__prefix_line_sl)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ + ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ + ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ + ^%(__prefix_line_sl)srefused connect from \S+ \(\)\s*%(__suff)s$ + ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ + ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ + ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ + ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from : 11: .+%(__suff)s$ + ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ + +ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ + ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ + ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$ + ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ + +aggressive = %(normal)s + %(ddos)s + +[Definition] + +failregex = %(mode)s + +ignoreregex = + +# "maxlines" is number of log lines to buffer for multi-line regex searches +maxlines = 10 + +journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd + +datepattern = {^LN-BEG} + +# DEV Notes: +# +# "Failed \S+ for .*? from ..." failregex uses non-greedy catch-all because +# it is coming before use of which is not hard-anchored at the end as well, +# and later catch-all's could contain user-provided input, which need to be greedily +# matched away first. +# +# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black + diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 7b08b936..af7af1bc 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -38,6 +38,7 @@ def _test_output(*args): fail2banregex.output = _test_output +TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config") TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") DEV_NULL = None @@ -85,6 +86,11 @@ class Fail2banRegexTest(LogCaptureTestCase): FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') + FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log') + FILTER_ZZZ_SSHD = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-sshd-obsolete-multiline.conf') + + FILENAME_ZZZ_GEN = os.path.join(TEST_FILES_DIR, "logs", "zzz-generic-example") + FILTER_ZZZ_GEN = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-generic-example.conf') def setUp(self): """Call before every test case.""" @@ -210,6 +216,39 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged("[29116]: User root not allowed because account is locked", "[29116]: Received disconnect from 1.2.3.4", all=True) + def testFastSshd(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages + "--print-all-matched", + Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(args)) + # test failure line and all not-failure lines presents: + self.assertLogged( + "[29116]: Connection from 192.0.2.4", + "[29116]: User root not allowed because account is locked", + "[29116]: Received disconnect from 192.0.2.4", all=True) + + def testMultilineSshd(self): + # by the way test of missing lines by multiline in `for bufLine in orgLineBuffer[int(fullBuffer):]` + (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages + "--print-all-matched", "--print-all-missed", + Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_ZZZ_SSHD + ) + self.assertTrue(fail2banRegex.start(args)) + # test "failure" line presents (2nd part only, because multiline fewer precise): + self.assertLogged( + "[29116]: Received disconnect from 192.0.2.4", all=True) + + def testFullGeneric(self): + # by the way test of ignoreregex (specified in filter file)... + (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages + Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN + ) + self.assertTrue(fail2banRegex.start(args)) + def _reset(self): # reset global warn-counter: from ..server.filter import _decode_line_warn diff --git a/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline b/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline new file mode 100644 index 00000000..d301ad91 --- /dev/null +++ b/fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline @@ -0,0 +1,2 @@ +# test sshd file: +# addFILE: "sshd" diff --git a/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log b/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log new file mode 100644 index 00000000..e0f6cc8a --- /dev/null +++ b/fail2ban/tests/files/zzz-sshd-obsolete-multiline.log @@ -0,0 +1,4 @@ +Apr 27 13:02:01 host sshd[29116]: Connection from 192.0.2.4 port 55555 +Apr 27 13:02:02 host sshd[29116]: User root not allowed because account is locked +Apr 27 13:02:03 host sshd[29116]: input_userauth_request: invalid user root [preauth] +Apr 27 13:02:04 host sshd[29116]: Received disconnect from 192.0.2.4: 11: Normal Shutdown, Thank you for playing [preauth] From 32ac383d06b70c5a095be833548f67a7fa67ab09 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 27 Feb 2017 15:51:33 +0100 Subject: [PATCH 23/23] Update ChangeLog --- ChangeLog | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ChangeLog b/ChangeLog index d720575c..83d61a55 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,45 @@ ver. 0.10.0 (2016/XX/XXX) - gonna-be-released-some-time-shining ----------- TODO: implementing of options resp. other tasks from PR #1346 + documentation should be extended (new options, etc) + +### Fixes +* `filter.d/pam-generic.conf`: + - [grave] injection on user name to host fixed +* `action.d/complain.conf` + - fixed using new tag `` (sh/dash compliant now) + +### New Features +* New Actions: + +* New Filters: + +### Enhancements +* Introduced new filter option `prefregex` for pre-filtering using single regular expression (gh-1698); +* Many times faster and fewer CPU-hungry because of parsing with `maxlines=1`, so without + line buffering (scrolling of the buffer-window). + Combination of tags `` and `` can be used now to process multi-line logs + using single-line expressions: + - tag ``: used to identify resp. store failure info for groups of log-lines with the same + identifier (e. g. combined failure-info for the same conn-id by `(?:conn-id)`, + see sshd.conf for example) + - tag ``: used as mark for no-failure (helper to accumulate common failure-info, + e. g. from lines that contain IP-address); +* Several filters optimized with pre-filtering using new option `prefregex`, and multiline filter + using `` + `` combination; +* Exposes filter group captures in actions (non-recursive interpolation of tags ``, + see gh-1698, gh-1110) +* Some filters extended with user name (can be used in gh-1243 to distinguish IP and user, + resp. to remove after success login the user-related failures only); +* Safer, more stable and faster replaceTag interpolation (switched from cycle over all tags + to re.sub with callable) +* substituteRecursiveTags optimization + moved in helpers facilities (because currently used + commonly in server and in client) +* Provides new tag `` for PTR reversed representation of IP address + + +ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc +----------- ### Fixes * [Grave] memory leak's fixed (gh-1277, gh-1234)