From 66d2436f217ba4587973a71e0e482043653097c2 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Mar 2018 14:16:34 +0100 Subject: [PATCH 01/10] filter.d/sshd.conf: extend suffix with optional port, move it to `prefregex` at end outside of the content --- ChangeLog | 1 + config/filter.d/sshd.conf | 4 ++-- .../config/filter.d/zzz-sshd-obsolete-multiline.conf | 10 +++++----- fail2ban/tests/files/logs/sshd | 2 ++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index ceb3c822..86b407ac 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,7 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition * `filter.d/sshd.conf`: - failregex got an optional space in order to match new log-format (see gh-2061); - fixed ddos-mode regex to match refactored message (some versions can contain port now, see gh-2062); + - fixed root login refused regex (optional port before preauth, gh-2080); * `action.d/badips.py`: implicit convert IPAddr to str, solves an issue "expected string, IPAddr found" (gh-2059); * `action.d/hostsdeny.conf`: fixed IPv6 syntax (enclosed in square brackets, gh-2066); * (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054); diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index d8bb5edf..6b75f9dd 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -21,7 +21,7 @@ _daemon = sshd # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: " __pref = (?:(?:error|fatal): (?:PAM: )?)? # optional suffix (logged from several ssh versions) like " [preauth]" -__suff = (?: \[preauth\])?\s* +__suff = (?: port \d+)?(?: \[preauth\])?\s* __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", @@ -36,7 +36,7 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User not known to the underlying authentication module for .* from \s*%(__suff)s$ ^Failed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^ROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ + ^ROOT LOGIN REFUSED FROM \s*%(__suff)s$ ^[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ ^User .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 5560716d..0379a626 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -14,7 +14,7 @@ _daemon = sshd # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: " __pref = (?:(?:error|fatal): (?:PAM: )?)? # optional suffix (logged from several ssh versions) like " [preauth]" -__suff = (?: \[preauth\])?\s* +__suff = (?: port \d+)?(?: \[preauth\])?\s* __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? # single line prefix: @@ -33,8 +33,8 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ - ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ + ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM %(__suff)s$ + ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ @@ -50,8 +50,8 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for mdre-normal = -mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__on_port_opt)s%(__suff)s - ^%(__prefix_line_sl)sConnection reset by %(__on_port_opt)s%(__suff)s +mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s + ^%(__prefix_line_sl)sConnection reset by %(__suff)s ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e80eb30c..7d93642d 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -24,6 +24,8 @@ Feb 25 14:34:11 belka sshd[31603]: Failed password for invalid user ROOT from aa # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } +Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 port 12345 [preauth] +# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 #4 From 8028d3940d0173582adede443f8692b510465917 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Mar 2018 17:28:24 +0100 Subject: [PATCH 02/10] amend with better match of optional suffix-groups; remove end-anchors for expressions are precise enough (with clear flow, simple branches, without catch-all's, etc.); --- config/filter.d/sshd.conf | 35 ++++++++++--------- .../filter.d/zzz-sshd-obsolete-multiline.conf | 16 ++++----- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 6b75f9dd..100d918c 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -21,8 +21,9 @@ _daemon = sshd # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: " __pref = (?:(?:error|fatal): (?:PAM: )?)? # optional suffix (logged from several ssh versions) like " [preauth]" -__suff = (?: port \d+)?(?: \[preauth\])?\s* -__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? +#__suff = (?: port \d+)?(?: \[preauth\])?\s* +__suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* +__on_port_opt = (?: (?:port \d+|on \S+)){0,2} # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. @@ -32,19 +33,19 @@ __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+) prefregex = ^%(__prefix_line)s%(__pref)s.+$ -cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ - ^User not known to the underlying authentication module for .* from \s*%(__suff)s$ +cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?%(__suff)s$ + ^User not known to the underlying authentication module for .* from %(__suff)s$ ^Failed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^ROOT LOGIN REFUSED FROM \s*%(__suff)s$ - ^[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ - ^User .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ - ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ - ^User .+ from not allowed because not in any group\s*%(__suff)s$ - ^refused connect from \S+ \(\)\s*%(__suff)s$ + ^ROOT LOGIN REFUSED FROM %(__suff)s$ + ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ + ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ + ^User .+ from not allowed because listed in DenyUsers%(__suff)s$ + ^User .+ from not allowed because not in any group%(__suff)s$ + ^refused connect from \S+ \(\) ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ - ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ + ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ + ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s @@ -55,14 +56,14 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* mdre-normal = -mdre-ddos = ^Did not receive identification string from %(__on_port_opt)s%(__suff)s - ^Connection reset by %(__on_port_opt)s%(__suff)s +mdre-ddos = ^Did not receive identification string from + ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: - ^Read from socket failed: Connection reset by peer%(__suff)s + ^Read from socket failed: Connection reset by peer -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. - ^Unable to negotiate a <__alg_match>%(__suff)s$ + ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: mdre-aggressive = %(mdre-ddos)s diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 0379a626..283e725c 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -14,8 +14,8 @@ _daemon = sshd # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: " __pref = (?:(?:error|fatal): (?:PAM: )?)? # optional suffix (logged from several ssh versions) like " [preauth]" -__suff = (?: port \d+)?(?: \[preauth\])?\s* -__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? +__suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* +__on_port_opt = (?: (?:port \d+|on \S+)){0,2} # single line prefix: __prefix_line_sl = %(__prefix_line)s%(__pref)s @@ -33,12 +33,12 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM %(__suff)s$ + ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not in any group\s*%(__suff)s$ - ^%(__prefix_line_sl)srefused connect from \S+ \(\)\s*%(__suff)s$ + ^%(__prefix_line_sl)srefused connect from \S+ \(\) ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ @@ -50,13 +50,13 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for mdre-normal = -mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s - ^%(__prefix_line_sl)sConnection reset by %(__suff)s +mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from + ^%(__prefix_line_sl)sConnection reset by ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match>%(__suff)s$ + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: mdre-aggressive = %(mdre-ddos)s From 5603055a58f16f444a09bf72b5c65e6417795d27 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Mar 2018 21:44:22 +0100 Subject: [PATCH 03/10] failregex: introduced capturing alternate groups, for example non-empty values of `alt_user_1`, `alt_user_2` will overwrite `user` if it is empty (or `alt_host` -> `host`, etc.) --- fail2ban/server/failregex.py | 24 ++++++++++- fail2ban/server/filter.py | 69 ++++++++++++++++--------------- fail2ban/tests/samplestestcase.py | 6 ++- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index d5c9345f..7aa5d3df 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -89,6 +89,11 @@ def mapTag2Opt(tag): except KeyError: return tag.lower() + +# alternate names to be merged, e. g. alt_user_1 -> user ... +ALTNAME_PRE = 'alt_' +ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') + ## # Regular expression class. # @@ -114,6 +119,14 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex + self._altValues = {} + for k in filter( + lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), + self._regexObj.groupindex + ): + n = ALTNAME_CRE.match(k).group(1) + self._altValues[k] = n + self._altValues = list(self._altValues.items()) if len(self._altValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) @@ -248,7 +261,16 @@ class Regex: # def getGroups(self): - return self._matchCache.groupdict() + if not self._altValues: + return self._matchCache.groupdict() + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + fail = self._matchCache.groupdict() + #fail = fail.copy() + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + return fail ## # Returns skipped lines. diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 5b9125ed..8326f049 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -692,43 +692,46 @@ class Filter(JailThread): # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): - if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover - logSys.log(5, " Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex()) - failRegex.search(self.__lineBuffer, orgBuffer) - if not failRegex.hasMatched(): - continue - # The failregex matched. - logSys.log(7, " Matched failregex %d: %s", failRegexIndex, failRegex.getGroups()) - # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: - # The ignoreregex matched. Remove ignored match. - self.__lineBuffer = failRegex.getUnmatchedTupleLines() - logSys.log(7, " Matched ignoreregex and was ignored") - if not self.checkAllRegex: - break - else: - continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), timeText) - continue - self.__lineBuffer = failRegex.getUnmatchedTupleLines() # retrieve failure-id, host, etc from failure match: try: + if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover + logSys.log(5, " Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex()) + failRegex.search(self.__lineBuffer, orgBuffer) + if not failRegex.hasMatched(): + continue + # current failure data (matched group dict): + fail = failRegex.getGroups() + # The failregex matched. + logSys.log(7, " Matched failregex %d: %s", failRegexIndex, fail) + # Checks if we must ignore this match. + if self.ignoreLine(failRegex.getMatchedTupleLines()) \ + is not None: + # The ignoreregex matched. Remove ignored match. + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + logSys.log(7, " Matched ignoreregex and was ignored") + if not self.checkAllRegex: + break + else: + continue + if date is None: + logSys.warning( + "Found a match for %r but no valid date/time " + "found for %r. Please try setting a custom " + "date pattern (see man page jail.conf(5)). " + "If format is complex, please " + "file a detailed issue on" + " https://github.com/fail2ban/fail2ban/issues " + "in order to get support for this format.", + "\n".join(failRegex.getMatchedLines()), timeText) + continue + # we should check all regex (bypass on multi-line, otherwise too complex): + if not self.checkAllRegex or self.getMaxLines() > 1: + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + # merge data if multi-line failure: raw = returnRawHost if preGroups: - fail = preGroups.copy() - fail.update(failRegex.getGroups()) - else: - fail = failRegex.getGroups() + currFail, fail = fail, preGroups.copy() + fail.update(currFail) # first try to check we have mlfid case (caching of connection id by multi-line): mlfid = fail.get('mlfid') if mlfid is not None: diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 5f0a447a..90ce0b7f 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -144,6 +144,7 @@ def testSampleRegexsFactory(name, basedir): regexsUsedRe = set() # process each test-file (note: array filenames can grow during processing): + faildata = {} i = 0 while i < len(filenames): filename = filenames[i]; i += 1; @@ -195,6 +196,7 @@ def testSampleRegexsFactory(name, basedir): regexList = flt.getFailRegex() try: + fail = {} ret = flt.processLine(line) if not ret: # Bypass if filter constraint specified: @@ -246,8 +248,8 @@ def testSampleRegexsFactory(name, basedir): regexsUsedIdx.add(failregex) regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover - raise AssertionError("%s: %s on: %s:%i, line:\n%s" % ( - fltName, e, logFile.filename(), logFile.filelineno(), line)) + raise AssertionError("%s: %s on: %s:%i, line:\n%s\nfaildata:%r, fail:%r" % ( + fltName, e, logFile.filename(), logFile.filelineno(), line, faildata, fail)) # check missing samples for regex using each filter-options combination: for fltName, flt in self._filters.iteritems(): From a9c94686b6aef3e8524de6f06f4bb6a27cea9544 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 09:09:42 +0100 Subject: [PATCH 04/10] fixed multiple regexs matched --- config/filter.d/apache-auth.conf | 4 ++-- config/filter.d/pam-generic.conf | 5 +---- config/filter.d/sshd.conf | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index d9a6fa5e..f2d5f793 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -15,10 +15,10 @@ prefregex = ^%(_apache_error_client)s (?:AH\d+: )?.+$ auth_type = ([A-Z]\w+: )? failregex = ^client (?:denied by server configuration|used wrong authentication scheme)\b - ^user (?:\S*|.*?) (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b + ^user (?!`)(?:\S*|.*?) (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b ^Authorization of user (?:\S*|.*?) to access .*? failed\b ^%(auth_type)suser (?:\S*|.*?): password mismatch\b - ^%(auth_type)suser `(?:[^']*|.*?)' in realm `.+' (not found|denied by provider)\b + ^%(auth_type)suser `(?:[^']*|.*?)' in realm `.+' (auth(?:oriz|entic)ation failure|not found|denied by provider)\b ^%(auth_type)sinvalid nonce .* received - length is not\b ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected\b ^%(auth_type)sunknown algorithm `(?:[^']*|.*?)' received\b diff --git a/config/filter.d/pam-generic.conf b/config/filter.d/pam-generic.conf index ff4ea802..8fd51826 100644 --- a/config/filter.d/pam-generic.conf +++ b/config/filter.d/pam-generic.conf @@ -18,10 +18,7 @@ _daemon = \S+ prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s .+$ -failregex = ^ruser=\S* rhost=\s*$ - ^ruser= rhost=\s+user=\S*\s*$ - ^ruser= rhost=\s+user=.*?\s*$ - ^ruser=.*? rhost=\s*$ +failregex = ^ruser=(?:\S*|.*?) rhost=(?:\s+user=(?:\S*|.*?))?\s*$ ignoreregex = diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 100d918c..b5a997f5 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -35,7 +35,7 @@ prefregex = ^%(__prefix_line)s%(__pref)s.+.* from ( via \S+)?%(__suff)s$ ^User not known to the underlying authentication module for .* from %(__suff)s$ - ^Failed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^ROOT LOGIN REFUSED FROM %(__suff)s$ ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ From 25cc42129a16b3698cc6529be8a45a8f38946629 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 13:09:05 +0100 Subject: [PATCH 05/10] hold all user names affected by interim attempts in order to avoid forget a failures after success login: intruder (as legitimate user) firstly tries to login with another user-name (brute-force), so hopes to reset failure counter by succeeded login; this is fixed and covered in tests now; sshd-filter extended to cover multiple-login attempts (also fully implements gh-2070); --- config/filter.d/sshd.conf | 7 +- fail2ban/server/filter.py | 68 ++++++++++++++----- .../filter.d/zzz-sshd-obsolete-multiline.conf | 5 +- fail2ban/tests/files/logs/sshd | 45 ++++++++++-- fail2ban/tests/samplestestcase.py | 22 ++++-- 5 files changed, 118 insertions(+), 29 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index b5a997f5..aa01c85a 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -29,6 +29,9 @@ __on_port_opt = (?: (?:port \d+|on \S+)){0,2} # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+) +# PAM authentication mechanism, can be overridden, e. g. `filter = sshd[__pam_auth='pam_ldap']`: +__pam_auth = pam_[a-z]+ + [Definition] prefregex = ^%(__prefix_line)s%(__pref)s.+$ @@ -46,13 +49,13 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ - ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s ^Received disconnect from %(__on_port_opt)s:\s*11: ^Connection closed by %(__suff)s$ - ^Accepted publickey for \S+ from (?:\s|$) + ^Accepted \w+ for \S+ from (?:\s|$) mdre-normal = diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8326f049..01c2e059 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -588,31 +588,62 @@ class Filter(JailThread): return ignoreRegexIndex return None + def _updateUsers(self, fail, user=()): + users = fail.get('users') + # only for regex contains user: + if user: + if not users: + fail['users'] = users = set() + users.add(user) + return users + return None + def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None + users = None + nfflgs = 0 + if fail.get('nofail'): nfflgs |= 1 + if fail.get('mlfforget'): nfflgs |= 2 # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] - # update - if not forget (disconnect/reset): - if not fail.get('mlfforget'): - mlfidGroups.update(fail) - else: - self.mlfidCache.unset(mlfid) # remove cached entry - # merge with previous info: - fail2 = mlfidGroups.copy() - fail2.update(fail) - if not fail.get('nofail'): # be sure we've correct current state - try: - del fail2['nofail'] - except KeyError: - pass - fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() - fail = fail2 - elif not fail.get('mlfforget'): + # update users set (hold all users of connect): + users = self._updateUsers(mlfidGroups, fail.get('user')) + # be sure we've correct current state ('nofail' only from last failure) + try: + del mlfidGroups['nofail'] + except KeyError: + pass + # update not empty values: + mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) + fail = mlfidGroups + # if forget (disconnect/reset) - remove cached entry: + if nfflgs & 2: + self.mlfidCache.unset(mlfid) + elif not (nfflgs & 2): # not mlfforget + users = self._updateUsers(fail, fail.get('user')) mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) - if fail.get('nofail'): - fail["matches"] = failRegex.getMatchedTupleLines() + # check users in order to avoid reset failure by multiple logon-attempts: + if users and len(users) > 1: + # we've new user, reset 'nofail' because of multiple users attempts: + try: + del fail['nofail'] + except KeyError: + pass + # merge matches: + if not fail.get('nofail'): # current state (corresponding users) + try: + m = fail.pop("nofail-matches") + m += fail.get("matches", []) + except KeyError: + m = fail.get("matches", []) + if not (nfflgs & 2): # not mlfforget: + m += failRegex.getMatchedTupleLines() + fail["matches"] = m + elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail: + fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines() + # return merged: return fail @@ -738,6 +769,7 @@ class Filter(JailThread): fail = self._mergeFailure(mlfid, fail, failRegex) # bypass if no-failure case: if fail.get('nofail'): + # if not users or len(users) <= 1: logSys.log(7, "Nofail by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure")) if not self.checkAllRegex: return failList diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 283e725c..5717c316 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -27,6 +27,9 @@ __prefix_line_ml2 = %(__suff)s$^(?P=__prefix)%(__pref)s # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+) +# PAM authentication mechanism, can be overridden, e. g. `filter = sshd[__pam_auth='pam_ldap']`: +__pam_auth = pam_[a-z]+ + [Definition] cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ @@ -42,7 +45,7 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ - ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^%(__prefix_line_ml1)s%(__pam_auth)s\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$%(__prefix_line_ml2)sConnection closed ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from %(__on_port_opt)s:\s*11: .+%(__suff)s$ ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s%(__prefix_line_ml2)sConnection closed by %(__suff)s$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7d93642d..8f7ba264 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -134,7 +134,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +Sep 29 17:15:03 spaceman sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -212,9 +212,46 @@ Apr 27 13:02:04 host sshd[29116]: input_userauth_request: invalid user root [pre # failJSON: { "time": "2005-04-27T13:02:04", "match": true , "host": "1.2.3.4", "desc": "No Bye-Bye" } Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank you for playing [preauth] -# Match sshd auth errors on OpenSUSE systems -# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } -2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root +# Match sshd auth errors on OpenSUSE systems (gh-1024) +# failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } +2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root +# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 + +# disable this case for obsolete multi-line filter (zzz-sshd-obsolete...): +# filterOptions: [{"test.condition": "name == 'sshd'"}] + +# 2 methods auth: pam_unix and pam_ldap are used in combination (gh-2070), succeeded after "failure" in first method: +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:20 bar sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.113 user=rda +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:20 bar sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=rda rhost=192.0.2.113 [preauth] +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:20 bar sshd[1556]: Accepted password for rda from 192.0.2.113 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:20 bar sshd[1556]: pam_unix(sshd:session): session opened for user rda by (uid=0) +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113 + +# several attempts, intruder tries to "forget" failed attempts by success login (all 3 attempts with different users): +# failJSON: { "match": false , "desc": "Still no failure (first try)" } +Mar 7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 +# failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } +Mar 7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 +# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +Mar 7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) + +# several attempts, intruder tries to "forget" failed attempts by success login (accepted for other user as in first failed attempt): +# failJSON: { "match": false , "desc": "Still no failure (first try)" } +Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 +# failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } +Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 +# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +Mar 7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116 # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 90ce0b7f..3322b5b0 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -151,6 +151,7 @@ def testSampleRegexsFactory(name, basedir): logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", filename)) + ignoreBlock = False for line in logFile: jsonREMatch = re.match("^#+ ?(failJSON|filterOptions|addFILE):(.+)$", line) if jsonREMatch: @@ -160,9 +161,13 @@ def testSampleRegexsFactory(name, basedir): if jsonREMatch.group(1) == 'filterOptions': # following lines with another filter options: self._filterTests = [] + ignoreBlock = False for opts in (faildata if isinstance(faildata, list) else [faildata]): # unique filter name (using options combination): self.assertTrue(isinstance(opts, dict)) + if opts.get('test.condition'): + ignoreBlock = not eval(opts.get('test.condition')) + del opts['test.condition'] fltName = opts.get('filterName') if not fltName: fltName = str(opts) if opts else '' fltName = name + fltName @@ -179,10 +184,11 @@ def testSampleRegexsFactory(name, basedir): raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) line = next(logFile) - elif line.startswith("#") or not line.strip(): + elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable faildata = {} + if ignoreBlock: continue # if filter options was not yet specified: if not self._filterTests: @@ -224,9 +230,17 @@ def testSampleRegexsFactory(name, basedir): for k, v in faildata.iteritems(): if k not in ("time", "match", "desc", "filter"): fv = fail.get(k, None) - # Fallback for backwards compatibility (previously no fid, was host only): - if k == "host" and fv is None: - fv = fid + if fv is None: + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host": + fv = fid + # special case for attempts counter: + if k == "attempts": + fv = len(fail.get('matches', {})) + # compare sorted (if set) + if isinstance(fv, (set, list, dict)): + self.assertSortedEqual(fv, v) + continue self.assertEqual(fv, v) t = faildata.get("time", None) From 4129f940bbd86064f0a6cbf66c08c5db4e34c8c5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 15:27:59 +0100 Subject: [PATCH 06/10] revert non-empty incremental multi-line failure merge (just simply overwrite method used ATM); revert sshd test case (better to use last given failure-id, so ipv6 instead ipv4, e. g. because of some wrong multi-line-id recognition); improved output on AssertionError in samples-testcase factory. --- fail2ban/server/filter.py | 28 ++++++++++++++++++++++++++-- fail2ban/tests/files/logs/sshd | 2 +- fail2ban/tests/samplestestcase.py | 8 ++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 01c2e059..2b8d89f1 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -598,6 +598,24 @@ class Filter(JailThread): return users return None + # # ATM incremental (non-empty only) merge deactivated ... + # @staticmethod + # def _updateFailure(self, mlfidGroups, fail): + # # reset old failure-ids when new types of id available in this failure: + # fids = set() + # for k in ('fid', 'ip4', 'ip6', 'dns'): + # if fail.get(k): + # fids.add(k) + # if fids: + # for k in ('fid', 'ip4', 'ip6', 'dns'): + # if k not in fids: + # try: + # del mlfidGroups[k] + # except: + # pass + # # update not empty values: + # mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) + def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None users = None @@ -614,8 +632,14 @@ class Filter(JailThread): del mlfidGroups['nofail'] except KeyError: pass - # update not empty values: - mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) + # # ATM incremental (non-empty only) merge deactivated (for future version only), + # # it can be simulated using alternate value tags, like ..., + # # so previous value 'val' will be overwritten only if 'alt_val' is not empty... + # _updateFailure(mlfidGroups, fail) + # + # overwrite multi-line failure with all values, available in fail: + mlfidGroups.update(fail) + # new merged failure data: fail = mlfidGroups # if forget (disconnect/reset) - remove cached entry: if nfflgs & 2: diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 8f7ba264..7526c32b 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -134,7 +134,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -Sep 29 17:15:03 spaceman sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 3322b5b0..1d326c31 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -262,8 +262,12 @@ def testSampleRegexsFactory(name, basedir): regexsUsedIdx.add(failregex) regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover - raise AssertionError("%s: %s on: %s:%i, line:\n%s\nfaildata:%r, fail:%r" % ( - fltName, e, logFile.filename(), logFile.filelineno(), line, faildata, fail)) + import pprint + raise AssertionError("%s: %s on: %s:%i, line:\n%s\n" + "faildata: %s\nfail: %s" % ( + fltName, e, logFile.filename(), logFile.filelineno(), line, + '\n'.join(pprint.pformat(faildata).splitlines()), + '\n'.join(pprint.pformat(fail).splitlines()))) # check missing samples for regex using each filter-options combination: for fltName, flt in self._filters.iteritems(): From c31eb1c562f2797aeb72c9d56e47cf134b5d504c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 16:00:21 +0100 Subject: [PATCH 07/10] quick optimization: normalizes pam-generic prefregex (more similar to the same regex within sshd-filter) + datepattern anchored now; --- config/filter.d/pam-generic.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/pam-generic.conf b/config/filter.d/pam-generic.conf index 8fd51826..0cadbeee 100644 --- a/config/filter.d/pam-generic.conf +++ b/config/filter.d/pam-generic.conf @@ -16,12 +16,14 @@ _ttys_re=\S* __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:? _daemon = \S+ -prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s .+$ +prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure;(?:\s+(?:(?:logname|e?uid)=\S*)){0,3} tty=%(_ttys_re)s .+$ failregex = ^ruser=(?:\S*|.*?) rhost=(?:\s+user=(?:\S*|.*?))?\s*$ ignoreregex = +datepattern = {^LN-BEG} + # DEV Notes: # # for linux-pam before 0.99.2.0 (late 2005) (removed before 0.8.11 release) From ed7d5d8ea1de30206956817666a0f96dd6441085 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 16:04:42 +0100 Subject: [PATCH 08/10] ChangeLog updated --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 86b407ac..beb38482 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,9 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition - failregex got an optional space in order to match new log-format (see gh-2061); - fixed ddos-mode regex to match refactored message (some versions can contain port now, see gh-2062); - fixed root login refused regex (optional port before preauth, gh-2080); + - avoid banning of legitimate users when pam_unix used in combination with other password method, so + bypass pam_unix failures if accepted available for this user gh-2070; + - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediatelly); * `action.d/badips.py`: implicit convert IPAddr to str, solves an issue "expected string, IPAddr found" (gh-2059); * `action.d/hostsdeny.conf`: fixed IPv6 syntax (enclosed in square brackets, gh-2066); * (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054); From cd7f1354c6cdda1f4cc26071537c53a82730d496 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 20 Mar 2018 18:47:42 +0100 Subject: [PATCH 09/10] remove end-anchors for expressions that are precise enough (with clear flow, simple branches, without catch-all's, etc.) --- config/filter.d/sshd.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index aa01c85a..d6f39bc5 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -40,7 +40,7 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User not known to the underlying authentication module for .* from %(__suff)s$ ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^ROOT LOGIN REFUSED FROM %(__suff)s$ + ^ROOT LOGIN REFUSED FROM ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ ^User .+ from not allowed because listed in DenyUsers%(__suff)s$ @@ -52,9 +52,9 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s + ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ ^Received disconnect from %(__on_port_opt)s:\s*11: - ^Connection closed by %(__suff)s$ + ^Connection closed by ^Accepted \w+ for \S+ from (?:\s|$) mdre-normal = From 80725ae8707251e3b2abfa023c1d14029745e35a Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 20 Mar 2018 19:02:44 +0100 Subject: [PATCH 10/10] Update sshd comment/minimalistic: no functional change --- fail2ban/tests/files/logs/sshd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7526c32b..28b2f065 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -218,8 +218,8 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } 2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 -# disable this case for obsolete multi-line filter (zzz-sshd-obsolete...): -# filterOptions: [{"test.condition": "name == 'sshd'"}] +# disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...): +# filterOptions: [{"test.condition":"name=='sshd'"}] # 2 methods auth: pam_unix and pam_ldap are used in combination (gh-2070), succeeded after "failure" in first method: # failJSON: { "match": false , "desc": "No failure" }