From f13fac5ae9a4e219d88d1d01f5d7af46e1ac5e46 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 00:15:57 +0100 Subject: [PATCH 1/4] amend to 5561423be3b2d4636f5484183c3ad470fd326d06: fixed incorrect failure counting despite the `` marked regex; extra: introduced new tag `` as mark to forget current multi-line MLFID (e. g. connection closed); Closes gh-1727 --- config/filter.d/sshd.conf | 14 +++++++------- fail2ban/server/filter.py | 32 ++++++++++++++++++++++---------- fail2ban/tests/files/logs/sshd | 5 +++++ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 320ab59c..95915fcc 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -37,24 +37,24 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ ^User .+ from not allowed because not in any group\s*%(__suff)s$ ^refused connect from \S+ \(\)\s*%(__suff)s$ - ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ + ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ ^User .+ not allowed because account is locked%(__suff)s - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s - ^Received disconnect from : 11: - ^Connection closed by %(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s + ^Received disconnect from : 11: + ^Connection closed by %(__suff)s$ mdre-normal = mdre-ddos = ^Did not receive identification string from %(__suff)s$ - ^Connection reset by %(__on_port_opt)s%(__suff)s + ^Connection reset by %(__on_port_opt)s%(__suff)s ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: - ^Read from socket failed: Connection reset by peer%(__suff)s + ^Read from socket failed: Connection reset by peer%(__suff)s -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$ diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ca2dae86..cbdb4857 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -554,20 +554,29 @@ class Filter(JailThread): mlfidGroups = mlfidFail[1] # if current line not failure, but previous was failure: if fail.get('nofail') and not mlfidGroups.get('nofail'): - del fail['nofail'] # remove nofail flag - was already market as failure + del fail['nofail'] # remove nofail flag - completed with fid (host, ip) self.mlfidCache.unset(mlfid) # remove cache entry # if current line is failure, but previous was not: elif not fail.get('nofail') and mlfidGroups.get('nofail'): - del mlfidGroups['nofail'] # remove nofail flag + del mlfidGroups['nofail'] # remove nofail flag - completed as failure self.mlfidCache.unset(mlfid) # remove cache entry + else: + # cache this line info (if not forget): + if not fail.get('mlfforget'): + mlfidFail = [self.__lastDate, fail] + self.mlfidCache.set(mlfid, mlfidFail) + else: + self.mlfidCache.unset(mlfid) # remove cache entry + return fail fail2 = mlfidGroups.copy() fail2.update(fail) fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() fail = fail2 - elif fail.get('nofail'): - fail["matches"] = failRegex.getMatchedTupleLines() + elif not fail.get('mlfforget'): mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) + if fail.get('nofail'): + fail["matches"] = failRegex.getMatchedTupleLines() return fail @@ -683,6 +692,11 @@ class Filter(JailThread): mlfid = fail.get('mlfid') if mlfid is not None: fail = self._mergeFailure(mlfid, fail, failRegex) + # bypass if no-failure case: + if fail.get('nofail'): + logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure", + mlfid, failRegexIndex) + if not self.checkAllRegex: return failList else: # matched lines: fail["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() @@ -702,18 +716,16 @@ class Filter(JailThread): host = fail.get('dns') if host is None: # first try to check we have mlfid case (cache connection id): - if fid is None: - if mlfid: - fail = self._mergeFailure(mlfid, fail, failRegex) - else: + if fid is None and mlfid is None: # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: fid = failRegex.getFailID() host = fid cidr = IPAddr.CIDR_RAW # if mlfid case (not failure): if host is None: - if not self.checkAllRegex: # or fail.get('nofail'): - return failList + logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier", + mlfid, failRegexIndex) + if not self.checkAllRegex: return failList ips = [None] # if raw - add single ip or failure-id, # otherwise expand host to multiple ips using dns (or ignore it if not valid): diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index fe19591c..6f9a1468 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -113,6 +113,11 @@ May 27 00:16:33 host sshd[2364]: Received disconnect from 198.51.100.76: 11: Bye # failJSON: { "time": "2004-09-29T16:28:02", "match": true , "host": "127.0.0.1" } Sep 29 16:28:02 spaceman sshd[16699]: Failed password for dan from 127.0.0.1 port 45416 ssh1 +# failJSON: { "match": false, "desc": "no failure, just cache mlfid (conn-id)" } +Sep 29 16:28:05 localhost sshd[16700]: Connection from 192.0.2.5 +# failJSON: { "match": false, "desc": "no failure, just covering mlfid (conn-id) forget" } +Sep 29 16:28:05 localhost sshd[16700]: Connection closed by 192.0.2.5 [preauth] + # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1" } Sep 29 17:15:02 spaceman sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: RSA 8c:e3:aa:0f:64:51:02:f7:14:79:89:3f:65:84:7c:30, client user "dan", client host "localhost.localdomain" From 1971fd4bd3eda0bfe91ee688ee41906af93043fc Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 00:30:40 +0100 Subject: [PATCH 2/4] don't remove MLFID from cache (can recognize multiple attempt within the same connection) --- fail2ban/client/jailreader.py | 4 ++-- fail2ban/server/filter.py | 26 +++++++++++--------------- fail2ban/tests/files/logs/sshd | 4 +++- fail2ban/tests/samplestestcase.py | 6 +++--- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 2bef2c4f..bdb3564f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -220,8 +220,8 @@ class JailReader(ConfigReader): if self.__filter: stream.extend(self.__filter.convert()) for opt, value in self.__opts.iteritems(): - if opt == "logpath" and \ - not self.__opts.get('backend', None).startswith("systemd"): + if opt == "logpath": + if self.__opts.get('backend', None).startswith("systemd"): continue found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cbdb4857..76787cc7 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -550,26 +550,22 @@ class Filter(JailThread): def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None + # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] - # if current line not failure, but previous was failure: - if fail.get('nofail') and not mlfidGroups.get('nofail'): - del fail['nofail'] # remove nofail flag - completed with fid (host, ip) - self.mlfidCache.unset(mlfid) # remove cache entry - # if current line is failure, but previous was not: - elif not fail.get('nofail') and mlfidGroups.get('nofail'): - del mlfidGroups['nofail'] # remove nofail flag - completed as failure - self.mlfidCache.unset(mlfid) # remove cache entry + # update - if not forget (disconnect/reset): + if not fail.get('mlfforget'): + mlfidGroups.update(fail) else: - # cache this line info (if not forget): - if not fail.get('mlfforget'): - mlfidFail = [self.__lastDate, fail] - self.mlfidCache.set(mlfid, mlfidFail) - else: - self.mlfidCache.unset(mlfid) # remove cache entry - return fail + self.mlfidCache.unset(mlfid) # remove cached entry + # merge with previous info: fail2 = mlfidGroups.copy() fail2.update(fail) + if not fail.get('nofail'): # be sure we've correct current state + try: + del fail2['nofail'] + except KeyError: + pass fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines() fail = fail2 elif not fail.get('mlfforget'): diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 6f9a1468..f465c9a7 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -238,4 +238,6 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 # failJSON: { "match": false } Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } -Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file +Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" } +Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 00a8f305..5c45e729 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -200,13 +200,13 @@ def testSampleRegexsFactory(name, basedir): self.assertEqual(len(ret), 1, "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) - # Fallback for backwards compatibility (previously no fid, was host only): - if faildata.get("host", None) is not None and fail.get("host", None) is None: - fail["host"] = fid # Verify match captures (at least fid/host) and timestamp as expected for k, v in faildata.iteritems(): if k not in ("time", "match", "desc"): fv = fail.get(k, None) + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host" and fv is None: + fv = fid self.assertEqual(fv, v) t = faildata.get("time", None) From b6886f2e519829dbf6124eb64e7887f3b6d6f171 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 09:42:27 +0100 Subject: [PATCH 3/4] SampleRegexsFactory extended with optional filter constraint, if testing the same log-file with multiple filters (no possibility to match by the old sshd-filter 'zzz-sshd-obsolete-multiline') --- fail2ban/tests/files/logs/sshd | 4 ++-- fail2ban/tests/samplestestcase.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index f465c9a7..fb3defea 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -239,5 +239,5 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "desc": "Second attempt within the same connect" } -Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 5c45e729..0ba11c2e 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -182,6 +182,9 @@ def testSampleRegexsFactory(name, basedir): try: ret = self.filter.processLine(line) if not ret: + # Bypass if filter constraint specified: + if faildata.get('filter') and name != faildata.get('filter'): + continue # Check line is flagged as none match self.assertFalse(faildata.get('match', True), "Line not matched when should have") @@ -202,7 +205,7 @@ def testSampleRegexsFactory(name, basedir): # Verify match captures (at least fid/host) and timestamp as expected for k, v in faildata.iteritems(): - if k not in ("time", "match", "desc"): + if k not in ("time", "match", "desc", "filter"): fv = fail.get(k, None) # Fallback for backwards compatibility (previously no fid, was host only): if k == "host" and fv is None: From 43d2cae8dae53eb51d3841450de6b0a03a2e1218 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Mar 2017 10:17:16 +0100 Subject: [PATCH 4/4] small amend that correct log trace output by forget MLFID (outputs the reason why it was forgotten - close, disconnect, etc.) --- fail2ban/server/filter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 76787cc7..72bf47d8 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -690,8 +690,8 @@ class Filter(JailThread): fail = self._mergeFailure(mlfid, fail, failRegex) # bypass if no-failure case: if fail.get('nofail'): - logSys.log(7, "Nofail by mlfid %r in regex %s: waiting for failure", - mlfid, failRegexIndex) + logSys.log(7, "Nofail by mlfid %r in regex %s: %s", + mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure")) if not self.checkAllRegex: return failList else: # matched lines: @@ -719,8 +719,8 @@ class Filter(JailThread): cidr = IPAddr.CIDR_RAW # if mlfid case (not failure): if host is None: - logSys.log(7, "No failure-id by mlfid %r in regex %s: waiting for identifier", - mlfid, failRegexIndex) + logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", + mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) if not self.checkAllRegex: return failList ips = [None] # if raw - add single ip or failure-id,