Merge pull request #2090 from sebres/fix-sshd-filter-suff

sshd, multi-line failures, alternate groups capture, etc.

ChangeLog

@@ -41,6 +41,10 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition
 * `filter.d/sshd.conf`:
   - failregex got an optional space in order to match new log-format (see gh-2061);
   - fixed ddos-mode regex to match refactored message (some versions can contain port now, see gh-2062);
+  - fixed root login refused regex (optional port before preauth, gh-2080);
+  - avoid banning of legitimate users when pam_unix used in combination with other password method, so
+    bypass pam_unix failures if accepted available for this user gh-2070;
+  - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediatelly);
 * `action.d/badips.py`: implicit convert IPAddr to str, solves an issue "expected string, IPAddr found" (gh-2059);
 * `action.d/hostsdeny.conf`: fixed IPv6 syntax (enclosed in square brackets, gh-2066);
 * (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054);

config/filter.d/apache-auth.conf

@@ -15,10 +15,10 @@ prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$
 auth_type = ([A-Z]\w+: )?
 
 failregex = ^client (?:denied by server configuration|used wrong authentication scheme)\b
-            ^user <F-USER>(?:\S*|.*?)</F-USER> (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b
+            ^user (?!`)<F-USER>(?:\S*|.*?)</F-USER> (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b
             ^Authorization of user <F-USER>(?:\S*|.*?)</F-USER> to access .*? failed\b
             ^%(auth_type)suser <F-USER>(?:\S*|.*?)</F-USER>: password mismatch\b
-            ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (not found|denied by provider)\b
+            ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (auth(?:oriz|entic)ation failure|not found|denied by provider)\b
             ^%(auth_type)sinvalid nonce .* received - length is not\b
             ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected\b
             ^%(auth_type)sunknown algorithm `(?:[^']*|.*?)' received\b

config/filter.d/pam-generic.conf

@@ -16,15 +16,14 @@ _ttys_re=\S*
 __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:?
 _daemon = \S+
 
-prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure;(?:\s+(?:(?:logname|e?uid)=\S*)){0,3} tty=%(_ttys_re)s <F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^ruser=<F-USER>\S*</F-USER> rhost=<HOST>\s*$
-            ^ruser= rhost=<HOST>\s+user=<F-USER>\S*</F-USER>\s*$
-            ^ruser= rhost=<HOST>\s+user=<F-USER>.*?</F-USER>\s*$
-            ^ruser=<F-USER>.*?</F-USER> rhost=<HOST>\s*$
+failregex = ^ruser=<F-ALT_USER>(?:\S*|.*?)</F-ALT_USER> rhost=<HOST>(?:\s+user=<F-USER>(?:\S*|.*?)</F-USER>)?\s*$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 #
 # for linux-pam before 0.99.2.0 (late 2005) (removed before 0.8.11 release)

config/filter.d/sshd.conf

@@ -21,48 +21,52 @@ _daemon = sshd
 # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
 __pref = (?:(?:error|fatal): (?:PAM: )?)?
 # optional suffix (logged from several ssh versions) like " [preauth]"
-__suff = (?: \[preauth\])?\s*
-__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
+#__suff = (?: port \d+)?(?: \[preauth\])?\s*
+__suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s*
+__on_port_opt = (?: (?:port \d+|on \S+)){0,2}
 
 # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
 # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
 __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
 
+# PAM authentication mechanism, can be overridden, e. g. `filter = sshd[__pam_auth='pam_ldap']`:
+__pam_auth = pam_[a-z]+
+
 [Definition]
 
 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
-            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
-            ^Failed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$
+            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$
+            ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
             ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-            ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
-            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
-            ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+            ^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST>
+            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
+            ^refused connect from \S+ \(<HOST>\)
             ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
-            ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
+            ^<F-NOFAIL>%(__pam_auth)s\(sshd:auth\):\s+authentication failure;</F-NOFAIL>(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=<F-ALT_USER>\S*</F-ALT_USER>\s+rhost=<HOST>(?:\s+user=<F-USER>\S*</F-USER>)?%(__suff)s$
             ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
             ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
-            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
+            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s$
             ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>%(__on_port_opt)s:\s*11:
-            ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by <HOST>%(__suff)s$
-            ^<F-MLFFORGET><F-NOFAIL>Accepted publickey</F-NOFAIL></F-MLFFORGET> for \S+ from <HOST>(?:\s|$)
+            ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by <HOST>
+            ^<F-MLFFORGET><F-NOFAIL>Accepted \w+</F-NOFAIL></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$)
 
 mdre-normal =
 
-mdre-ddos = ^Did not receive identification string from <HOST>%(__on_port_opt)s%(__suff)s
-            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>%(__on_port_opt)s%(__suff)s
+mdre-ddos = ^Did not receive identification string from <HOST>
+            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
-            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer%(__suff)s
+            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer
 
-mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
             ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
-            ^Unable to negotiate a <__alg_match>%(__suff)s$
+            ^Unable to negotiate a <__alg_match>
             ^no matching <__alg_match> found:
 
 mdre-aggressive = %(mdre-ddos)s

fail2ban/server/failregex.py

@@ -89,6 +89,11 @@ def mapTag2Opt(tag):
 	except KeyError:
 		return tag.lower()
 
+
+# alternate names to be merged, e. g. alt_user_1 -> user ...
+ALTNAME_PRE = 'alt_'
+ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$')
+
 ##
 # Regular expression class.
 #
@@ -114,6 +119,14 @@ class Regex:
 		try:
 			self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
 			self._regex = regex
+			self._altValues = {}
+			for k in filter(
+				lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE),
+				self._regexObj.groupindex
+			):
+				n = ALTNAME_CRE.match(k).group(1)
+				self._altValues[k] = n
+			self._altValues = list(self._altValues.items()) if len(self._altValues) else None
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
 								 regex)
@@ -257,7 +270,16 @@ class Regex:
 	#
 
 	def getGroups(self):
-		return self._matchCache.groupdict()
+		if not self._altValues:
+			return self._matchCache.groupdict()
+		# merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'):
+		fail = self._matchCache.groupdict()
+		#fail = fail.copy()
+		for k,n in self._altValues:
+			v = fail.get(k)
+			if v and not fail.get(n):
+				fail[n] = v
+		return fail
 
 	##
 	# Returns skipped lines.

fail2ban/server/filter.py

@@ -589,31 +589,86 @@ class Filter(JailThread):
 				return ignoreRegexIndex
 		return None
 
+	def _updateUsers(self, fail, user=()):
+		users = fail.get('users')
+		# only for regex contains user:
+		if user:
+			if not users:
+				fail['users'] = users = set()
+			users.add(user)
+			return users
+		return None
+
+	# # ATM incremental (non-empty only) merge deactivated ...
+	# @staticmethod
+	# def _updateFailure(self, mlfidGroups, fail):
+	# 	# reset old failure-ids when new types of id available in this failure:
+	# 	fids = set()
+	# 	for k in ('fid', 'ip4', 'ip6', 'dns'):
+	# 		if fail.get(k):
+	# 			fids.add(k)
+	# 	if fids:
+	# 		for k in ('fid', 'ip4', 'ip6', 'dns'):
+	# 			if k not in fids:
+	# 				try:
+	# 					del mlfidGroups[k]
+	# 				except:
+	# 					pass
+	# 	# update not empty values:
+	# 	mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v))
+
 	def _mergeFailure(self, mlfid, fail, failRegex):
 		mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
+		users = None
+		nfflgs = 0
+		if fail.get('nofail'): nfflgs |= 1
+		if fail.get('mlfforget'): nfflgs |= 2
 		# if multi-line failure id (connection id) known:
 		if mlfidFail:
 			mlfidGroups = mlfidFail[1]
-			# update - if not forget (disconnect/reset):
-			if not fail.get('mlfforget'):
-				mlfidGroups.update(fail)
-			else:
-				self.mlfidCache.unset(mlfid) # remove cached entry
-			# merge with previous info:
-			fail2 = mlfidGroups.copy()
-			fail2.update(fail)
-			if not fail.get('nofail'): # be sure we've correct current state
-				try:
-					del fail2['nofail']
-				except KeyError:
-					pass
-			fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
-			fail = fail2
-		elif not fail.get('mlfforget'):
+			# update users set (hold all users of connect):
+			users = self._updateUsers(mlfidGroups, fail.get('user'))
+			# be sure we've correct current state ('nofail' only from last failure)
+			try:
+				del mlfidGroups['nofail']
+			except KeyError:
+				pass
+			# # ATM incremental (non-empty only) merge deactivated (for future version only),
+			# # it can be simulated using alternate value tags, like <F-ALT_VAL>...</F-ALT_VAL>,
+			# # so previous value 'val' will be overwritten only if 'alt_val' is not empty...		
+			# _updateFailure(mlfidGroups, fail)
+			#
+			# overwrite multi-line failure with all values, available in fail:
+			mlfidGroups.update(fail)
+			# new merged failure data:
+			fail = mlfidGroups
+			# if forget (disconnect/reset) - remove cached entry:
+			if nfflgs & 2:
+				self.mlfidCache.unset(mlfid)
+		elif not (nfflgs & 2): # not mlfforget
+			users = self._updateUsers(fail, fail.get('user'))
 			mlfidFail = [self.__lastDate, fail]
 			self.mlfidCache.set(mlfid, mlfidFail)
-			if fail.get('nofail'):
-				fail["matches"] = failRegex.getMatchedTupleLines()
+		# check users in order to avoid reset failure by multiple logon-attempts:
+		if users and len(users) > 1:
+			# we've new user, reset 'nofail' because of multiple users attempts:
+			try:
+				del fail['nofail']
+			except KeyError:
+				pass
+		# merge matches:
+		if not fail.get('nofail'): # current state (corresponding users)
+			try:
+				m = fail.pop("nofail-matches")
+				m += fail.get("matches", [])
+			except KeyError:
+				m = fail.get("matches", [])
+			if not (nfflgs & 2): # not mlfforget:
+				m += failRegex.getMatchedTupleLines()
+			fail["matches"] = m
+		elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail:
+			fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines()
+		# return merged:
 		return fail
 
 
@@ -702,8 +757,10 @@ class Filter(JailThread):
 				failRegex.search(buf, orgBuffer)
 				if not failRegex.hasMatched():
 					continue
+				# current failure data (matched group dict):
+				fail = failRegex.getGroups()
 				# The failregex matched.
-				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, failRegex.getGroups())
+				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, fail)
 				# Checks if we must ignore this match.
 				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
 						is not None:
@@ -725,14 +782,14 @@ class Filter(JailThread):
 						"in order to get support for this format.",
 						 "\n".join(failRegex.getMatchedLines()), timeText)
 					continue
-				self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
-				# retrieve failure-id, host, etc from failure match:
+				# we should check all regex (bypass on multi-line, otherwise too complex):
+				if not self.checkAllRegex or self.getMaxLines() > 1:
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
+				# merge data if multi-line failure:
 				raw = returnRawHost
 				if preGroups:
-					fail = preGroups.copy()
-					fail.update(failRegex.getGroups())
-				else:
-					fail = failRegex.getGroups()
+					currFail, fail = fail, preGroups.copy()
+					fail.update(currFail)
 				# first try to check we have mlfid case (caching of connection id by multi-line):
 				mlfid = fail.get('mlfid')
 				if mlfid is not None:

fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf

@@ -14,8 +14,8 @@ _daemon = sshd
 # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
 __pref = (?:(?:error|fatal): (?:PAM: )?)?
 # optional suffix (logged from several ssh versions) like " [preauth]"
-__suff = (?: \[preauth\])?\s*
-__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
+__suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s*
+__on_port_opt = (?: (?:port \d+|on \S+)){0,2}
 
 # single line prefix:
 __prefix_line_sl = %(__prefix_line)s%(__pref)s
@@ -27,22 +27,25 @@ __prefix_line_ml2 = %(__suff)s$<SKIPLINES>^(?P=__prefix)%(__pref)s
 # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
 __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
 
+# PAM authentication mechanism, can be overridden, e. g. `filter = sshd[__pam_auth='pam_ldap']`:
+__pam_auth = pam_[a-z]+
+
 [Definition]
 
 cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
          ^%(__prefix_line_sl)sFailed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
          ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-         ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
-         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__on_port_opt)s\s*$
+         ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM <HOST>
+         ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__suff)s$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not in any group\s*%(__suff)s$
-         ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+         ^%(__prefix_line_sl)srefused connect from \S+ \(<HOST>\)
          ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
          ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
-         ^%(__prefix_line_sl)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$
+         ^%(__prefix_line_ml1)s%(__pam_auth)s\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=<HOST>\s.*%(__suff)s$%(__prefix_line_ml2)sConnection closed
          ^%(__prefix_line_sl)s(error: )?maximum authentication attempts exceeded for .* from <HOST>%(__on_port_opt)s(?: ssh\d*)? \[preauth\]$
          ^%(__prefix_line_ml1)sUser .+ not allowed because account is locked%(__prefix_line_ml2)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*11: .+%(__suff)s$
          ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures(?: for .+?)?%(__suff)s%(__prefix_line_ml2)sConnection closed by <HOST>%(__suff)s$
@@ -50,13 +53,13 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for
 
 mdre-normal =
 
-mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HOST>%(__on_port_opt)s%(__suff)s
-             ^%(__prefix_line_sl)sConnection reset by <HOST>%(__on_port_opt)s%(__suff)s
+mdre-ddos =  ^%(__prefix_line_sl)sDid not receive identification string from <HOST>
+             ^%(__prefix_line_sl)sConnection reset by <HOST>
              ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$
 
-mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
              ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
-             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match>%(__suff)s$
+             ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match>
              ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found:
 
 mdre-aggressive = %(mdre-ddos)s

fail2ban/tests/files/logs/sshd

@@ -24,6 +24,8 @@ Feb 25 14:34:11 belka sshd[31603]: Failed password for invalid user ROOT from aa
 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
 Jan  5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4
 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
+Jan  5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 port 12345 [preauth]
+# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
 Jan  5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4
 
 #4
@@ -210,9 +212,46 @@ Apr 27 13:02:04 host sshd[29116]: input_userauth_request: invalid user root [pre
 # failJSON: { "time": "2005-04-27T13:02:04", "match": true , "host": "1.2.3.4", "desc": "No Bye-Bye" }
 Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank you for playing [preauth]
 
-# Match sshd auth errors on OpenSUSE systems
-# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" }
-2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217  user=root
+# Match sshd auth errors on OpenSUSE systems (gh-1024)
+# failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" }
+2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112  user=root
+# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" }
+2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112
+
+# disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...):
+# filterOptions: [{"test.condition":"name=='sshd'"}]
+
+# 2 methods auth: pam_unix and pam_ldap are used in combination (gh-2070), succeeded after "failure" in first method:
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:20 bar sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.113  user=rda
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:20 bar sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=rda rhost=192.0.2.113 [preauth]
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:20 bar sshd[1556]: Accepted password for rda from 192.0.2.113 port 52100 ssh2
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:20 bar sshd[1556]: pam_unix(sshd:session): session opened for user rda by (uid=0)
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113
+
+# several attempts, intruder tries to "forget" failed attempts by success login (all 3 attempts with different users):
+# failJSON: { "match": false , "desc": "Still no failure (first try)" }
+Mar  7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114
+# failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" }
+Mar  7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114
+# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" }
+Mar  7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0)
+
+# several attempts, intruder tries to "forget" failed attempts by success login (accepted for other user as in first failed attempt):
+# failJSON: { "match": false , "desc": "Still no failure (first try)" }
+Mar  7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
+# failJSON: { "match": false , "desc": "Still no failure (second try, same user)" }
+Mar  7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
+# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" }
+Mar  7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2
+# failJSON: { "match": false , "desc": "No failure" }
+Mar  7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116
 
 # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}]
 

fail2ban/tests/samplestestcase.py

@@ -144,12 +144,14 @@ def testSampleRegexsFactory(name, basedir):
 		regexsUsedRe = set()
 
 		# process each test-file (note: array filenames can grow during processing):
+		faildata = {}
 		i = 0
 		while i < len(filenames):
 			filename = filenames[i]; i += 1;
 			logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs",
 				filename))
 
+			ignoreBlock = False
 			for line in logFile:
 				jsonREMatch = re.match("^#+ ?(failJSON|filterOptions|addFILE):(.+)$", line)
 				if jsonREMatch:
@@ -159,9 +161,13 @@ def testSampleRegexsFactory(name, basedir):
 						if jsonREMatch.group(1) == 'filterOptions':
 							# following lines with another filter options:
 							self._filterTests = []
+							ignoreBlock = False
 							for opts in (faildata if isinstance(faildata, list) else [faildata]):
 								# unique filter name (using options combination):
 								self.assertTrue(isinstance(opts, dict))
+								if opts.get('test.condition'):
+									ignoreBlock = not eval(opts.get('test.condition'))
+									del opts['test.condition']
 								fltName = opts.get('filterName')
 								if not fltName: fltName = str(opts) if opts else ''
 								fltName = name + fltName
@@ -178,10 +184,11 @@ def testSampleRegexsFactory(name, basedir):
 						raise ValueError("%s: %s:%i" %
 							(e, logFile.filename(), logFile.filelineno()))
 					line = next(logFile)
-				elif line.startswith("#") or not line.strip():
+				elif ignoreBlock or line.startswith("#") or not line.strip():
 					continue
 				else: # pragma: no cover - normally unreachable
 					faildata = {}
+				if ignoreBlock: continue
 
 				# if filter options was not yet specified:
 				if not self._filterTests:
@@ -195,6 +202,7 @@ def testSampleRegexsFactory(name, basedir):
 					regexList = flt.getFailRegex()
 
 					try:
+						fail = {}
 						ret = flt.processLine(line)
 						if not ret:
 							# Bypass if filter constraint specified:
@@ -222,9 +230,17 @@ def testSampleRegexsFactory(name, basedir):
 						for k, v in faildata.iteritems():
 							if k not in ("time", "match", "desc", "filter"):
 								fv = fail.get(k, None)
-								# Fallback for backwards compatibility (previously no fid, was host only):
-								if k == "host" and fv is None:
-									fv = fid
+								if fv is None:
+									# Fallback for backwards compatibility (previously no fid, was host only):
+									if k == "host":
+										fv = fid
+									# special case for attempts counter:
+									if k == "attempts":
+										fv = len(fail.get('matches', {}))
+								# compare sorted (if set)
+								if isinstance(fv, (set, list, dict)):
+									self.assertSortedEqual(fv, v)
+									continue
 								self.assertEqual(fv, v)
 
 						t = faildata.get("time", None)
@@ -246,8 +262,12 @@ def testSampleRegexsFactory(name, basedir):
 						regexsUsedIdx.add(failregex)
 						regexsUsedRe.add(regexList[failregex])
 					except AssertionError as e: # pragma: no cover
-						raise AssertionError("%s: %s on: %s:%i, line:\n%s" % (
-									fltName, e, logFile.filename(), logFile.filelineno(), line))
+						import pprint
+						raise AssertionError("%s: %s on: %s:%i, line:\n%s\n"
+							"faildata: %s\nfail: %s" % (
+								fltName, e, logFile.filename(), logFile.filelineno(), line,
+								'\n'.join(pprint.pformat(faildata).splitlines()),
+								'\n'.join(pprint.pformat(fail).splitlines())))
 
 		# check missing samples for regex using each filter-options combination:
 		for fltName, flt in self._filters.iteritems():