filter processing:

- avoid duplicates in "matches" (previously always added matches of pending failures to every next real failure, or nofail-helper recognized IP, now first failure only);
  - several optimizations of merge mechanism (multi-line parsing);
fail2ban-regex: better output handling, extended with tag substitution (ex.: `-o 'fail <ip>, user <F-USER>: <msg>'`); consider a string containing new-line as multi-line log-excerpt (not as a single log-line)
filter.d/sshd.conf: introduced parameter `publickey` (allowing change behavior of "Failed publickey" failures):
  - `nofail` (default) - consider failed publickey (legitimate users) as no failure (helper to get IP and user-name only)
  - `invalid` - consider failed publickey for invalid users only;
  - `any` - consider failed publickey for valid users too;
  - `ignore` - ignore "Failed publickey ..." failures (don't consider failed publickey at all)
tests/samplestestcase.py: SampleRegexsFactory gets new failJSON option `constraint` to allow ignore of some tests depending on filter name, options and test parameters
pull/2638/head
sebres 2020-02-12 21:27:45 +01:00
parent 1492ab2247
commit 9137c7bb23
9 changed files with 260 additions and 135 deletions

View File

@ -40,8 +40,8 @@ prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONT
cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$
^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$
^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
<cmnfailre-failed-pub-<publickey>>
^Failed <cmnfailed> for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST>
^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$
^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
@ -60,6 +60,12 @@ cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER>
<mdre-<mode>-other>
^<F-MLFFORGET><F-MLFGAINED>Accepted \w+</F-MLFGAINED></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$)
cmnfailed-any = \S+
cmnfailed-ignore = \b(?!publickey)\S+
cmnfailed-invalid = <cmnfailed-ignore>
cmnfailed-nofail = (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+)
cmnfailed = <cmnfailed-<publickey>>
mdre-normal =
# used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode)
mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__suff)s|\s*)$
@ -84,6 +90,17 @@ mdre-aggressive = %(mdre-ddos)s
# mdre-extra-other is fully included within mdre-ddos-other:
mdre-aggressive-other = %(mdre-ddos-other)s
# Parameter "publickey": nofail (default), invalid, any, ignore
publickey = nofail
# consider failed publickey for invalid users only:
cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
# consider failed publickey for valid users too (don't need RE, see cmnfailed):
cmnfailre-failed-pub-any =
# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed):
cmnfailre-failed-pub-nofail = <cmnfailre-failed-pub-invalid>
# don't consider failed publickey as failures (don't need RE, see cmnfailed):
cmnfailre-failed-pub-ignore =
cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
failregex = %(cmnfailre)s

View File

@ -493,8 +493,69 @@ class Fail2banRegex(object):
line = "\n".join(lines)
return line, ret, is_ignored
def _prepaireOutput(self):
"""Prepares output- and fetch-function corresponding given '--out' option (format)"""
ofmt = self._opts.out
if ofmt in ('id', 'ip'):
def _out(ret):
for r in ret:
output(r[1])
elif ofmt == 'msg':
def _out(ret):
for r in ret:
for r in r[3].get('matches'):
if not isinstance(r, basestring):
r = ''.join(r for r in r)
output(r)
elif ofmt == 'row':
def _out(ret):
for r in ret:
output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches')))
elif '<' not in ofmt:
def _out(ret):
for r in ret:
output(r[3].get(ofmt))
else: # extended format with tags substitution:
from ..server.actions import Actions, CommandAction, BanTicket
def _escOut(t, v):
# use safe escape (avoid inject on pseudo tag "\x00msg\x00"):
if t not in ('msg',):
return v.replace('\x00', '\\x00')
return v
def _out(ret):
rows = []
wrap = {'NL':0}
for r in ret:
ticket = BanTicket(r[1], time=r[2], data=r[3])
aInfo = Actions.ActionInfo(ticket)
# if msg tag is used - output if single line (otherwise let it as is to wrap multilines later):
def _get_msg(self):
if not wrap['NL'] and len(r[3].get('matches', [])) <= 1:
return self['matches']
else: # pseudo tag for future replacement:
wrap['NL'] = 1
return "\x00msg\x00"
aInfo['msg'] = _get_msg
# not recursive interpolation (use safe escape):
v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut)
if wrap['NL']: # contains multiline tags (msg):
rows.append((r, v))
continue
output(v)
# wrap multiline tag (msg) interpolations to single line:
for r, v in rows:
for r in r[3].get('matches'):
if not isinstance(r, basestring):
r = ''.join(r for r in r)
r = v.replace("\x00msg\x00", r)
output(r)
return _out
def process(self, test_lines):
t0 = time.time()
if self._opts.out: # get out function
out = self._prepaireOutput()
for line in test_lines:
if isinstance(line, tuple):
line_datetimestripped, ret, is_ignored = self.testRegex(
@ -509,49 +570,35 @@ class Fail2banRegex(object):
if not is_ignored:
is_ignored = self.testIgnoreRegex(line_datetimestripped)
if self._opts.out: # (formated) output:
if len(ret) > 0: out(ret)
continue
if is_ignored:
self._line_stats.ignored += 1
if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1):
self._line_stats.ignored_lines.append(line)
if self._debuggex:
self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped)
if len(ret) > 0:
assert(not is_ignored)
if self._opts.out:
if self._opts.out in ('id', 'ip'):
for ret in ret:
output(ret[1])
elif self._opts.out == 'msg':
for ret in ret:
for ret in ret[3].get('matches'):
output(''.join(v for v in ret))
elif self._opts.out == 'row':
for ret in ret:
output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches')))
else:
for ret in ret:
output(ret[3].get(self._opts.out))
continue
elif len(ret) > 0:
self._line_stats.matched += 1
if self._print_all_matched:
self._line_stats.matched_lines.append(line)
if self._debuggex:
self._line_stats.matched_lines_timeextracted.append(line_datetimestripped)
else:
if not is_ignored:
self._line_stats.missed += 1
if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
self._line_stats.missed_lines.append(line)
if self._debuggex:
self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
self._line_stats.missed += 1
if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1):
self._line_stats.missed_lines.append(line)
if self._debuggex:
self._line_stats.missed_lines_timeextracted.append(line_datetimestripped)
self._line_stats.tested += 1
self._time_elapsed = time.time() - t0
def printLines(self, ltype):
lstats = self._line_stats
assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored))
lines = lstats[ltype]
l = lstats[ltype + '_lines']
multiline = self._filter.getMaxLines() > 1
@ -688,10 +735,10 @@ class Fail2banRegex(object):
test_lines = journal_lines_gen(flt, myjournal)
else:
# if single line parsing (without buffering)
if self._filter.getMaxLines() <= 1:
if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log:
self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
test_lines = [ cmd_log ]
else: # multi line parsing (with buffering)
else: # multi line parsing (with and without buffering)
test_lines = cmd_log.split("\n")
self.output( "Use multi line : %s line(s)" % len(test_lines) )
for i, l in enumerate(test_lines):

View File

@ -772,7 +772,7 @@ class CommandAction(ActionBase):
ESCAPE_VN_CRE = re.compile(r"\W")
@classmethod
def replaceDynamicTags(cls, realCmd, aInfo):
def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None):
"""Replaces dynamical tags in `query` with property values.
**Important**
@ -797,16 +797,17 @@ class CommandAction(ActionBase):
# array for escaped vars:
varsDict = dict()
def escapeVal(tag, value):
# if the value should be escaped:
if cls.ESCAPE_CRE.search(value):
# That one needs to be escaped since its content is
# out of our control
tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
varsDict[tag] = value # add variable
value = '$'+tag # replacement as variable
# replacement for tag:
return value
if not escapeVal:
def escapeVal(tag, value):
# if the value should be escaped:
if cls.ESCAPE_CRE.search(value):
# That one needs to be escaped since its content is
# out of our control
tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag)
varsDict[tag] = value # add variable
value = '$'+tag # replacement as variable
# replacement for tag:
return value
# additional replacement as calling map:
ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS)

View File

@ -655,30 +655,12 @@ class Filter(JailThread):
return users
return users
# # ATM incremental (non-empty only) merge deactivated ...
# @staticmethod
# def _updateFailure(self, mlfidGroups, fail):
# # reset old failure-ids when new types of id available in this failure:
# fids = set()
# for k in ('fid', 'ip4', 'ip6', 'dns'):
# if fail.get(k):
# fids.add(k)
# if fids:
# for k in ('fid', 'ip4', 'ip6', 'dns'):
# if k not in fids:
# try:
# del mlfidGroups[k]
# except:
# pass
# # update not empty values:
# mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v))
def _mergeFailure(self, mlfid, fail, failRegex):
mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
users = None
nfflgs = 0
if fail.get("mlfgained"):
nfflgs |= 9
nfflgs |= (8|1)
if not fail.get('nofail'):
fail['nofail'] = fail["mlfgained"]
elif fail.get('nofail'): nfflgs |= 1
@ -689,13 +671,11 @@ class Filter(JailThread):
# update users set (hold all users of connect):
users = self._updateUsers(mlfidGroups, fail.get('user'))
# be sure we've correct current state ('nofail' and 'mlfgained' only from last failure)
mlfidGroups.pop('nofail', None)
mlfidGroups.pop('mlfgained', None)
# # ATM incremental (non-empty only) merge deactivated (for future version only),
# # it can be simulated using alternate value tags, like <F-ALT_VAL>...</F-ALT_VAL>,
# # so previous value 'val' will be overwritten only if 'alt_val' is not empty...
# _updateFailure(mlfidGroups, fail)
#
if mlfidGroups.pop('nofail', None): nfflgs |= 4
if mlfidGroups.pop('mlfgained', None): nfflgs |= 4
# if we had no pending failures then clear the matches (they are already provided):
if (nfflgs & 4) == 0 and not mlfidGroups.get('mlfpending', 0):
mlfidGroups.pop("matches", None)
# overwrite multi-line failure with all values, available in fail:
mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None))
# new merged failure data:
@ -709,17 +689,18 @@ class Filter(JailThread):
self.mlfidCache.set(mlfid, mlfidFail)
# check users in order to avoid reset failure by multiple logon-attempts:
if fail.pop('mlfpending', 0) or users and len(users) > 1:
# we've new user, reset 'nofail' because of multiple users attempts:
# we've pending failures or new user, reset 'nofail' because of failures or multiple users attempts:
fail.pop('nofail', None)
nfflgs &= ~1 # reset nofail
fail.pop('mlfgained', None)
nfflgs &= ~(8|1) # reset nofail and gained
# merge matches:
if not (nfflgs & 1): # current nofail state (corresponding users)
if (nfflgs & 1) == 0: # current nofail state (corresponding users)
m = fail.pop("nofail-matches", [])
m += fail.get("matches", [])
if not (nfflgs & 8): # no gain signaled
if (nfflgs & 8) == 0: # no gain signaled
m += failRegex.getMatchedTupleLines()
fail["matches"] = m
elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail:
elif (nfflgs & 3) == 1: # not mlfforget and nofail:
fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines()
# return merged:
return fail
@ -895,6 +876,9 @@ class Filter(JailThread):
# otherwise, try to use dns conversion:
else:
ips = DNSUtils.textToIp(host, self.__useDns)
# if checkAllRegex we must make a copy (to be sure next RE doesn't change merged/cached failure):
if self.checkAllRegex and mlfid is not None:
fail = fail.copy()
# append failure with match to the list:
for ip in ips:
failList.append([failRegexIndex, ip, date, fail])

View File

@ -37,7 +37,7 @@ __pam_auth = pam_[a-z]+
cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$
^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$
^%(__prefix_line_sl)sFailed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
^%(__prefix_line_sl)sFailed (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+) for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM <HOST>
^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__suff)s$
^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$

View File

@ -83,13 +83,29 @@ def _test_exec_command_line(*args):
STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0"
RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>"
RE_00_ID = r"Authentication failure for <F-ID>.*?</F-ID> from <HOST>$"
RE_00_USER = r"Authentication failure for <F-USER>.*?</F-USER> from <HOST>$"
RE_00_ID = r"Authentication failure for <F-ID>.*?</F-ID> from <ADDR>$"
RE_00_USER = r"Authentication failure for <F-USER>.*?</F-USER> from <ADDR>$"
FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log")
FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log")
FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log")
# STR_ML_SSHD -- multiline log-excerpt with two sessions:
# 192.0.2.1 (sshd[32307]) makes 2 failed attempts using public keys (without "Disconnecting: Too many authentication"),
# and delayed success on accepted (STR_ML_SSHD_OK) or no success by close on preauth phase (STR_ML_SSHD_FAIL)
# 192.0.2.2 (sshd[32310]) makes 2 failed attempts using public keys (with "Disconnecting: Too many authentication"),
# and closed on preauth phase
STR_ML_SSHD = """Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth]
Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:...
Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth]
Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.2 [preauth]"""
STR_ML_SSHD_OK = "Nov 28 09:16:06 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:..."
STR_ML_SSHD_FAIL = "Nov 28 09:16:06 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]"
FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd")
FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf')
FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log')
@ -291,10 +307,10 @@ class Fail2banRegexTest(LogCaptureTestCase):
#
self.assertTrue(_test_exec(
"--usedns", "no", "-d", "^Epoch", "--print-all-matched",
"1490349000 FAIL: failure\nhost: 192.0.2.35",
"-L", "2", "1490349000 FAIL: failure\nhost: 192.0.2.35",
r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$"
))
self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
self.assertLogged('Lines: 2 lines, 0 ignored, 2 matched, 0 missed')
def testRegexEpochPatterns(self):
self.assertTrue(_test_exec(
@ -340,6 +356,55 @@ class Fail2banRegexTest(LogCaptureTestCase):
self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER))
self.assertLogged('kevin')
self.pruneLog()
# complex substitution using tags (ip, user, family):
self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <family>', STR_00, RE_00_USER))
self.assertLogged('192.0.2.0, kevin, inet4')
self.pruneLog()
def testFrmtOutputWrapML(self):
unittest.F2B.SkipIfCfgMissing(stock=True)
# complex substitution using tags and message (ip, user, msg):
self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
'-c', CONFIG_DIR, '--usedns', 'no',
STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]'))
# be sure we don't have IP in one line and have it in another:
lines = STR_ML_SSHD.split("\n")
self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1])
# but both are in output "merged" with IP and user:
self.assertLogged(
'192.0.2.2, git, '+lines[-2],
'192.0.2.2, git, '+lines[-1],
all=True)
# nothing should be found for 192.0.2.1 (mode is not aggressive):
self.assertNotLogged('192.0.2.1, git, ')
# test with publickey (nofail) - would not produce output for 192.0.2.1 because accepted:
self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)")
self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
'-c', CONFIG_DIR, '--usedns', 'no',
STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]'))
self.assertLogged(
'192.0.2.2, git, '+lines[-4],
'192.0.2.2, git, '+lines[-3],
'192.0.2.2, git, '+lines[-2],
'192.0.2.2, git, '+lines[-1],
all=True)
# nothing should be found for 192.0.2.1 (access gained so failures ignored):
self.assertNotLogged('192.0.2.1, git, ')
# now same test but "accepted" replaced with "closed" on preauth phase:
self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)")
self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>',
'-c', CONFIG_DIR, '--usedns', 'no',
STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]'))
# 192.0.2.1 should be found for every failure (2x failed key + 1x closed):
lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:]
self.assertLogged(
'192.0.2.1, git, '+lines[-3],
'192.0.2.1, git, '+lines[-2],
'192.0.2.1, git, '+lines[-1],
all=True)
def testWrongFilterFile(self):
# use test log as filter file to cover eror cases...

View File

@ -166,9 +166,11 @@ Nov 28 09:16:03 srv sshd[32307]: Connection closed by 192.0.2.1
Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
# failJSON: { "match": false }
Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
# failJSON: { "match": false }
# failJSON: { "constraint": "name == 'sshd'", "time": "2004-11-28T09:16:05", "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" }
Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth]
# failJSON: { "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" }
# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" }
Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth]
# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" }
Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth]
# failJSON: { "match": false }
@ -215,7 +217,7 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S
# Match sshd auth errors on OpenSUSE systems (gh-1024)
# failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" }
2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root
# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" }
# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" }
2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 [preauth]
# disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...):
@ -238,7 +240,7 @@ Mar 7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113
Mar 7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114
# failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" }
Mar 7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114
# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" }
# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" }
Mar 7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2
# failJSON: { "match": false , "desc": "No failure" }
Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0)
@ -248,14 +250,14 @@ Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user
Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
# failJSON: { "match": false , "desc": "Still no failure (second try, same user)" }
Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" }
# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" }
Mar 7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2
# failJSON: { "match": false , "desc": "No failure" }
Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116
# failJSON: { "time": "2005-03-19T16:47:48", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" }
Mar 19 16:47:48 test sshd[5672]: Invalid user admin from 192.0.2.117 port 44004
# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" }
# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" }
Mar 19 16:47:49 test sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth]
# failJSON: { "time": "2005-03-19T16:47:50", "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" }
Mar 19 16:47:50 srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth]
@ -332,7 +334,7 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554
Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
# failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" }
# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" }
Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
# gh-1943 (previous OpenSSH log-format)

View File

@ -167,9 +167,11 @@ srv sshd[32307]: Connection closed by 192.0.2.1
srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
# failJSON: { "match": false }
srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx
# failJSON: { "match": false }
# failJSON: { "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" }
srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth]
# failJSON: { "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" }
# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" }
srv sshd[32310]: Connection closed by 192.0.2.111 [preauth]
# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" }
srv sshd[32310]: Connection closed by 192.0.2.111 [preauth]
# failJSON: { "match": false }
@ -216,7 +218,7 @@ srv sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank yo
# Match sshd auth errors on OpenSUSE systems (gh-1024)
# failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" }
srv sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root
# failJSON: { "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" }
# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" }
srv sshd[2716]: Connection closed by 192.0.2.112 [preauth]
# filterOptions: [{}]
@ -238,7 +240,7 @@ srv sshd[1556]: Connection closed by 192.0.2.113
srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114
# failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" }
srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114
# failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" }
# failJSON: { "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" }
srv sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2
# failJSON: { "match": false , "desc": "No failure" }
srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0)
@ -248,14 +250,14 @@ srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0)
srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
# failJSON: { "match": false , "desc": "Still no failure (second try, same user)" }
srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116
# failJSON: { "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" }
# failJSON: { "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" }
srv sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2
# failJSON: { "match": false , "desc": "No failure" }
srv sshd[1559]: Connection closed by 192.0.2.116
# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" }
srv sshd[5672]: Invalid user admin from 192.0.2.117 port 44004
# failJSON: { "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" }
# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" }
srv sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth]
# failJSON: { "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" }
srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth]
@ -325,7 +327,7 @@ srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching
srv sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22
# failJSON: { "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" }
srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
# failJSON: { "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" }
# failJSON: { "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" }
srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth]
# gh-1943 (previous OpenSSH log-format)

View File

@ -216,9 +216,11 @@ def testSampleRegexsFactory(name, basedir):
# process line using several filter options (if specified in the test-file):
for fltName, flt, opts in self._filterTests:
# Bypass if constraint (as expression) is not valid:
if faildata.get('constraint') and not eval(faildata['constraint']):
continue
flt, regexsUsedIdx = flt
regexList = flt.getFailRegex()
failregex = -1
try:
fail = {}
@ -229,20 +231,23 @@ def testSampleRegexsFactory(name, basedir):
if opts.get('test.prefix-line'): # journal backends creates common prefix-line:
line = opts.get('test.prefix-line') + line
ret = flt.processLine(('', TEST_NOW_STR, line.rstrip('\r\n')), TEST_NOW)
if not ret:
# Bypass if filter constraint specified:
if faildata.get('filter') and name != faildata.get('filter'):
continue
# Check line is flagged as none match
self.assertFalse(faildata.get('match', True),
"Line not matched when should have")
continue
if ret:
# filter matched only (in checkAllRegex mode it could return 'nofail' too):
found = []
for ret in ret:
failregex, fid, fail2banTime, fail = ret
# bypass pending and nofail:
if fid is None or fail.get('nofail'):
regexsUsedIdx.add(failregex)
regexsUsedRe.add(regexList[failregex])
continue
found.append(ret)
ret = found
failregex, fid, fail2banTime, fail = ret[0]
# Bypass no failure helpers-regexp:
if not faildata.get('match', False) and (fid is None or fail.get('nofail')):
regexsUsedIdx.add(failregex)
regexsUsedRe.add(regexList[failregex])
if not ret:
# Check line is flagged as none match
self.assertFalse(faildata.get('match', False),
"Line not matched when should have")
continue
# Check line is flagged to match
@ -251,39 +256,41 @@ def testSampleRegexsFactory(name, basedir):
self.assertEqual(len(ret), 1,
"Multiple regexs matched %r" % (map(lambda x: x[0], ret)))
# Verify match captures (at least fid/host) and timestamp as expected
for k, v in faildata.iteritems():
if k not in ("time", "match", "desc", "filter"):
fv = fail.get(k, None)
if fv is None:
# Fallback for backwards compatibility (previously no fid, was host only):
if k == "host":
fv = fid
# special case for attempts counter:
if k == "attempts":
fv = len(fail.get('matches', {}))
# compare sorted (if set)
if isinstance(fv, (set, list, dict)):
self.assertSortedEqual(fv, v)
continue
self.assertEqual(fv, v)
for ret in ret:
failregex, fid, fail2banTime, fail = ret
# Verify match captures (at least fid/host) and timestamp as expected
for k, v in faildata.iteritems():
if k not in ("time", "match", "desc", "constraint"):
fv = fail.get(k, None)
if fv is None:
# Fallback for backwards compatibility (previously no fid, was host only):
if k == "host":
fv = fid
# special case for attempts counter:
if k == "attempts":
fv = len(fail.get('matches', {}))
# compare sorted (if set)
if isinstance(fv, (set, list, dict)):
self.assertSortedEqual(fv, v)
continue
self.assertEqual(fv, v)
t = faildata.get("time", None)
if t is not None:
try:
jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
except ValueError:
jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f")
jsonTime = time.mktime(jsonTimeLocal.timetuple())
jsonTime += jsonTimeLocal.microsecond / 1000000.0
self.assertEqual(fail2banTime, jsonTime,
"UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" %
(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
fail2banTime - jsonTime) )
t = faildata.get("time", None)
if t is not None:
try:
jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
except ValueError:
jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f")
jsonTime = time.mktime(jsonTimeLocal.timetuple())
jsonTime += jsonTimeLocal.microsecond / 1000000.0
self.assertEqual(fail2banTime, jsonTime,
"UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" %
(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
fail2banTime - jsonTime) )
regexsUsedIdx.add(failregex)
regexsUsedRe.add(regexList[failregex])
regexsUsedIdx.add(failregex)
regexsUsedRe.add(regexList[failregex])
except AssertionError as e: # pragma: no cover
import pprint
raise AssertionError("%s: %s on: %s:%i, line:\n %sregex (%s):\n %s\n"