mirror of https://github.com/fail2ban/fail2ban
Merge pull request #1733 from sebres/0.10-repl-skiplines
Normalizes replacement of `<SKIPLINES>` + no multiline failregex per defaultpull/1739/head
Serg G. Brester
GitHub
commit
cea8ba7831
|
|
@ -55,11 +55,14 @@ from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack,
|
|||
# Gets the instance of the logger.
|
||||
logSys = getLogger("fail2ban")
|
||||
|
||||
def debuggexURL(sample, regex, useDns="yes"):
|
||||
q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns),
|
||||
'str': sample,
|
||||
'flavor': 'python' })
|
||||
return 'https://www.debuggex.com/?' + q
|
||||
def debuggexURL(sample, regex, multiline=False, useDns="yes"):
|
||||
args = {
|
||||
're': Regex._resolveHostTag(regex, useDns=useDns),
|
||||
'str': sample,
|
||||
'flavor': 'python'
|
||||
}
|
||||
if multiline: args['flags'] = 'm'
|
||||
return 'https://www.debuggex.com/?' + urllib.urlencode(args)
|
||||
|
||||
def output(args): # pragma: no cover (overriden in test-cases)
|
||||
print(args)
|
||||
|
|
@ -400,6 +403,7 @@ class Fail2banRegex(object):
|
|||
fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
|
||||
try:
|
||||
ret = self._filter.processLine(line, date)
|
||||
lines = []
|
||||
line = self._filter.processedLine()
|
||||
for match in ret:
|
||||
# Append True/False flag depending if line was matched by
|
||||
|
|
@ -422,9 +426,17 @@ class Fail2banRegex(object):
|
|||
"".join(bufLine[::2])))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
self._line_stats.matched += 1
|
||||
self._line_stats.missed -= 1
|
||||
# if buffering - add also another lines from match:
|
||||
if self._print_all_matched:
|
||||
if not self._debuggex:
|
||||
self._line_stats.matched_lines.append("".join(bufLine))
|
||||
else:
|
||||
lines.append(bufLine[0] + bufLine[2])
|
||||
self._line_stats.matched += 1
|
||||
self._line_stats.missed -= 1
|
||||
if lines: # pre-lines parsed in multiline mode (buffering)
|
||||
lines.append(line)
|
||||
line = "\n".join(lines)
|
||||
return line, ret
|
||||
|
||||
def process(self, test_lines):
|
||||
|
|
@ -472,6 +484,7 @@ class Fail2banRegex(object):
|
|||
assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored))
|
||||
lines = lstats[ltype]
|
||||
l = lstats[ltype + '_lines']
|
||||
multiline = self._filter.getMaxLines() > 1
|
||||
if lines:
|
||||
header = "%s line(s):" % (ltype.capitalize(),)
|
||||
if self._debuggex:
|
||||
|
|
@ -485,7 +498,8 @@ class Fail2banRegex(object):
|
|||
for arg in [l, regexlist]:
|
||||
ans = [ x + [y] for x in ans for y in arg ]
|
||||
b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' +
|
||||
debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans)
|
||||
debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(),
|
||||
multiline, self._opts.usedns), ans)
|
||||
pprint_list([x.rstrip() for x in b], header)
|
||||
else:
|
||||
output( "%s too many to print. Use --print-all-%s " \
|
||||
|
|
@ -599,8 +613,19 @@ class Fail2banRegex(object):
|
|||
output( "Use journal match : %s" % " ".join(journalmatch) )
|
||||
test_lines = journal_lines_gen(flt, myjournal)
|
||||
else:
|
||||
output( "Use single line : %s" % shortstr(cmd_log) )
|
||||
test_lines = [ cmd_log ]
|
||||
# if single line parsing (without buffering)
|
||||
if self._filter.getMaxLines() <= 1:
|
||||
output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) )
|
||||
test_lines = [ cmd_log ]
|
||||
else: # multi line parsing (with buffering)
|
||||
test_lines = cmd_log.split("\n")
|
||||
output( "Use multi line : %s line(s)" % len(test_lines) )
|
||||
for i, l in enumerate(test_lines):
|
||||
if i >= 5:
|
||||
output( "| ..." ); break
|
||||
output( "| %2.2s: %s" % (i+1, shortstr(l)) )
|
||||
output( "`-" )
|
||||
|
||||
output( "" )
|
||||
|
||||
self.process(test_lines)
|
||||
|
|
|
|||
|
|
@ -103,20 +103,16 @@ class Regex:
|
|||
# avoid construction of invalid object.
|
||||
# @param value the regular expression
|
||||
|
||||
def __init__(self, regex, **kwargs):
|
||||
def __init__(self, regex, multiline=False, **kwargs):
|
||||
self._matchCache = None
|
||||
# Perform shortcuts expansions.
|
||||
# Resolve "<HOST>" tag using default regular expression for host:
|
||||
# Replace standard f2b-tags (like "<HOST>", etc) using default regular expressions:
|
||||
regex = Regex._resolveHostTag(regex, **kwargs)
|
||||
# Replace "<SKIPLINES>" with regular expression for multiple lines.
|
||||
regexSplit = regex.split("<SKIPLINES>")
|
||||
regex = regexSplit[0]
|
||||
for n, regexLine in enumerate(regexSplit[1:]):
|
||||
regex += "\n(?P<skiplines%i>(?:(.*\n)*?))" % n + regexLine
|
||||
#
|
||||
if regex.lstrip() == '':
|
||||
raise RegexException("Cannot add empty regex")
|
||||
try:
|
||||
self._regexObj = re.compile(regex, re.MULTILINE)
|
||||
self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
|
||||
self._regex = regex
|
||||
except sre_constants.error:
|
||||
raise RegexException("Unable to compile regular expression '%s'" %
|
||||
|
|
@ -135,6 +131,9 @@ class Regex:
|
|||
def _resolveHostTag(regex, useDns="yes"):
|
||||
|
||||
openTags = dict()
|
||||
props = {
|
||||
'nl': 0, # new lines counter by <SKIPLINES> tag;
|
||||
}
|
||||
# tag interpolation callable:
|
||||
def substTag(m):
|
||||
tag = m.group()
|
||||
|
|
@ -142,6 +141,11 @@ class Regex:
|
|||
# 3 groups instead of <HOST> - separated ipv4, ipv6 and host (dns)
|
||||
if tn == "HOST":
|
||||
return R_HOST[RI_HOST if useDns not in ("no",) else RI_ADDR]
|
||||
# replace "<SKIPLINES>" with regular expression for multiple lines (by buffering with maxlines)
|
||||
if tn == "SKIPLINES":
|
||||
nl = props['nl']
|
||||
props['nl'] = nl + 1
|
||||
return r"\n(?P<skiplines%i>(?:(?:.*\n)*?))" % (nl,)
|
||||
# static replacement from RH4TAG:
|
||||
try:
|
||||
return RH4TAG[tn]
|
||||
|
|
|
|||
|
|
@ -161,13 +161,11 @@ class Filter(JailThread):
|
|||
# @param value the regular expression
|
||||
|
||||
def addFailRegex(self, value):
|
||||
multiLine = self.getMaxLines() > 1
|
||||
try:
|
||||
regex = FailRegex(value, prefRegex=self.__prefRegex, useDns=self.__useDns)
|
||||
regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine,
|
||||
useDns=self.__useDns)
|
||||
self.__failRegex.append(regex)
|
||||
if "\n" in regex.getRegex() and not self.getMaxLines() > 1:
|
||||
logSys.warning(
|
||||
"Mutliline regex set for jail %r "
|
||||
"but maxlines not greater than 1", self.jailName)
|
||||
except RegexException as e:
|
||||
logSys.error(e)
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -252,6 +252,44 @@ class Fail2banRegexTest(LogCaptureTestCase):
|
|||
)
|
||||
self.assertTrue(fail2banRegex.start(args))
|
||||
|
||||
def testDirectMultilineBuf(self):
|
||||
# test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed):
|
||||
for preLines in (0, 20):
|
||||
self.pruneLog("[test-phase %s]" % preLines)
|
||||
(opts, args, fail2banRegex) = _Fail2banRegex(
|
||||
"--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5",
|
||||
("1490349000 TEST-NL\n"*preLines) +
|
||||
"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
|
||||
r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
|
||||
)
|
||||
self.assertTrue(fail2banRegex.start(args))
|
||||
self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2))
|
||||
# both matched lines were printed:
|
||||
self.assertLogged("| 1490349000 FAIL", "| 1490349001 HOST 192.0.2.34", all=True)
|
||||
|
||||
|
||||
def testDirectMultilineBufDebuggex(self):
|
||||
(opts, args, fail2banRegex) = _Fail2banRegex(
|
||||
"--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5",
|
||||
"1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34",
|
||||
r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$"
|
||||
)
|
||||
self.assertTrue(fail2banRegex.start(args))
|
||||
self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed')
|
||||
# the sequence in args-dict is currently undefined (so can be 1st argument)
|
||||
self.assertLogged("&flags=m", "?flags=m")
|
||||
|
||||
def testSinglelineWithNLinContent(self):
|
||||
#
|
||||
(opts, args, fail2banRegex) = _Fail2banRegex(
|
||||
"--usedns", "no", "-d", "^Epoch", "--print-all-matched",
|
||||
"1490349000 FAIL: failure\nhost: 192.0.2.35",
|
||||
r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$"
|
||||
)
|
||||
self.assertTrue(fail2banRegex.start(args))
|
||||
self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed')
|
||||
|
||||
|
||||
def testWrongFilterFile(self):
|
||||
# use test log as filter file to cover eror cases...
|
||||
(opts, args, fail2banRegex) = _Fail2banRegex(
|
||||
|
|
|
|||
|
|
@ -1479,8 +1479,8 @@ class GetFailures(LogCaptureTestCase):
|
|||
output = [("192.0.43.10", 2, 1124013599.0),
|
||||
("192.0.43.11", 1, 1124013598.0)]
|
||||
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
|
||||
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
|
||||
self.filter.setMaxLines(100)
|
||||
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
|
||||
self.filter.setMaxRetry(1)
|
||||
|
||||
self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
|
||||
|
|
@ -1497,9 +1497,9 @@ class GetFailures(LogCaptureTestCase):
|
|||
def testGetFailuresMultiLineIgnoreRegex(self):
|
||||
output = [("192.0.43.10", 2, 1124013599.0)]
|
||||
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
|
||||
self.filter.setMaxLines(100)
|
||||
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
|
||||
self.filter.addIgnoreRegex("rsync error: Received SIGINT")
|
||||
self.filter.setMaxLines(100)
|
||||
self.filter.setMaxRetry(1)
|
||||
|
||||
self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
|
||||
|
|
@ -1513,9 +1513,9 @@ class GetFailures(LogCaptureTestCase):
|
|||
("192.0.43.11", 1, 1124013598.0),
|
||||
("192.0.43.15", 1, 1124013598.0)]
|
||||
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False)
|
||||
self.filter.setMaxLines(100)
|
||||
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
|
||||
self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")
|
||||
self.filter.setMaxLines(100)
|
||||
self.filter.setMaxRetry(1)
|
||||
|
||||
self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
|
|||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
|
||||
|
||||
# regexp to test greedy catch-all should be not-greedy:
|
||||
RE_HOST = Regex('<HOST>').getRegex()
|
||||
RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?).*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
|
||||
RE_HOST = Regex._resolveHostTag('<HOST>')
|
||||
RE_WRONG_GREED = re.compile(r'\.[+\*](?!\?)[^\$\^]*' + re.escape(RE_HOST) + r'.*(?:\.[+\*].*|[^\$])$')
|
||||
|
||||
|
||||
class FilterSamplesRegex(unittest.TestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue