github
/
fail2ban
mirror of https://github.com/fail2ban/fail2ban
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #476 from kwirk/multiline-matches 

Capture multiline matched lines into fail ticket
pull/491/head
Steven Hiscocks 2013-12-13 08:47:08 -08:00
parent 80df01bf15 d9afcc178a
commit a60fbcc116
5 changed files with 108 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
bin/fail2ban-regex
View File

@ -309,7 +309,7 @@ class Fail2banRegex(object):
def testIgnoreRegex(self, line):
found = False
try:
ret = self._filter.ignoreLine(line)
ret = self._filter.ignoreLine([(line, "", "")])
if ret is not None:
found = True
regex = self._ignoreregex[ret].inc()
@ -338,35 +338,27 @@ class Fail2banRegex(object):
return False
for bufLine in orgLineBuffer[int(fullBuffer):]:
if bufLine not in self._filter._Filter__lineBuffer:
if self.removeMissedLine(bufLine):
try:
self._line_stats.missed_lines.pop(
self._line_stats.missed_lines.index("".join(bufLine)))
self._line_stats.missed_lines_timeextracted.pop(
self._line_stats.missed_lines_timeextracted.index(
"".join(bufLine[::2])))
except ValueError:
pass
else:
self._line_stats.matched += 1
return line, ret
def removeMissedLine(self, line):
"""Remove `line` from missed lines, by comparing without time match"""
for n, missed_line in \
enumerate(reversed(self._line_stats.missed_lines)):
timeMatch = self._filter.dateDetector.matchTime(
missed_line, incHits=False)
if timeMatch:
logLine = (missed_line[:timeMatch.start()] +
missed_line[timeMatch.end():])
else:
logLine = missed_line
if logLine.rstrip("\r\n") == line:
self._line_stats.missed_lines.pop(
len(self._line_stats.missed_lines) - n - 1)
return True
return False
def process(self, test_lines):
for line_no, line in enumerate(test_lines):
if line.startswith('#') or not line.strip():
line = line.strip('\r\n')
if line.startswith('#') or not line:
# skip comment and empty lines
continue
is_ignored = fail2banRegex.testIgnoreRegex(line)
line_datetimestripped, ret = fail2banRegex.testRegex(line)
is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped)
if is_ignored:
self._line_stats.ignored_lines.append(line)
@ -436,7 +428,7 @@ class Fail2banRegex(object):
" %s %s%s" % (
ip[1],
timeString,
ip[3] and " (multiple regex matched)" or ""))
ip[-1] and " (multiple regex matched)" or ""))
print "\n%s: %d total" % (title, total)
pprint_list(out, " #) [# of hits] regular expression")

59
fail2ban/server/failregex.py
View File

@ -74,8 +74,9 @@ class Regex:
# method of this object.
# @param value the line
def search(self, value):
self._matchCache = self._regexObj.search(value)
def search(self, tupleLines):
self._matchCache = self._regexObj.search(
"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
if self.hasMatched():
# Find start of the first line where the match was found
try:
@ -89,8 +90,26 @@ class Regex:
"\n", self._matchCache.end() - 1) + 1
except ValueError:
self._matchLineEnd = len(self._matchCache.string)
##
lineCount1 = self._matchCache.string.count(
"\n", 0, self._matchLineStart)
lineCount2 = self._matchCache.string.count(
"\n", 0, self._matchLineEnd)
self._matchedTupleLines = tupleLines[lineCount1:lineCount2]
self._unmatchedTupleLines = tupleLines[:lineCount1]
n = 0
for skippedLine in self.getSkippedLines():
for m, matchedTupleLine in enumerate(
self._matchedTupleLines[n:]):
if "".join(matchedTupleLine[::2]) == skippedLine:
self._unmatchedTupleLines.append(
self._matchedTupleLines.pop(n+m))
n += m
break
self._unmatchedTupleLines.extend(tupleLines[lineCount2:])
# Checks if the previous call to search() matched.
#
# @return True if a match was found, False otherwise
@ -114,7 +133,8 @@ class Regex:
n = 0
while True:
try:
skippedLines += self._matchCache.group("skiplines%i" % n)
if self._matchCache.group("skiplines%i" % n) is not None:
skippedLines += self._matchCache.group("skiplines%i" % n)
n += 1
except IndexError:
break
@ -125,15 +145,18 @@ class Regex:
#
# This returns unmatched lines including captured by the <SKIPLINES> tag.
# @return list of unmatched lines
def getUnmatchedTupleLines(self):
if not self.hasMatched():
return []
else:
return self._unmatchedTupleLines
def getUnmatchedLines(self):
if not self.hasMatched():
return []
unmatchedLines = (
self._matchCache.string[:self._matchLineStart].splitlines(False)
+ self.getSkippedLines()
+ self._matchCache.string[self._matchLineEnd:].splitlines(False))
return unmatchedLines
else:
return ["".join(line) for line in self._unmatchedTupleLines]
##
# Returns matched lines.
@ -141,14 +164,18 @@ class Regex:
# This returns matched lines by excluding those captured
# by the <SKIPLINES> tag.
# @return list of matched lines
def getMatchedTupleLines(self):
if not self.hasMatched():
return []
else:
return self._matchedTupleLines
def getMatchedLines(self):
if not self.hasMatched():
return []
matchedLines = self._matchCache.string[
self._matchLineStart:self._matchLineEnd].splitlines(False)
return [line for line in matchedLines
if line not in self.getSkippedLines()]
else:
return ["".join(line) for line in self._matchedTupleLines]
##
# Exception dedicated to the class Regex.

85
fail2ban/server/filter.py
View File

@ -366,17 +366,15 @@ class Filter(JailThread):
timeMatch = self.dateDetector.matchTime(l)
if timeMatch:
# Lets split into time part and log part of the line
timeText = timeMatch.group()
# Lets leave the beginning in as well, so if there is no
# anchore at the beginning of the time regexp, we don't
# at least allow injection. Should be harmless otherwise
logLine = l[:timeMatch.start()] + l[timeMatch.end():]
tupleLine = (
l[:timeMatch.start()],
l[timeMatch.start():timeMatch.end()],
l[timeMatch.end():])
else:
timeText = None
logLine = l
tupleLine = (l, "", "")
return logLine, self.findFailure(timeText, logLine, returnRawHost, checkAllRegex)
return "".join(tupleLine[::2]), self.findFailure(
tupleLine, returnRawHost, checkAllRegex)
def processLineAndAdd(self, line):
"""Processes the line for failures and populates failManager
@ -385,6 +383,7 @@ class Filter(JailThread):
failregex = element[0]
ip = element[1]
unixTime = element[2]
lines = element[3]
logSys.debug("Processing line with time:%s and ip:%s"
% (unixTime, ip))
if unixTime < MyTime.time() - self.getFindTime():
@ -396,7 +395,7 @@ class Filter(JailThread):
continue
logSys.debug("Found %s" % ip)
## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),)
self.failManager.addFailure(FailTicket(ip, unixTime, [line]))
self.failManager.addFailure(FailTicket(ip, unixTime, lines))
##
# Returns true if the line should be ignored.
@ -405,9 +404,9 @@ class Filter(JailThread):
# @param line: the line
# @return: a boolean
def ignoreLine(self, line):
def ignoreLine(self, tupleLines):
for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
ignoreRegex.search(line)
ignoreRegex.search(tupleLines)
if ignoreRegex.hasMatched():
return ignoreRegexIndex
return None
@ -419,17 +418,17 @@ class Filter(JailThread):
# to find the logging time.
# @return a dict with IP and timestamp.
def findFailure(self, timeText, logLine,
returnRawHost=False, checkAllRegex=False):
def findFailure(self, tupleLine, returnRawHost=False, checkAllRegex=False):
failList = list()
# Checks if we must ignore this line.
if self.ignoreLine(logLine) is not None:
if self.ignoreLine([tupleLine[::2]]) is not None:
# The ignoreregex matched. Return.
logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine)
logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
"".join(tupleLine[::2]))
return failList
timeText = tupleLine[1]
if timeText:
dateTimeMatch = self.dateDetector.getTime(timeText)
@ -446,49 +445,53 @@ class Filter(JailThread):
self.__lastTimeText = timeText
self.__lastDate = date
else:
timeText = self.__lastTimeText or logLine
timeText = self.__lastTimeText or "".join(tupleLine[::2])
date = self.__lastDate
self.__lineBuffer = (self.__lineBuffer + [logLine])[-self.__lineBufferSize:]
logLine = "\n".join(self.__lineBuffer) + "\n"
self.__lineBuffer = (
self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:]
# Iterates over all the regular expressions.
for failRegexIndex, failRegex in enumerate(self.__failRegex):
failRegex.search(logLine)
failRegex.search(self.__lineBuffer)
if failRegex.hasMatched():
# Checks if we must ignore this match.
if self.ignoreLine(
"\n".join(failRegex.getMatchedLines()) + "\n") \
is not None:
# The ignoreregex matched. Remove ignored match.
self.__lineBuffer = failRegex.getUnmatchedLines()
logSys.log(7, "Matched ignoreregex and was ignored")
continue
# The failregex matched.
logSys.log(7, "Matched %s", failRegex)
# Checks if we must ignore this match.
if self.ignoreLine(failRegex.getMatchedTupleLines()) \
is not None:
# The ignoreregex matched. Remove ignored match.
self.__lineBuffer = failRegex.getUnmatchedTupleLines()
logSys.log(7, "Matched ignoreregex and was ignored")
if not checkAllRegex:
break
else:
continue
if date is None:
logSys.debug("Found a match for %r but no valid date/time "
"found for %r. Please try setting a custom "
"date pattern (see man page jail.conf(5)). "
"If format is complex, please "
"file a detailed issue on"
" https://github.com/fail2ban/fail2ban/issues "
"in order to get support for this format."
% (logLine, timeText))
logSys.debug(
"Found a match for %r but no valid date/time "
"found for %r. Please try setting a custom "
"date pattern (see man page jail.conf(5)). "
"If format is complex, please "
"file a detailed issue on"
" https://github.com/fail2ban/fail2ban/issues "
"in order to get support for this format."
% ("\n".join(failRegex.getMatchedLines()), timeText))
else:
self.__lineBuffer = failRegex.getUnmatchedLines()
self.__lineBuffer = failRegex.getUnmatchedTupleLines()
try:
host = failRegex.getHost()
if returnRawHost:
failList.append([failRegexIndex, host, date])
failList.append([failRegexIndex, host, date,
failRegex.getMatchedLines()])
if not checkAllRegex:
break
else:
ipMatch = DNSUtils.textToIp(host, self.__useDns)
if ipMatch:
for ip in ipMatch:
failList.append([failRegexIndex, ip, date])
failList.append([failRegexIndex, ip, date,
failRegex.getMatchedLines()])
if not checkAllRegex:
break
except RegexException, e: # pragma: no cover - unsure if reachable

15
fail2ban/tests/filtertestcase.py
View File

@ -710,7 +710,7 @@ class GetFailures(unittest.TestCase):
# so that they could be reused by other tests
FAILURES_01 = ('193.168.0.128', 3, 1124017199.0,
[u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
[u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3)
def setUp(self):
"""Call before every test case."""
@ -747,16 +747,13 @@ class GetFailures(unittest.TestCase):
fout.close()
# now see if we should be getting the "same" failures
self.testGetFailures01(filename=fname,
failures=GetFailures.FAILURES_01[:3] +
([x.rstrip('\n') + '\r\n' for x in
GetFailures.FAILURES_01[-1]],))
self.testGetFailures01(filename=fname)
_killfile(fout, fname)
def testGetFailures02(self):
output = ('141.3.81.106', 4, 1124017139.0,
[u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
[u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2'
% m for m in 53, 54, 57, 58])
self.filter.addLogPath(GetFailures.FILENAME_02)
@ -789,11 +786,11 @@ class GetFailures(unittest.TestCase):
def testGetFailuresUseDNS(self):
# We should still catch failures with usedns = no ;-)
output_yes = ('93.184.216.119', 2, 1124017139.0,
[u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
[u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2'])
output_no = ('93.184.216.119', 1, 1124017139.0,
[u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
[u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2'])
# Actually no exception would be raised -- it will be just set to 'no'
#self.assertRaises(ValueError,

2
fail2ban/tests/samplestestcase.py
View File

@ -119,7 +119,7 @@ def testSampleRegexsFactory(name):
(map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
# Verify timestamp and host as expected
failregex, host, fail2banTime = ret[0]
failregex, host, fail2banTime, lines = ret[0]
self.assertEqual(host, faildata.get("host", None))
t = faildata.get("time", None)