github
/
fail2ban
mirror of https://github.com/fail2ban/fail2ban
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #476 from kwirk/multiline-matches 

Capture multiline matched lines into fail ticket
pull/491/head
Steven Hiscocks 2013-12-13 08:47:08 -08:00
parent 80df01bf15 d9afcc178a
commit a60fbcc116
5 changed files with 108 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
bin/fail2ban-regex
View File

@ -309,7 +309,7 @@ class Fail2banRegex(object):
def testIgnoreRegex(self, line): def testIgnoreRegex(self, line):
found = False found = False
try: try:
ret = self._filter.ignoreLine(line) ret = self._filter.ignoreLine([(line, "", "")])
if ret is not None: if ret is not None:
found = True found = True
regex = self._ignoreregex[ret].inc() regex = self._ignoreregex[ret].inc()
@ -338,35 +338,27 @@ class Fail2banRegex(object):
return False return False
for bufLine in orgLineBuffer[int(fullBuffer):]: for bufLine in orgLineBuffer[int(fullBuffer):]:
if bufLine not in self._filter._Filter__lineBuffer: if bufLine not in self._filter._Filter__lineBuffer:
if self.removeMissedLine(bufLine): try:
self._line_stats.missed_lines.pop(
self._line_stats.missed_lines.index("".join(bufLine)))
self._line_stats.missed_lines_timeextracted.pop(
self._line_stats.missed_lines_timeextracted.index(
"".join(bufLine[::2])))
except ValueError:
pass
else:
self._line_stats.matched += 1 self._line_stats.matched += 1
return line, ret return line, ret
def removeMissedLine(self, line):
"""Remove `line` from missed lines, by comparing without time match"""
for n, missed_line in \
enumerate(reversed(self._line_stats.missed_lines)):
timeMatch = self._filter.dateDetector.matchTime(
missed_line, incHits=False)
if timeMatch:
logLine = (missed_line[:timeMatch.start()] +
missed_line[timeMatch.end():])
else:
logLine = missed_line
if logLine.rstrip("\r\n") == line:
self._line_stats.missed_lines.pop(
len(self._line_stats.missed_lines) - n - 1)
return True
return False
def process(self, test_lines): def process(self, test_lines):
for line_no, line in enumerate(test_lines): for line_no, line in enumerate(test_lines):
if line.startswith('#') or not line.strip(): line = line.strip('\r\n')
if line.startswith('#') or not line:
# skip comment and empty lines # skip comment and empty lines
continue continue
is_ignored = fail2banRegex.testIgnoreRegex(line)
line_datetimestripped, ret = fail2banRegex.testRegex(line) line_datetimestripped, ret = fail2banRegex.testRegex(line)
is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped)
if is_ignored: if is_ignored:
self._line_stats.ignored_lines.append(line) self._line_stats.ignored_lines.append(line)
@ -436,7 +428,7 @@ class Fail2banRegex(object):
" %s %s%s" % ( " %s %s%s" % (
ip[1], ip[1],
timeString, timeString,
ip[3] and " (multiple regex matched)" or "")) ip[-1] and " (multiple regex matched)" or ""))
print "\n%s: %d total" % (title, total) print "\n%s: %d total" % (title, total)
pprint_list(out, " #) [# of hits] regular expression") pprint_list(out, " #) [# of hits] regular expression")

59
fail2ban/server/failregex.py
View File

@ -74,8 +74,9 @@ class Regex:
# method of this object. # method of this object.
# @param value the line # @param value the line
def search(self, value): def search(self, tupleLines):
self._matchCache = self._regexObj.search(value) self._matchCache = self._regexObj.search(
"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
if self.hasMatched(): if self.hasMatched():
# Find start of the first line where the match was found # Find start of the first line where the match was found
try: try:
@ -89,8 +90,26 @@ class Regex:
"\n", self._matchCache.end() - 1) + 1 "\n", self._matchCache.end() - 1) + 1
except ValueError: except ValueError:
self._matchLineEnd = len(self._matchCache.string) self._matchLineEnd = len(self._matchCache.string)
##
lineCount1 = self._matchCache.string.count(
"\n", 0, self._matchLineStart)
lineCount2 = self._matchCache.string.count(
"\n", 0, self._matchLineEnd)
self._matchedTupleLines = tupleLines[lineCount1:lineCount2]
self._unmatchedTupleLines = tupleLines[:lineCount1]
n = 0
for skippedLine in self.getSkippedLines():
for m, matchedTupleLine in enumerate(
self._matchedTupleLines[n:]):
if "".join(matchedTupleLine[::2]) == skippedLine:
self._unmatchedTupleLines.append(
self._matchedTupleLines.pop(n+m))
n += m
break
self._unmatchedTupleLines.extend(tupleLines[lineCount2:])
# Checks if the previous call to search() matched. # Checks if the previous call to search() matched.
# #
# @return True if a match was found, False otherwise # @return True if a match was found, False otherwise
@ -114,7 +133,8 @@ class Regex:
n = 0 n = 0
while True: while True:
try: try:
skippedLines += self._matchCache.group("skiplines%i" % n) if self._matchCache.group("skiplines%i" % n) is not None:
skippedLines += self._matchCache.group("skiplines%i" % n)
n += 1 n += 1
except IndexError: except IndexError:
break break
@ -125,15 +145,18 @@ class Regex:
# #
# This returns unmatched lines including captured by the <SKIPLINES> tag. # This returns unmatched lines including captured by the <SKIPLINES> tag.
# @return list of unmatched lines # @return list of unmatched lines
def getUnmatchedTupleLines(self):
if not self.hasMatched():
return []
else:
return self._unmatchedTupleLines
def getUnmatchedLines(self): def getUnmatchedLines(self):
if not self.hasMatched(): if not self.hasMatched():
return [] return []
unmatchedLines = ( else:
self._matchCache.string[:self._matchLineStart].splitlines(False) return ["".join(line) for line in self._unmatchedTupleLines]
+ self.getSkippedLines()
+ self._matchCache.string[self._matchLineEnd:].splitlines(False))
return unmatchedLines
## ##
# Returns matched lines. # Returns matched lines.
@ -141,14 +164,18 @@ class Regex:
# This returns matched lines by excluding those captured # This returns matched lines by excluding those captured
# by the <SKIPLINES> tag. # by the <SKIPLINES> tag.
# @return list of matched lines # @return list of matched lines
def getMatchedTupleLines(self):
if not self.hasMatched():
return []
else:
return self._matchedTupleLines
def getMatchedLines(self): def getMatchedLines(self):
if not self.hasMatched(): if not self.hasMatched():
return [] return []
matchedLines = self._matchCache.string[ else:
self._matchLineStart:self._matchLineEnd].splitlines(False) return ["".join(line) for line in self._matchedTupleLines]
return [line for line in matchedLines
if line not in self.getSkippedLines()]
## ##
# Exception dedicated to the class Regex. # Exception dedicated to the class Regex.

85
fail2ban/server/filter.py
View File

@ -366,17 +366,15 @@ class Filter(JailThread):
timeMatch = self.dateDetector.matchTime(l) timeMatch = self.dateDetector.matchTime(l)
if timeMatch: if timeMatch:
# Lets split into time part and log part of the line tupleLine = (
timeText = timeMatch.group() l[:timeMatch.start()],
# Lets leave the beginning in as well, so if there is no l[timeMatch.start():timeMatch.end()],
# anchore at the beginning of the time regexp, we don't l[timeMatch.end():])
# at least allow injection. Should be harmless otherwise
logLine = l[:timeMatch.start()] + l[timeMatch.end():]
else: else:
timeText = None tupleLine = (l, "", "")
logLine = l
return logLine, self.findFailure(timeText, logLine, returnRawHost, checkAllRegex) return "".join(tupleLine[::2]), self.findFailure(
tupleLine, returnRawHost, checkAllRegex)
def processLineAndAdd(self, line): def processLineAndAdd(self, line):
"""Processes the line for failures and populates failManager """Processes the line for failures and populates failManager
@ -385,6 +383,7 @@ class Filter(JailThread):
failregex = element[0] failregex = element[0]
ip = element[1] ip = element[1]
unixTime = element[2] unixTime = element[2]
lines = element[3]
logSys.debug("Processing line with time:%s and ip:%s" logSys.debug("Processing line with time:%s and ip:%s"
% (unixTime, ip)) % (unixTime, ip))
if unixTime < MyTime.time() - self.getFindTime(): if unixTime < MyTime.time() - self.getFindTime():
@ -396,7 +395,7 @@ class Filter(JailThread):
continue continue
logSys.debug("Found %s" % ip) logSys.debug("Found %s" % ip)
## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),)
self.failManager.addFailure(FailTicket(ip, unixTime, [line])) self.failManager.addFailure(FailTicket(ip, unixTime, lines))
## ##
# Returns true if the line should be ignored. # Returns true if the line should be ignored.
@ -405,9 +404,9 @@ class Filter(JailThread):
# @param line: the line # @param line: the line
# @return: a boolean # @return: a boolean
def ignoreLine(self, line): def ignoreLine(self, tupleLines):
for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
ignoreRegex.search(line) ignoreRegex.search(tupleLines)
if ignoreRegex.hasMatched(): if ignoreRegex.hasMatched():
return ignoreRegexIndex return ignoreRegexIndex
return None return None
@ -419,17 +418,17 @@ class Filter(JailThread):
# to find the logging time. # to find the logging time.
# @return a dict with IP and timestamp. # @return a dict with IP and timestamp.
def findFailure(self, timeText, logLine, def findFailure(self, tupleLine, returnRawHost=False, checkAllRegex=False):
returnRawHost=False, checkAllRegex=False):
failList = list() failList = list()
# Checks if we must ignore this line. # Checks if we must ignore this line.
if self.ignoreLine(logLine) is not None: if self.ignoreLine([tupleLine[::2]]) is not None:
# The ignoreregex matched. Return. # The ignoreregex matched. Return.
logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine) logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
"".join(tupleLine[::2]))
return failList return failList
timeText = tupleLine[1]
if timeText: if timeText:
dateTimeMatch = self.dateDetector.getTime(timeText) dateTimeMatch = self.dateDetector.getTime(timeText)
@ -446,49 +445,53 @@ class Filter(JailThread):
self.__lastTimeText = timeText self.__lastTimeText = timeText
self.__lastDate = date self.__lastDate = date
else: else:
timeText = self.__lastTimeText or logLine timeText = self.__lastTimeText or "".join(tupleLine[::2])
date = self.__lastDate date = self.__lastDate
self.__lineBuffer = (self.__lineBuffer + [logLine])[-self.__lineBufferSize:] self.__lineBuffer = (
self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:]
logLine = "\n".join(self.__lineBuffer) + "\n"
# Iterates over all the regular expressions. # Iterates over all the regular expressions.
for failRegexIndex, failRegex in enumerate(self.__failRegex): for failRegexIndex, failRegex in enumerate(self.__failRegex):
failRegex.search(logLine) failRegex.search(self.__lineBuffer)
if failRegex.hasMatched(): if failRegex.hasMatched():
# Checks if we must ignore this match.
if self.ignoreLine(
"\n".join(failRegex.getMatchedLines()) + "\n") \
is not None:
# The ignoreregex matched. Remove ignored match.
self.__lineBuffer = failRegex.getUnmatchedLines()
logSys.log(7, "Matched ignoreregex and was ignored")
continue
# The failregex matched. # The failregex matched.
logSys.log(7, "Matched %s", failRegex) logSys.log(7, "Matched %s", failRegex)
# Checks if we must ignore this match.
if self.ignoreLine(failRegex.getMatchedTupleLines()) \
is not None:
# The ignoreregex matched. Remove ignored match.
self.__lineBuffer = failRegex.getUnmatchedTupleLines()
logSys.log(7, "Matched ignoreregex and was ignored")
if not checkAllRegex:
break
else:
continue
if date is None: if date is None:
logSys.debug("Found a match for %r but no valid date/time " logSys.debug(
"found for %r. Please try setting a custom " "Found a match for %r but no valid date/time "
"date pattern (see man page jail.conf(5)). " "found for %r. Please try setting a custom "
"If format is complex, please " "date pattern (see man page jail.conf(5)). "
"file a detailed issue on" "If format is complex, please "
" https://github.com/fail2ban/fail2ban/issues " "file a detailed issue on"
"in order to get support for this format." " https://github.com/fail2ban/fail2ban/issues "
% (logLine, timeText)) "in order to get support for this format."
% ("\n".join(failRegex.getMatchedLines()), timeText))
else: else:
self.__lineBuffer = failRegex.getUnmatchedLines() self.__lineBuffer = failRegex.getUnmatchedTupleLines()
try: try:
host = failRegex.getHost() host = failRegex.getHost()
if returnRawHost: if returnRawHost:
failList.append([failRegexIndex, host, date]) failList.append([failRegexIndex, host, date,
failRegex.getMatchedLines()])
if not checkAllRegex: if not checkAllRegex:
break break
else: else:
ipMatch = DNSUtils.textToIp(host, self.__useDns) ipMatch = DNSUtils.textToIp(host, self.__useDns)
if ipMatch: if ipMatch:
for ip in ipMatch: for ip in ipMatch:
failList.append([failRegexIndex, ip, date]) failList.append([failRegexIndex, ip, date,
failRegex.getMatchedLines()])
if not checkAllRegex: if not checkAllRegex:
break break
except RegexException, e: # pragma: no cover - unsure if reachable except RegexException, e: # pragma: no cover - unsure if reachable

15
fail2ban/tests/filtertestcase.py
View File

@ -710,7 +710,7 @@ class GetFailures(unittest.TestCase):
# so that they could be reused by other tests # so that they could be reused by other tests
FAILURES_01 = ('193.168.0.128', 3, 1124017199.0, FAILURES_01 = ('193.168.0.128', 3, 1124017199.0,
[u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3)
def setUp(self): def setUp(self):
"""Call before every test case.""" """Call before every test case."""
@ -747,16 +747,13 @@ class GetFailures(unittest.TestCase):
fout.close() fout.close()
# now see if we should be getting the "same" failures # now see if we should be getting the "same" failures
self.testGetFailures01(filename=fname, self.testGetFailures01(filename=fname)
failures=GetFailures.FAILURES_01[:3] +
([x.rstrip('\n') + '\r\n' for x in
GetFailures.FAILURES_01[-1]],))
_killfile(fout, fname) _killfile(fout, fname)
def testGetFailures02(self): def testGetFailures02(self):
output = ('141.3.81.106', 4, 1124017139.0, output = ('141.3.81.106', 4, 1124017139.0,
[u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2'
% m for m in 53, 54, 57, 58]) % m for m in 53, 54, 57, 58])
self.filter.addLogPath(GetFailures.FILENAME_02) self.filter.addLogPath(GetFailures.FILENAME_02)
@ -789,11 +786,11 @@ class GetFailures(unittest.TestCase):
def testGetFailuresUseDNS(self): def testGetFailuresUseDNS(self):
# We should still catch failures with usedns = no ;-) # We should still catch failures with usedns = no ;-)
output_yes = ('93.184.216.119', 2, 1124017139.0, output_yes = ('93.184.216.119', 2, 1124017139.0,
[u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2'])
output_no = ('93.184.216.119', 1, 1124017139.0, output_no = ('93.184.216.119', 1, 1124017139.0,
[u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2'])
# Actually no exception would be raised -- it will be just set to 'no' # Actually no exception would be raised -- it will be just set to 'no'
#self.assertRaises(ValueError, #self.assertRaises(ValueError,

2
fail2ban/tests/samplestestcase.py
View File

@ -119,7 +119,7 @@ def testSampleRegexsFactory(name):
(map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno())) (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
# Verify timestamp and host as expected # Verify timestamp and host as expected
failregex, host, fail2banTime = ret[0] failregex, host, fail2banTime, lines = ret[0]
self.assertEqual(host, faildata.get("host", None)) self.assertEqual(host, faildata.get("host", None))
t = faildata.get("time", None) t = faildata.get("time", None)