processLine etc. rewritten:

- normalize calling parameters (persistent parameters moved from function arguments to filter member variables)
- save last line as lambda instead of return it as string (lazy convert of process line tuple to string on demand, needed in fail2ban-regex only)
pull/1616/head
sebres 2016-11-18 14:42:29 +01:00
parent ea4c1f6356
commit 189e70d99c
3 changed files with 26 additions and 14 deletions

View File

@ -247,6 +247,9 @@ class Fail2banRegex(object):
self.raw = True if opts.raw else False self.raw = True if opts.raw else False
if opts.usedns: if opts.usedns:
self._filter.setUseDns(opts.usedns) self._filter.setUseDns(opts.usedns)
self._filter.returnRawHost = self.raw
self._filter.checkFindTime = False
self._filter.checkAllRegex = True
def decode_line(self, line): def decode_line(self, line):
return FileContainer.decode_line('<LOG>', self.encoding, line) return FileContainer.decode_line('<LOG>', self.encoding, line)
@ -350,7 +353,8 @@ class Fail2banRegex(object):
orgLineBuffer = self._filter._Filter__lineBuffer orgLineBuffer = self._filter._Filter__lineBuffer
fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
try: try:
line, ret = self._filter.processLine(line, date, checkAllRegex=True, returnRawHost=self.raw) ret = self._filter.processLine(line, date)
line = self._filter.processedLine()
for match in ret: for match in ret:
# Append True/False flag depending if line was matched by # Append True/False flag depending if line was matched by
# more than one regex # more than one regex

View File

@ -90,6 +90,12 @@ class Filter(JailThread):
## Error counter (protected, so can be used in filter implementations) ## Error counter (protected, so can be used in filter implementations)
## if it reached 100 (at once), run-cycle will go idle ## if it reached 100 (at once), run-cycle will go idle
self._errors = 0 self._errors = 0
## return raw host (host is not dns):
self.returnRawHost = False
## check each regex (used for test purposes):
self.checkAllRegex = False
## if true ignores obsolete failures (failure time < now - findTime):
self.checkFindTime = True
## Ticks counter ## Ticks counter
self.ticks = 0 self.ticks = 0
@ -455,8 +461,7 @@ class Filter(JailThread):
return False return False
def processLine(self, line, date=None, returnRawHost=False, def processLine(self, line, date=None):
checkAllRegex=False, checkFindTime=False):
"""Split the time portion from log msg and return findFailures on them """Split the time portion from log msg and return findFailures on them
""" """
if date: if date:
@ -476,14 +481,15 @@ class Filter(JailThread):
else: else:
tupleLine = (l, "", "", None) tupleLine = (l, "", "", None)
return "".join(tupleLine[::2]), self.findFailure( # save last line (lazy convert of process line tuple to string on demand):
tupleLine, date, returnRawHost, checkAllRegex, checkFindTime) self.processedLine = lambda: "".join(tupleLine[::2])
return self.findFailure(tupleLine, date)
def processLineAndAdd(self, line, date=None): def processLineAndAdd(self, line, date=None):
"""Processes the line for failures and populates failManager """Processes the line for failures and populates failManager
""" """
try: try:
for element in self.processLine(line, date, checkFindTime=True)[1]: for element in self.processLine(line, date):
ip = element[1] ip = element[1]
unixTime = element[2] unixTime = element[2]
lines = element[3] lines = element[3]
@ -539,10 +545,10 @@ class Filter(JailThread):
# to find the logging time. # to find the logging time.
# @return a dict with IP and timestamp. # @return a dict with IP and timestamp.
def findFailure(self, tupleLine, date=None, returnRawHost=False, def findFailure(self, tupleLine, date=None):
checkAllRegex=False, checkFindTime=False):
failList = list() failList = list()
returnRawHost = self.returnRawHost
cidr = IPAddr.CIDR_UNSPEC cidr = IPAddr.CIDR_UNSPEC
if self.__useDns == "raw": if self.__useDns == "raw":
returnRawHost = True returnRawHost = True
@ -577,7 +583,7 @@ class Filter(JailThread):
timeText = self.__lastTimeText or "".join(tupleLine[::2]) timeText = self.__lastTimeText or "".join(tupleLine[::2])
date = self.__lastDate date = self.__lastDate
if checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
logSys.log(5, "Ignore line since time %s < %s - %s", logSys.log(5, "Ignore line since time %s < %s - %s",
date, MyTime.time(), self.getFindTime()) date, MyTime.time(), self.getFindTime())
return failList return failList
@ -598,7 +604,7 @@ class Filter(JailThread):
# The ignoreregex matched. Remove ignored match. # The ignoreregex matched. Remove ignored match.
self.__lineBuffer = failRegex.getUnmatchedTupleLines() self.__lineBuffer = failRegex.getUnmatchedTupleLines()
logSys.log(7, "Matched ignoreregex and was ignored") logSys.log(7, "Matched ignoreregex and was ignored")
if not checkAllRegex: if not self.checkAllRegex:
break break
else: else:
continue continue
@ -641,7 +647,7 @@ class Filter(JailThread):
ip = IPAddr(fid, IPAddr.CIDR_RAW) ip = IPAddr(fid, IPAddr.CIDR_RAW)
failList.append([failRegexIndex, ip, date, failList.append([failRegexIndex, ip, date,
failRegex.getMatchedLines(), fail]) failRegex.getMatchedLines(), fail])
if not checkAllRegex: if not self.checkAllRegex:
break break
else: else:
ips = DNSUtils.textToIp(host, self.__useDns) ips = DNSUtils.textToIp(host, self.__useDns)
@ -649,7 +655,7 @@ class Filter(JailThread):
for ip in ips: for ip in ips:
failList.append([failRegexIndex, ip, date, failList.append([failRegexIndex, ip, date,
failRegex.getMatchedLines(), fail]) failRegex.getMatchedLines(), fail])
if not checkAllRegex: if not self.checkAllRegex:
break break
except RegexException as e: # pragma: no cover - unsure if reachable except RegexException as e: # pragma: no cover - unsure if reachable
logSys.error(e) logSys.error(e)

View File

@ -44,6 +44,9 @@ class FilterSamplesRegex(unittest.TestCase):
def setUp(self): def setUp(self):
"""Call before every test case.""" """Call before every test case."""
self.filter = Filter(None) self.filter = Filter(None)
self.filter.returnRawHost = True
self.filter.checkAllRegex = True
self.filter.checkFindTime = False
self.filter.active = True self.filter.active = True
setUpMyTime() setUpMyTime()
@ -111,8 +114,7 @@ def testSampleRegexsFactory(name, basedir):
else: else:
faildata = {} faildata = {}
ret = self.filter.processLine( ret = self.filter.processLine(line)
line, returnRawHost=True, checkAllRegex=True)[1]
if not ret: if not ret:
# Check line is flagged as none match # Check line is flagged as none match
self.assertFalse(faildata.get('match', True), self.assertFalse(faildata.get('match', True),