added possibility to specify more precise default date pattern:

- `datepattern = {^LN-BEG}` - only line-begin anchored default patterns
     (matches date only at begin of line, or with max distance up to 2 non-alphanumeric characters from line-begin);
  - `datepattern = {*WD-BEG}` - only word-begin anchored default patterns;
  - `datepattern = ^prefix{DATE}suffix` - exact specified default patterns (using prefix and suffix);
common filter configs gets a more precise, line-begin anchored (datepattern = {^LN-BEG}) resp. custom anchoring default date-patterns;
pull/1583/head
sebres 2016-10-05 19:34:21 +02:00
parent f56ff5f48b
commit ab0ac2111c
29 changed files with 120 additions and 57 deletions

View File

@ -9,6 +9,8 @@ failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ <HOST>:\d+ [\d.]+:\d+ \d+ \d+ \d+\s
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# DEV Notes: # DEV Notes:
# http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are # http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are
# all authentication problems (%E field) # all authentication problems (%E field)

View File

@ -10,6 +10,8 @@ after = apache-common.local
_apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\] _apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
datepattern = {^LN-BEG}
# Common prefix for [error] apache messages which also would include <HOST> # Common prefix for [error] apache messages which also would include <HOST>
# Depending on the version it could be # Depending on the version it could be
# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]

View File

@ -3,10 +3,6 @@
# #
# The knocking request must have a referer. # The knocking request must have a referer.
[INCLUDES]
before = apache-common.conf
[Definition] [Definition]
failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$ failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$

View File

@ -20,6 +20,8 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# DEV Notes: # DEV Notes:
# V1 Examples matches: # V1 Examples matches:
# Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); # Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);

View File

@ -31,6 +31,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# Author: Xavier Devlamynck / Daniel Black # Author: Xavier Devlamynck / Daniel Black
# #

View File

@ -61,4 +61,7 @@ __prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostn
# pam_ldap # pam_ldap
__pam_auth = pam_unix __pam_auth = pam_unix
# standardly all formats using prefix have line-begin anchored date:
datepattern = {^LN-BEG}
# Author: Yaroslav Halchenko # Author: Yaroslav Halchenko

View File

@ -8,8 +8,6 @@ failregex = ^: Bad Rcon: "rcon \d+ "\S+" sv_contact ".*?"" from "<HOST>:\d+"$
ignoreregex = ignoreregex =
[Init]
datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S

View File

@ -15,5 +15,7 @@ failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# Author: Christoph Haas # Author: Christoph Haas
# Modified by: Cyril Jaquier # Modified by: Cyril Jaquier

View File

@ -13,7 +13,6 @@ failregex = ^: \'<HOST>\' \d{1,3} failed login attempt(s)?. \s*
ignoreregex = ignoreregex =
[Init]
datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S
# #

View File

@ -26,6 +26,8 @@ failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challe
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# Author: Rupa SChomaker, soapee01, Daniel Black # Author: Rupa SChomaker, soapee01, Daniel Black
# https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban # https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban
# Thanks to Jim on mailing list of samples and guidance # Thanks to Jim on mailing list of samples and guidance

View File

@ -9,8 +9,6 @@ failregex = ^ SMTP Spam attack detected from <HOST>,
ignoreregex = ignoreregex =
[Init]
datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\] datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
# DEV NOTES: # DEV NOTES:

View File

@ -13,7 +13,7 @@ before = common.conf
_daemon = monit _daemon = monit
# Regexp for previous (accessing monit httpd) and new (access denied) versions # Regexp for previous (accessing monit httpd) and new (access denied) versions
failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$ ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
# Ignore login with empty user (first connect, no user specified) # Ignore login with empty user (first connect, no user specified)

View File

@ -15,13 +15,14 @@ _daemon = murmurd
# variable in your server config file (murmur.ini / mumble-server.ini). # variable in your server config file (murmur.ini / mumble-server.ini).
_usernameregex = [^>]+ _usernameregex = [^>]+
_prefix = <W>[\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+: _prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
failregex = ^%(_prefix)s Invalid server password$ failregex = ^%(_prefix)s Invalid server password$
^%(_prefix)s Wrong certificate or password for existing user$ ^%(_prefix)s Wrong certificate or password for existing user$
ignoreregex = ignoreregex =
datepattern = ^<W>{DATE}
# DEV Notes: # DEV Notes:
# #

View File

@ -8,6 +8,8 @@ failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not f
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}
# DEV NOTES: # DEV NOTES:
# Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files
# Extensive search of all nginx auth failures not done yet. # Extensive search of all nginx auth failures not done yet.

View File

@ -43,3 +43,4 @@ failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by z
ignoreregex = ignoreregex =
datepattern = {^LN-BEG}

View File

@ -9,7 +9,6 @@
[Definition] [Definition]
failregex = ^<HOST>\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$ failregex = ^<HOST>\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$
[Init]
datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z

View File

@ -52,10 +52,12 @@ before = common.conf
# Note that you MUST have LOG_FORMAT=4 for this to work! # Note that you MUST have LOG_FORMAT=4 for this to work!
# #
failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$ failregex = tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
# Option: ignoreregex # Option: ignoreregex
# Notes.: regex to ignore. If this regex matches, the line is ignored. # Notes.: regex to ignore. If this regex matches, the line is ignored.
# Values: TEXT # Values: TEXT
# #
ignoreregex = ignoreregex =
datepattern = ^<co ts="{DATE}"\s+

View File

@ -18,4 +18,6 @@ failregex = ^type=%(_type)s msg=audit\(:\d+\): (user )?pid=\d+ uid=%(_uid)s auid
ignoreregex = ignoreregex =
datepattern = EPOCH
# Author: Daniel Black # Author: Daniel Black

View File

@ -6,7 +6,9 @@
failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$ failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$
ignoreregex = ignoreregex = "^<ADDR>"
datepattern = {^LN-BEG}
# #
# DEV Notes: # DEV Notes:

View File

@ -5,8 +5,6 @@ failregex = ^ \[LOGIN_ERROR\].*from <HOST>: Unknown user or password incorrect\.
ignoreregex = ignoreregex =
[Init]
datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S
# DEV NOTES: # DEV NOTES:

View File

@ -38,13 +38,13 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro
ignoreregex = ignoreregex =
[Init]
# "maxlines" is number of log lines to buffer for multi-line regex searches # "maxlines" is number of log lines to buffer for multi-line regex searches
maxlines = 10 maxlines = 10
journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
datepattern = {^LN-BEG}
# DEV Notes: # DEV Notes:
# #
# "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because # "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because

View File

@ -474,8 +474,9 @@ class Fail2banRegex(object):
if self._verbose or template.hits: if self._verbose or template.hits:
out.append("[%d] %s" % (template.hits, template.name)) out.append("[%d] %s" % (template.hits, template.name))
if self._verbose_date: if self._verbose_date:
out.append(" # weight: %3s, pattern: %s" % ( out.append(" # weight: %.3f (%.3f), pattern: %s" % (
template.weight, getattr(template, 'pattern', ''),)) template.weight, template.template.weight,
getattr(template, 'pattern', ''),))
out.append(" # regex: %s" % (getattr(template, 'regex', ''),)) out.append(" # regex: %s" % (getattr(template, 'regex', ''),))
pprint_list(out, "[# of hits] date format") pprint_list(out, "[# of hits] date format")

View File

@ -40,6 +40,9 @@ class FilterReader(DefinitionInitConfigReader):
_configOpts = { _configOpts = {
"ignoreregex": ["string", None], "ignoreregex": ["string", None],
"failregex": ["string", ""], "failregex": ["string", ""],
"maxlines": ["int", None],
"datepattern": ["string", None],
"journalmatch": ["string", None],
} }
def setFile(self, fileName): def setFile(self, fileName):
@ -74,16 +77,16 @@ class FilterReader(DefinitionInitConfigReader):
stream.append(["multi-set", self._jailName, "add" + opt, multi]) stream.append(["multi-set", self._jailName, "add" + opt, multi])
elif len(multi): elif len(multi):
stream.append(["set", self._jailName, "add" + opt, multi[0]]) stream.append(["set", self._jailName, "add" + opt, multi[0]])
if self._initOpts: elif opt == 'maxlines':
if 'maxlines' in self._initOpts:
# We warn when multiline regex is used without maxlines > 1 # We warn when multiline regex is used without maxlines > 1
# therefore keep sure we set this option first. # therefore keep sure we set this option first.
stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]]) stream.insert(0, ["set", self._jailName, "maxlines", value])
if 'datepattern' in self._initOpts: elif opt == 'datepattern':
stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]]) stream.append(["set", self._jailName, "datepattern", value])
# Do not send a command if the match is empty. # Do not send a command if the match is empty.
if self._initOpts.get("journalmatch", '') != '': elif opt == 'journalmatch':
for match in self._initOpts["journalmatch"].split("\n"): for match in value.split("\n"):
if match == '': continue
stream.append( stream.append(
["set", self._jailName, "addjournalmatch"] + ["set", self._jailName, "addjournalmatch"] +
shlex.split(match)) shlex.split(match))

View File

@ -26,7 +26,7 @@ import time
from threading import Lock from threading import Lock
from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
from ..helpers import getLogger from ..helpers import getLogger
# Gets the instance of the logger. # Gets the instance of the logger.
@ -34,6 +34,8 @@ logSys = getLogger(__name__)
logLevel = 6 logLevel = 6
RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE)
class DateDetectorCache(object): class DateDetectorCache(object):
"""Implements the caching of the default templates list. """Implements the caching of the default templates list.
@ -54,15 +56,18 @@ class DateDetectorCache(object):
self._addDefaultTemplate() self._addDefaultTemplate()
return self.__templates return self.__templates
def _cacheTemplate(self, template): def _cacheTemplate(self, template, lineBeginOnly=False):
"""Cache Fail2Ban's default template. """Cache Fail2Ban's default template.
""" """
if isinstance(template, str): if isinstance(template, str):
# exact given template with word benin-end boundary: # exact given template with word benin-end boundary:
template = DatePatternRegex(template) if not lineBeginOnly:
template = DatePatternRegex(template)
else:
template = DatePatternRegex(template, wordBegin='start')
# additional template, that prefers datetime at start of a line (safety+performance feature): # additional template, that prefers datetime at start of a line (safety+performance feature):
if hasattr(template, 'regex'): if not lineBeginOnly and hasattr(template, 'regex'):
template2 = copy.copy(template) template2 = copy.copy(template)
regex = getattr(template, 'pattern', template.regex) regex = getattr(template, 'pattern', template.regex)
template2.setRegex(regex, wordBegin='start', wordEnd=True) template2.setRegex(regex, wordBegin='start', wordEnd=True)
@ -90,6 +95,7 @@ class DateDetectorCache(object):
# prefixed with optional time zone (monit): # prefixed with optional time zone (monit):
# PDT Apr 16 21:05:29 # PDT Apr 16 21:05:29
self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?") self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
# asctime with optional day, subsecond and/or year coming after day # asctime with optional day, subsecond and/or year coming after day
# http://bugs.debian.org/798923 # http://bugs.debian.org/798923
# Sun Jan 23 2005 21:59:59.011 # Sun Jan 23 2005 21:59:59.011
@ -114,17 +120,18 @@ class DateDetectorCache(object):
# TAI64N # TAI64N
self._cacheTemplate(DateTai64n()) self._cacheTemplate(DateTai64n())
# Epoch # Epoch
self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
self._cacheTemplate(DateEpoch()) self._cacheTemplate(DateEpoch())
# Only time information in the log # Only time information in the log
self._cacheTemplate("^%H:%M:%S") self._cacheTemplate("%H:%M:%S", lineBeginOnly=True)
# <09/16/08@05:03:30> # <09/16/08@05:03:30>
self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>") self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True)
# MySQL: 130322 11:46:11 # MySQL: 130322 11:46:11
self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S") self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S")
# Apache Tomcat # Apache Tomcat
self._cacheTemplate("%b %d, %ExY %I:%M:%S %p") self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
# ASSP: Apr-27-13 02:33:06 # ASSP: Apr-27-13 02:33:06
self._cacheTemplate("^%b-%d-%Exy %H:%M:%S") self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True)
self.__templates = self.__tmpcache[0] + self.__tmpcache[1] self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
del self.__tmpcache del self.__tmpcache
@ -172,6 +179,8 @@ class DateDetector(object):
self.__lastTemplIdx = 0x7fffffff self.__lastTemplIdx = 0x7fffffff
# first free place: # first free place:
self.__firstUnused = 0 self.__firstUnused = 0
# pre-match pattern:
self.__preMatch = None
def _appendTemplate(self, template): def _appendTemplate(self, template):
name = template.name name = template.name
@ -200,10 +209,18 @@ class DateDetector(object):
template = DatePatternRegex(template) template = DatePatternRegex(template)
self._appendTemplate(template) self._appendTemplate(template)
def addDefaultTemplate(self): def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
"""Add Fail2Ban's default set of date templates. """Add Fail2Ban's default set of date templates.
""" """
for template in DateDetector._defCache.templates: for template in DateDetector._defCache.templates:
# filter if specified:
if filterTemplate is not None and not filterTemplate(template): continue
# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
if preMatch is not None:
regex = getattr(template, 'pattern', template.regex)
template = copy.copy(template)
template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
# append date detector template:
self._appendTemplate(template) self._appendTemplate(template)
@property @property
@ -230,7 +247,7 @@ class DateDetector(object):
The regex match returned from the first successfully matched The regex match returned from the first successfully matched
template. template.
""" """
#logSys.log(logLevel, "try to match time for line: %.250s", line) #logSys.log(logLevel, "try to match time for line: %.120s", line)
match = None match = None
# first try to use last template with same start/end position: # first try to use last template with same start/end position:
i = self.__lastTemplIdx i = self.__lastTemplIdx
@ -238,13 +255,16 @@ class DateDetector(object):
ddtempl = self.__templates[i] ddtempl = self.__templates[i]
template = ddtempl.template template = ddtempl.template
distance, endpos = self.__lastPos[0], self.__lastEndPos[0] distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
if logSys.getEffectiveLevel() <= logLevel-1:
logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
i, distance, endpos,
line[distance-1:distance], self.__lastPos[1],
line[distance:endpos],
line[endpos:endpos+1], self.__lastEndPos[1])
# check same boundaries left/right, otherwise possible collision/pattern switch: # check same boundaries left/right, otherwise possible collision/pattern switch:
if (line[distance-1:distance] == self.__lastPos[1] and if (line[distance-1:distance] == self.__lastPos[1] and
line[endpos:endpos+1] == self.__lastEndPos[1] line[endpos:endpos+1] == self.__lastEndPos[1]
): ):
if logSys.getEffectiveLevel() <= logLevel-1:
logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ... %s ...",
i, distance, endpos, line[distance:endpos])
match = template.matchDate(line, distance, endpos) match = template.matchDate(line, distance, endpos)
if match: if match:
distance = match.start() distance = match.start()
@ -270,8 +290,8 @@ class DateDetector(object):
if logSys.getEffectiveLevel() <= logLevel: if logSys.getEffectiveLevel() <= logLevel:
logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s",
i, distance, ddtempl.distance, self.__lastPos[0], template.name) i, distance, ddtempl.distance, self.__lastPos[0], template.name)
## if line-begin anchored - stop searching: ## if line-begin/end anchored - stop searching:
if template.flags & DateTemplate.LINE_BEGIN: if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
break break
## [grave] if distance changed, possible date-match was found somewhere ## [grave] if distance changed, possible date-match was found somewhere
## in body of message, so save this template, and search further: ## in body of message, so save this template, and search further:

View File

@ -33,9 +33,12 @@ from ..helpers import getLogger
logSys = getLogger(__name__) logSys = getLogger(__name__)
RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)') RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$|\*\*)$') RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
RE_DEL_WRD_BOUNDS = re.compile(r'^\*\*|(?<!\\)\*\*$') RE_DEL_WRD_BOUNDS = re.compile(r'^\*\*|(?<!\\)\*\*$')
RE_LINE_BOUND_BEG = re.compile(r'^(?:\^|\(\?:\^(?!\|))')
RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
class DateTemplate(object): class DateTemplate(object):
"""A template which searches for and returns a date from a log line. """A template which searches for and returns a date from a log line.
@ -49,6 +52,7 @@ class DateTemplate(object):
""" """
LINE_BEGIN = 8 LINE_BEGIN = 8
LINE_END = 4
WORD_BEGIN = 2 WORD_BEGIN = 2
WORD_END = 1 WORD_END = 1
@ -86,6 +90,7 @@ class DateTemplate(object):
If regular expression fails to compile If regular expression fails to compile
""" """
regex = regex.strip() regex = regex.strip()
self.flags = 0
# if word or line start boundary: # if word or line start boundary:
if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex): if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
@ -96,9 +101,12 @@ class DateTemplate(object):
self.flags |= DateTemplate.WORD_END self.flags |= DateTemplate.WORD_END
regex += r'(?=\b|\W|$)' regex += r'(?=\b|\W|$)'
self.name += '{*WD-END}' self.name += '{*WD-END}'
if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
# remove possible special pattern "**" in front and end of regex: # remove possible special pattern "**" in front and end of regex:
regex = RE_DEL_WRD_BOUNDS.sub('', regex) regex = RE_DEL_WRD_BOUNDS.sub('', regex)
self._regex = regex self._regex = regex
self._cRegex = None
regex = property(getRegex, setRegex, doc= regex = property(getRegex, setRegex, doc=
"""Regex used to search for date. """Regex used to search for date.
@ -151,11 +159,15 @@ class DateEpoch(DateTemplate):
regex regex
""" """
def __init__(self): def __init__(self, lineBeginOnly=False):
DateTemplate.__init__(self) DateTemplate.__init__(self)
self.name = "Epoch" self.name = "Epoch"
self.setRegex(r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))", if not lineBeginOnly:
wordBegin=False) ;# already line begin resp. word begin anchored regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
else:
regex = r"(?P<square>(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))"
self.setRegex(regex, wordBegin='start', wordEnd=True)
def getDate(self, line, dateMatch=None): def getDate(self, line, dateMatch=None):
"""Method to return the date for a log line. """Method to return the date for a log line.

View File

@ -35,7 +35,7 @@ from .ipdns import DNSUtils, IPAddr
from .ticket import FailTicket from .ticket import FailTicket
from .jailthread import JailThread from .jailthread import JailThread
from .datedetector import DateDetector from .datedetector import DateDetector
from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n
from .mytime import MyTime from .mytime import MyTime
from .failregex import FailRegex, Regex, RegexException from .failregex import FailRegex, Regex, RegexException
from .action import CommandAction from .action import CommandAction
@ -257,14 +257,29 @@ class Filter(JailThread):
if pattern is None: if pattern is None:
self.dateDetector = None self.dateDetector = None
return return
elif pattern.upper() == "EPOCH":
template = DateEpoch()
template.name = "Epoch"
elif pattern.upper() == "TAI64N":
template = DateTai64n()
template.name = "TAI64N"
else: else:
template = DatePatternRegex(pattern) key = pattern.upper()
if key == "EPOCH":
template = DateEpoch()
template.name = "Epoch"
elif key == "TAI64N":
template = DateTai64n()
template.name = "TAI64N"
elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
self.dateDetector = DateDetector()
flt = \
lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
None
self.dateDetector.addDefaultTemplate(flt)
return
elif "{DATE}" in key:
self.dateDetector = DateDetector()
self.dateDetector.addDefaultTemplate(
lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
return
else:
template = DatePatternRegex(pattern)
self.dateDetector = DateDetector() self.dateDetector = DateDetector()
self.dateDetector.appendTemplate(template) self.dateDetector.appendTemplate(template)
logSys.info(" date pattern `%r`: `%s`", logSys.info(" date pattern `%r`: `%s`",
@ -280,9 +295,9 @@ class Filter(JailThread):
def getDatePattern(self): def getDatePattern(self):
if self.dateDetector is not None: if self.dateDetector is not None:
templates = self.dateDetector.templates templates = self.dateDetector.templates
if len(templates) > 1: if len(templates) > 2:
return None, "Default Detectors" return None, "Default Detectors"
elif len(templates) == 1: elif len(templates):
if hasattr(templates[0], "pattern"): if hasattr(templates[0], "pattern"):
pattern = templates[0].pattern pattern = templates[0].pattern
else: else:

View File

@ -40,6 +40,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): #todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
#timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] #timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)" timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
# Extend build-in TimeRE with some exact patterns # Extend build-in TimeRE with some exact patterns

View File

@ -33,7 +33,6 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro
# #
ignoreregex = ^.+ john from host 192.168.1.1\s*$ ignoreregex = ^.+ john from host 192.168.1.1\s*$
[Init]
# "maxlines" is number of log lines to buffer for multi-line regex searches # "maxlines" is number of log lines to buffer for multi-line regex searches
maxlines = 1 maxlines = 1

View File

@ -1107,7 +1107,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
# (we don't use it in this test at all): # (we don't use it in this test at all):
elif unittest.F2B.fast and ( elif unittest.F2B.fast and (
len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex' len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
): ): # pragma: no cover
cmd[0] = "set" cmd[0] = "set"
cmd[3] = "DUMMY-REGEX <HOST>" cmd[3] = "DUMMY-REGEX <HOST>"
# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`: # command to server, use cmdHandler direct instead of `transm.proceed(cmd)`: