added possibility to specify more precise default date pattern:

- `datepattern = {^LN-BEG}` - only line-begin anchored default patterns
     (matches date only at begin of line, or with max distance up to 2 non-alphanumeric characters from line-begin);
  - `datepattern = {*WD-BEG}` - only word-begin anchored default patterns;
  - `datepattern = ^prefix{DATE}suffix` - exact specified default patterns (using prefix and suffix);
common filter configs gets a more precise, line-begin anchored (datepattern = {^LN-BEG}) resp. custom anchoring default date-patterns;
pull/1583/head
sebres 2016-10-05 19:34:21 +02:00
parent f56ff5f48b
commit ab0ac2111c
29 changed files with 120 additions and 57 deletions

View File

@ -9,6 +9,8 @@ failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ <HOST>:\d+ [\d.]+:\d+ \d+ \d+ \d+\s
ignoreregex =
datepattern = {^LN-BEG}
# DEV Notes:
# http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are
# all authentication problems (%E field)

View File

@ -10,6 +10,8 @@ after = apache-common.local
_apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
datepattern = {^LN-BEG}
# Common prefix for [error] apache messages which also would include <HOST>
# Depending on the version it could be
# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]

View File

@ -3,10 +3,6 @@
#
# The knocking request must have a referer.
[INCLUDES]
before = apache-common.conf
[Definition]
failregex = ^<HOST> - \w+ \[\] "GET <knocking_url> HTTP/1\.[01]" 200 \d+ ".*" "[^-].*"$

View File

@ -20,6 +20,8 @@ failregex = ^(:? \[SSL-out\])? <HOST> max sender authentication errors \(\d{,3}\
ignoreregex =
datepattern = {^LN-BEG}
# DEV Notes:
# V1 Examples matches:
# Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41);

View File

@ -31,6 +31,7 @@ failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed fo
ignoreregex =
datepattern = {^LN-BEG}
# Author: Xavier Devlamynck / Daniel Black
#

View File

@ -61,4 +61,7 @@ __prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostn
# pam_ldap
__pam_auth = pam_unix
# standardly all formats using prefix have line-begin anchored date:
datepattern = {^LN-BEG}
# Author: Yaroslav Halchenko

View File

@ -8,8 +8,6 @@ failregex = ^: Bad Rcon: "rcon \d+ "\S+" sv_contact ".*?"" from "<HOST>:\d+"$
ignoreregex =
[Init]
datepattern = ^L %%d/%%m/%%Y - %%H:%%M:%%S

View File

@ -15,5 +15,7 @@ failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$
ignoreregex =
datepattern = {^LN-BEG}
# Author: Christoph Haas
# Modified by: Cyril Jaquier

View File

@ -13,7 +13,6 @@ failregex = ^: \'<HOST>\' \d{1,3} failed login attempt(s)?. \s*
ignoreregex =
[Init]
datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S
#

View File

@ -26,6 +26,8 @@ failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challe
ignoreregex =
datepattern = {^LN-BEG}
# Author: Rupa SChomaker, soapee01, Daniel Black
# https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban
# Thanks to Jim on mailing list of samples and guidance

View File

@ -9,8 +9,6 @@ failregex = ^ SMTP Spam attack detected from <HOST>,
ignoreregex =
[Init]
datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
# DEV NOTES:

View File

@ -13,7 +13,7 @@ before = common.conf
_daemon = monit
# Regexp for previous (accessing monit httpd) and new (access denied) versions
failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$
^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$
# Ignore login with empty user (first connect, no user specified)

View File

@ -15,13 +15,14 @@ _daemon = murmurd
# variable in your server config file (murmur.ini / mumble-server.ini).
_usernameregex = [^>]+
_prefix = <W>[\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
_prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
failregex = ^%(_prefix)s Invalid server password$
^%(_prefix)s Wrong certificate or password for existing user$
ignoreregex =
datepattern = ^<W>{DATE}
# DEV Notes:
#

View File

@ -8,6 +8,8 @@ failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not f
ignoreregex =
datepattern = {^LN-BEG}
# DEV NOTES:
# Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files
# Extensive search of all nginx auth failures not done yet.

View File

@ -43,3 +43,4 @@ failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by z
ignoreregex =
datepattern = {^LN-BEG}

View File

@ -9,7 +9,6 @@
[Definition]
failregex = ^<HOST>\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$
[Init]
datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z

View File

@ -52,10 +52,12 @@ before = common.conf
# Note that you MUST have LOG_FORMAT=4 for this to work!
#
failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
failregex = tr="[A-Z]+\|[0-9.]+\|\d+\|<HOST>\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$
# Option: ignoreregex
# Notes.: regex to ignore. If this regex matches, the line is ignored.
# Values: TEXT
#
ignoreregex =
datepattern = ^<co ts="{DATE}"\s+

View File

@ -18,4 +18,6 @@ failregex = ^type=%(_type)s msg=audit\(:\d+\): (user )?pid=\d+ uid=%(_uid)s auid
ignoreregex =
datepattern = EPOCH
# Author: Daniel Black

View File

@ -6,7 +6,9 @@
failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$
ignoreregex =
ignoreregex = "^<ADDR>"
datepattern = {^LN-BEG}
#
# DEV Notes:

View File

@ -5,8 +5,6 @@ failregex = ^ \[LOGIN_ERROR\].*from <HOST>: Unknown user or password incorrect\.
ignoreregex =
[Init]
datepattern = ^%%m/%%d/%%Y %%H:%%M:%%S
# DEV NOTES:

View File

@ -38,13 +38,13 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro
ignoreregex =
[Init]
# "maxlines" is number of log lines to buffer for multi-line regex searches
maxlines = 10
journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
datepattern = {^LN-BEG}
# DEV Notes:
#
# "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because

View File

@ -474,8 +474,9 @@ class Fail2banRegex(object):
if self._verbose or template.hits:
out.append("[%d] %s" % (template.hits, template.name))
if self._verbose_date:
out.append(" # weight: %3s, pattern: %s" % (
template.weight, getattr(template, 'pattern', ''),))
out.append(" # weight: %.3f (%.3f), pattern: %s" % (
template.weight, template.template.weight,
getattr(template, 'pattern', ''),))
out.append(" # regex: %s" % (getattr(template, 'regex', ''),))
pprint_list(out, "[# of hits] date format")

View File

@ -40,6 +40,9 @@ class FilterReader(DefinitionInitConfigReader):
_configOpts = {
"ignoreregex": ["string", None],
"failregex": ["string", ""],
"maxlines": ["int", None],
"datepattern": ["string", None],
"journalmatch": ["string", None],
}
def setFile(self, fileName):
@ -74,16 +77,16 @@ class FilterReader(DefinitionInitConfigReader):
stream.append(["multi-set", self._jailName, "add" + opt, multi])
elif len(multi):
stream.append(["set", self._jailName, "add" + opt, multi[0]])
if self._initOpts:
if 'maxlines' in self._initOpts:
elif opt == 'maxlines':
# We warn when multiline regex is used without maxlines > 1
# therefore keep sure we set this option first.
stream.insert(0, ["set", self._jailName, "maxlines", self._initOpts["maxlines"]])
if 'datepattern' in self._initOpts:
stream.append(["set", self._jailName, "datepattern", self._initOpts["datepattern"]])
stream.insert(0, ["set", self._jailName, "maxlines", value])
elif opt == 'datepattern':
stream.append(["set", self._jailName, "datepattern", value])
# Do not send a command if the match is empty.
if self._initOpts.get("journalmatch", '') != '':
for match in self._initOpts["journalmatch"].split("\n"):
elif opt == 'journalmatch':
for match in value.split("\n"):
if match == '': continue
stream.append(
["set", self._jailName, "addjournalmatch"] +
shlex.split(match))

View File

@ -26,7 +26,7 @@ import time
from threading import Lock
from .datetemplate import DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
from ..helpers import getLogger
# Gets the instance of the logger.
@ -34,6 +34,8 @@ logSys = getLogger(__name__)
logLevel = 6
RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE)
class DateDetectorCache(object):
"""Implements the caching of the default templates list.
@ -54,15 +56,18 @@ class DateDetectorCache(object):
self._addDefaultTemplate()
return self.__templates
def _cacheTemplate(self, template):
def _cacheTemplate(self, template, lineBeginOnly=False):
"""Cache Fail2Ban's default template.
"""
if isinstance(template, str):
# exact given template with word benin-end boundary:
template = DatePatternRegex(template)
if not lineBeginOnly:
template = DatePatternRegex(template)
else:
template = DatePatternRegex(template, wordBegin='start')
# additional template, that prefers datetime at start of a line (safety+performance feature):
if hasattr(template, 'regex'):
if not lineBeginOnly and hasattr(template, 'regex'):
template2 = copy.copy(template)
regex = getattr(template, 'pattern', template.regex)
template2.setRegex(regex, wordBegin='start', wordEnd=True)
@ -90,6 +95,7 @@ class DateDetectorCache(object):
# prefixed with optional time zone (monit):
# PDT Apr 16 21:05:29
self._cacheTemplate("(?:%z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?")
# asctime with optional day, subsecond and/or year coming after day
# http://bugs.debian.org/798923
# Sun Jan 23 2005 21:59:59.011
@ -114,17 +120,18 @@ class DateDetectorCache(object):
# TAI64N
self._cacheTemplate(DateTai64n())
# Epoch
self._cacheTemplate(DateEpoch(lineBeginOnly=True), lineBeginOnly=True)
self._cacheTemplate(DateEpoch())
# Only time information in the log
self._cacheTemplate("^%H:%M:%S")
self._cacheTemplate("%H:%M:%S", lineBeginOnly=True)
# <09/16/08@05:03:30>
self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>")
self._cacheTemplate("<%m/%d/%Exy@%H:%M:%S>", lineBeginOnly=True)
# MySQL: 130322 11:46:11
self._cacheTemplate("%Exy%Exm%Exd ?%H:%M:%S")
# Apache Tomcat
self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
# ASSP: Apr-27-13 02:33:06
self._cacheTemplate("^%b-%d-%Exy %H:%M:%S")
self._cacheTemplate("%b-%d-%Exy %H:%M:%S", lineBeginOnly=True)
self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
del self.__tmpcache
@ -172,6 +179,8 @@ class DateDetector(object):
self.__lastTemplIdx = 0x7fffffff
# first free place:
self.__firstUnused = 0
# pre-match pattern:
self.__preMatch = None
def _appendTemplate(self, template):
name = template.name
@ -200,10 +209,18 @@ class DateDetector(object):
template = DatePatternRegex(template)
self._appendTemplate(template)
def addDefaultTemplate(self):
def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
"""Add Fail2Ban's default set of date templates.
"""
for template in DateDetector._defCache.templates:
# filter if specified:
if filterTemplate is not None and not filterTemplate(template): continue
# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
if preMatch is not None:
regex = getattr(template, 'pattern', template.regex)
template = copy.copy(template)
template.setRegex(RE_DATE_PREMATCH.sub(regex, preMatch))
# append date detector template:
self._appendTemplate(template)
@property
@ -230,7 +247,7 @@ class DateDetector(object):
The regex match returned from the first successfully matched
template.
"""
#logSys.log(logLevel, "try to match time for line: %.250s", line)
#logSys.log(logLevel, "try to match time for line: %.120s", line)
match = None
# first try to use last template with same start/end position:
i = self.__lastTemplIdx
@ -238,13 +255,16 @@ class DateDetector(object):
ddtempl = self.__templates[i]
template = ddtempl.template
distance, endpos = self.__lastPos[0], self.__lastEndPos[0]
if logSys.getEffectiveLevel() <= logLevel-1:
logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...",
i, distance, endpos,
line[distance-1:distance], self.__lastPos[1],
line[distance:endpos],
line[endpos:endpos+1], self.__lastEndPos[1])
# check same boundaries left/right, otherwise possible collision/pattern switch:
if (line[distance-1:distance] == self.__lastPos[1] and
line[endpos:endpos+1] == self.__lastEndPos[1]
):
if logSys.getEffectiveLevel() <= logLevel-1:
logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ... %s ...",
i, distance, endpos, line[distance:endpos])
match = template.matchDate(line, distance, endpos)
if match:
distance = match.start()
@ -270,8 +290,8 @@ class DateDetector(object):
if logSys.getEffectiveLevel() <= logLevel:
logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s",
i, distance, ddtempl.distance, self.__lastPos[0], template.name)
## if line-begin anchored - stop searching:
if template.flags & DateTemplate.LINE_BEGIN:
## if line-begin/end anchored - stop searching:
if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END):
break
## [grave] if distance changed, possible date-match was found somewhere
## in body of message, so save this template, and search further:

View File

@ -33,9 +33,12 @@ from ..helpers import getLogger
logSys = getLogger(__name__)
RE_NO_WRD_BOUND_BEG = re.compile(r'^(?:\^|\*\*|\(\?:\^)')
RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$|\*\*)$')
RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*)$')
RE_DEL_WRD_BOUNDS = re.compile(r'^\*\*|(?<!\\)\*\*$')
RE_LINE_BOUND_BEG = re.compile(r'^(?:\^|\(\?:\^(?!\|))')
RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
class DateTemplate(object):
"""A template which searches for and returns a date from a log line.
@ -49,6 +52,7 @@ class DateTemplate(object):
"""
LINE_BEGIN = 8
LINE_END = 4
WORD_BEGIN = 2
WORD_END = 1
@ -86,6 +90,7 @@ class DateTemplate(object):
If regular expression fails to compile
"""
regex = regex.strip()
self.flags = 0
# if word or line start boundary:
if wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex):
self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN
@ -96,9 +101,12 @@ class DateTemplate(object):
self.flags |= DateTemplate.WORD_END
regex += r'(?=\b|\W|$)'
self.name += '{*WD-END}'
if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN
if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END
# remove possible special pattern "**" in front and end of regex:
regex = RE_DEL_WRD_BOUNDS.sub('', regex)
self._regex = regex
self._cRegex = None
regex = property(getRegex, setRegex, doc=
"""Regex used to search for date.
@ -151,11 +159,15 @@ class DateEpoch(DateTemplate):
regex
"""
def __init__(self):
def __init__(self, lineBeginOnly=False):
DateTemplate.__init__(self)
self.name = "Epoch"
self.setRegex(r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))",
wordBegin=False) ;# already line begin resp. word begin anchored
if not lineBeginOnly:
regex = r"(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
else:
regex = r"(?P<square>(?<=^\[))\d{10,11}\b(?:\.\d{3,6})?(?(square)(?=\]))"
self.setRegex(regex, wordBegin='start', wordEnd=True)
def getDate(self, line, dateMatch=None):
"""Method to return the date for a log line.

View File

@ -35,7 +35,7 @@ from .ipdns import DNSUtils, IPAddr
from .ticket import FailTicket
from .jailthread import JailThread
from .datedetector import DateDetector
from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n
from .datetemplate import DateTemplate, DatePatternRegex, DateEpoch, DateTai64n
from .mytime import MyTime
from .failregex import FailRegex, Regex, RegexException
from .action import CommandAction
@ -257,14 +257,29 @@ class Filter(JailThread):
if pattern is None:
self.dateDetector = None
return
elif pattern.upper() == "EPOCH":
template = DateEpoch()
template.name = "Epoch"
elif pattern.upper() == "TAI64N":
template = DateTai64n()
template.name = "TAI64N"
else:
template = DatePatternRegex(pattern)
key = pattern.upper()
if key == "EPOCH":
template = DateEpoch()
template.name = "Epoch"
elif key == "TAI64N":
template = DateTai64n()
template.name = "TAI64N"
elif key in ("{^LN-BEG}", "{*WD-BEG}", "{DEFAULT}"):
self.dateDetector = DateDetector()
flt = \
lambda template: template.flags & DateTemplate.LINE_BEGIN if key == "{^LN-BEG}" else \
lambda template: template.flags & DateTemplate.WORD_BEGIN if key == "{*WD-BEG}" else \
None
self.dateDetector.addDefaultTemplate(flt)
return
elif "{DATE}" in key:
self.dateDetector = DateDetector()
self.dateDetector.addDefaultTemplate(
lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
return
else:
template = DatePatternRegex(pattern)
self.dateDetector = DateDetector()
self.dateDetector.appendTemplate(template)
logSys.info(" date pattern `%r`: `%s`",
@ -280,9 +295,9 @@ class Filter(JailThread):
def getDatePattern(self):
if self.dateDetector is not None:
templates = self.dateDetector.templates
if len(templates) > 1:
if len(templates) > 2:
return None, "Default Detectors"
elif len(templates) == 1:
elif len(templates):
if hasattr(templates[0], "pattern"):
pattern = templates[0].pattern
else:

View File

@ -40,6 +40,7 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
#timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
timeRE['z'] = r"(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?)"
# Extend build-in TimeRE with some exact patterns

View File

@ -33,7 +33,6 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* fro
#
ignoreregex = ^.+ john from host 192.168.1.1\s*$
[Init]
# "maxlines" is number of log lines to buffer for multi-line regex searches
maxlines = 1

View File

@ -1107,7 +1107,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
# (we don't use it in this test at all):
elif unittest.F2B.fast and (
len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
):
): # pragma: no cover
cmd[0] = "set"
cmd[3] = "DUMMY-REGEX <HOST>"
# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`: