Merge pull request #368 from grooverdan/0.9_datedetmerge

MRG: general merge from master + date time zone
pull/413/head
Daniel Black 11 years ago
commit b8d9c07280

@ -66,7 +66,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
* action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across
all platforms to ensure permissions are the same before and after a ban -
closes gh-266. hostsdeny supports daemon_list now too.
* filter.d/roundcube-auth - timezone offset can be positive or negative
* filter.d/roundcube-auth - timezone offset can be positive or negative
* action.d/bsd-ipfw - action option unsed. Fixed to blocktype for
consistency. default to port unreach instead of deny
Rolf Fokkens
* action.d/dshield.conf and complain.conf -- reorder mailx arguments.
https://bugzilla.redhat.com/show_bug.cgi?id=998020
@ -78,6 +80,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
avoiding problems with getpid. Also $network and iptables moved
to Should- rc init fields
- New Features:
Andy Fragen and Daniel Black
* filter.d/osx-ipfw.conf - ipfw action for OSX based on random rule
numbers.
Daniel Black & ykimon
* filter.d/3proxy.conf -- filter added
Daniel Black
@ -95,6 +100,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests
and extra failure examples in sample logs
* filter.d/apache-auth - added expressions for mod_authz, mod_auth and
mod_auth_digest failures.
* Support %z (Timezone offset) and %f (sub-seconds) support for datedetector
Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий
* filter.d/exim.conf -- regex hardening and extra failure examples in
sample logs

@ -7,6 +7,7 @@ will be added
Adrien Clerc
ache
Andrey G. Grozin
Andy Fragen
Arturo 'Buanzo' Busleiman
Axel Thimm
Bill Heaton

@ -14,7 +14,7 @@
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD
#
actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num deny <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
actionstart = ipfw show | fgrep -q 'table(<table>)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
# Option: actionstop
@ -68,15 +68,16 @@ port =
# Values: STRING
startstatefile = /var/run/fail2ban/ipfw-started-table_<table>
# Option: action
# Notes: This is the action to take for automaticly created rules. See the
# ACTION defination at the top of man ipfw for allowed values.
# "deny" and "unreach port" are probably the useful.
# Values: STRING
action = deny
# Option: block
# Notes: This is how much to block.
# Can be "ip", "tcp", "udp" or various other options.
# Values: STRING
block = ip
# Option: blocktype
# Notes.: How to block the traffic. Use a action from man 5 ipfw
# Common values: deny, unreach port, reset
# ACTION defination at the top of man ipfw for allowed values.
# Values: STRING
#
blocktype = unreach port

@ -0,0 +1,87 @@
# Fail2Ban configuration file
#
# Author: Nick Munger
# Modified by: Andy Fragen and Daniel Black
#
# Mod for OS X, using random rulenum as OSX ipfw doesn't include tables
#
[Definition]
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD
#
actionstart =
# Option: actionstop
# Notes.: command executed once at the end of Fail2Ban
# Values: CMD
#
actionstop =
# Option: actioncheck
# Notes.: command executed once before each actionban command
# Values: CMD
#
actioncheck =
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.
# Tags: <ip> IP address
# Values: CMD
#
actionban = ipfw add <rulenum> set <setnum> <blocktype> log <block> from <ip> to <dst> <port>
# Option: actionunban
# Notes.: command executed when unbanning an IP. Take care that the
# command is executed with Fail2Ban user rights.
# Tags: <ip> IP address
# Values: CMD
#
actionunban = ipfw delete `ipfw -S list | grep -i 'set <setnum> <blocktype> log <block> from <ip> to <dst>' | awk '{print $1;}'`
[Init]
# Option: port
# Notes.: specifies port to block. Can be blank however may require block="ip"
# Values: [ NUM | STRING ]
#
port = ssh
# Option: dst
# Notes.: the local IP address of the network interface
# Values: IP, any, me or anything support by ipfw as a dst
#
dst = me
# Option: block
# Notes: This is how much to block.
# Can be "ip", "tcp", "udp" or various other options.
# Values: STRING
block = tcp
# Option: blocktype
# Notes.: How to block the traffic. Use a action from man 8 ipfw
# Common values: deny, unreach port, reset
# Values: STRING
#
blocktype = unreach port
# Option: set number
# Notes.: The ipset number this is added to.
# Values: 0-31
setnum = 10
# Option: number for ipfw rule
# Notes: This is meant to be automaticly generated and not overwritten
# Values: Random value between 10000 and 12000
rulenum="`echo $((RANDOM%%2000+10000))`"
# Duplicate prevention mechanism
#rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`"

@ -18,4 +18,4 @@ after = apache-common.local
# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]
# 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652]
# Reference: https://github.com/fail2ban/fail2ban/issues/268
_apache_error_client = \[[^]]*\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]
_apache_error_client = \[\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]

@ -22,10 +22,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)
__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)?
# note - (\.\d+)? is a really ugly catch of the microseconds not captured in
# in the date detector
#
failregex = ^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
^%(__line_prefix)s(\.\d+)?( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
failregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$
^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$

@ -17,7 +17,7 @@ before = common.conf
# (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
# Values: TEXT
#
failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. AUTHENTICATE .*)?\s*$
failregex = ^\s*(\[\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. AUTHENTICATE .*)?\s*$
# Option: ignoreregex
# Notes.: regex to ignore. If this regex matches, the line is ignored.

@ -531,3 +531,10 @@ action = iptables-allports[name=recidive]
bantime = 604800 ; 1 week
findtime = 86400 ; 1 day
maxretry = 5
[osx-ssh-ipfw]
enabled = false
filter = sshd
action = osx-ipfw
logpath = /var/log/secure.log

@ -53,31 +53,37 @@ class DateDetector:
def addDefaultTemplate(self):
self.__lock.acquire()
try:
# asctime
# asctime with subsecond: Sun Jan 23 21:59:59.011 2005
self.appendTemplate("%a %b %d %H:%M:%S.%f %Y")
# asctime: Sun Jan 23 21:59:59 2005
self.appendTemplate("%a %b %d %H:%M:%S %Y")
# asctime without year
# asctime without year: Sun Jan 23 21:59:59
self.appendTemplate("%a %b %d %H:%M:%S")
# standard
# standard: Jan 23 21:59:59
self.appendTemplate("%b %d %H:%M:%S")
# simple date
# simple date: 2005-01-23 21:59:59
self.appendTemplate("%Y-%m-%d %H:%M:%S")
# simple date: 2005/01/23 21:59:59
self.appendTemplate("%Y/%m/%d %H:%M:%S")
# simple date too (from x11vnc)
# simple date too (from x11vnc): 23/01/2005 21:59:59
self.appendTemplate("%d/%m/%Y %H:%M:%S")
# previous one but with year given by 2 digits
# previous one but with year given by 2 digits: 23/01/05 21:59:59
# (See http://bugs.debian.org/537610)
self.appendTemplate("%d/%m/%y %H:%M:%S")
# Apache format [31/Oct/2006:09:22:55 -0000]
self.appendTemplate("%d/%b/%Y:%H:%M:%S")
self.appendTemplate("%d/%b/%Y:%H:%M:%S %z")
# CPanel 05/20/2008:01:57:39
self.appendTemplate("%m/%d/%Y:%H:%M:%S")
# custom for syslog-ng 2006.12.21 06:43:20
self.appendTemplate("%Y.%m.%d %H:%M:%S")
# named 26-Jul-2007 15:20:52.252
self.appendTemplate("%d-%b-%Y %H:%M:%S")
self.appendTemplate("%d-%b-%Y %H:%M:%S.%f")
# roundcube 26-Jul-2007 15:20:52 +0200
self.appendTemplate("%d-%b-%Y %H:%M:%S %z")
# 17-07-2008 17:23:25
self.appendTemplate("%d-%m-%Y %H:%M:%S")
# 01-27-2012 16:22:44.252
self.appendTemplate("%m-%d-%Y %H:%M:%S")
self.appendTemplate("%m-%d-%Y %H:%M:%S.%f")
# TAI64N
template = DateTai64n()
template.setName("TAI64N")
@ -128,7 +134,7 @@ class DateDetector:
date = template.getDate(line)
if date is None:
continue
logSys.debug("Got time using template %s" % template.getName())
logSys.debug("Got time %i for \"%r\" using template %s" % (date[0], date[1].group(), template.getName()))
return date
except ValueError:
pass
@ -136,10 +142,6 @@ class DateDetector:
finally:
self.__lock.release()
def getUnixTime(self, line):
date = self.getTime(line)
return date and time.mktime(tuple(date))
##
# Sort the template lists using the hits score. This method is not called
# in this object and thus should be called from time to time.

@ -24,7 +24,10 @@ __author__ = "Cyril Jaquier"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import re, time
import re, time, calendar
from datetime import datetime
from datetime import timedelta
from mytime import MyTime
import iso8601
@ -48,6 +51,7 @@ class DateTemplate:
return self.__name
def setRegex(self, regex, wordBegin=True):
#logSys.debug(u"setRegex for %s is %r" % (self.__name, regex))
regex = regex.strip()
if (wordBegin and not re.search(r'^\^', regex)):
regex = r'\b' + regex
@ -82,12 +86,11 @@ class DateEpoch(DateTemplate):
self.setRegex("^\d{10}(\.\d{6})?")
def getDate(self, line):
date = None
dateMatch = self.matchDate(line)
if dateMatch:
# extract part of format which represents seconds since epoch
date = list(MyTime.localtime(float(dateMatch.group())))
return date
return (float(dateMatch.group()), dateMatch)
return None
##
@ -113,13 +116,16 @@ class DateStrptime(DateTemplate):
def __init__(self):
DateTemplate.__init__(self)
self.__pattern = ""
self._pattern = ""
self._unsupportedStrptimeBits = False
def setPattern(self, pattern):
self.__pattern = pattern.strip()
self._unsupported_f = not DateStrptime._f and re.search('%f', pattern)
self._unsupported_z = not DateStrptime._z and re.search('%z', pattern)
self._pattern = pattern
def getPattern(self):
return self.__pattern
return self._pattern
#@staticmethod
def convertLocale(date):
@ -133,17 +139,26 @@ class DateStrptime(DateTemplate):
convertLocale = staticmethod(convertLocale)
def getDate(self, line):
date = None
dateMatch = self.matchDate(line)
if dateMatch:
datePattern = self.getPattern()
if self._unsupported_f:
if dateMatch.group('_f'):
datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern)
logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern))
if self._unsupported_z:
if dateMatch.group('_z'):
datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern)
logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern))
try:
# Try first with 'C' locale
date = list(time.strptime(dateMatch.group(), self.getPattern()))
date = datetime.strptime(dateMatch.group(), datePattern)
except ValueError:
# Try to convert date string to 'C' locale
conv = self.convertLocale(dateMatch.group())
try:
date = list(time.strptime(conv, self.getPattern()))
date = datetime.strptime(conv, self.getPattern())
except (ValueError, re.error), e:
# Try to add the current year to the pattern. Should fix
# the "Feb 29" issue.
@ -152,7 +167,7 @@ class DateStrptime(DateTemplate):
if not '%Y' in opattern:
pattern = "%s %%Y" % opattern
conv += " %s" % MyTime.gmtime()[0]
date = list(time.strptime(conv, pattern))
date = datetime.strptime(conv, pattern)
else:
# we are helpless here
raise ValueError(
@ -160,44 +175,81 @@ class DateStrptime(DateTemplate):
"exception was %r and Feb 29 workaround could not "
"be tested due to already present year mark in the "
"pattern" % (opattern, e))
if date[0] < 2000:
if self._unsupported_z:
z = dateMatch.group('_z')
if z:
delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:]))
direction = z[0]
logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta))
# here we reverse the effect of the timezone and force it to UTC
if direction == '+':
date -= delta
else:
date += delta
date = date.replace(tzinfo=iso8601.Utc())
else:
logSys.warn("No _z group captured and %%z is not supported on current platform"
" - timezone ignored and assumed to be localtime. date: %s on line: %s"
% (date, line))
if date.year < 2000:
# There is probably no year field in the logs
# NOTE: Possibly makes week/year day incorrect
date[0] = MyTime.gmtime()[0]
date = date.replace(year=MyTime.gmtime()[0])
# Bug fix for #1241756
# If the date is greater than the current time, we suppose
# that the log is not from this year but from the year before
if time.mktime(tuple(date)) > MyTime.time():
if date > MyTime.now():
logSys.debug(
u"Correcting deduced year from %d to %d since %f > %f" %
(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
# NOTE: Possibly makes week/year day incorrect
date[0] -= 1
elif date[1] == 1 and date[2] == 1:
u"Correcting deduced year by one since %s > now (%s)" %
(date, MyTime.time()))
date = date.replace(year=date.year-1)
elif date.month == 1 and date.day == 1:
# If it is Jan 1st, it is either really Jan 1st or there
# is neither month nor day in the log.
# NOTE: Possibly makes week/year day incorrect
date[1] = MyTime.gmtime()[1]
date[2] = MyTime.gmtime()[2]
return date
date = date.replace(month=MyTime.gmtime()[1], day=1)
if date.tzinfo:
return ( calendar.timegm(date.utctimetuple()), dateMatch )
else:
return ( time.mktime(date.utctimetuple()), dateMatch )
return None
try:
time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f")
DateStrptime._f = True
except (ValueError, KeyError):
DateTemplate._f = False
try:
time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z")
DateStrptime._z = True
except ValueError:
DateStrptime._z = False
class DatePatternRegex(DateStrptime):
_reEscape = r"([\\.^$*+?\(\){}\[\]|])"
_patternRE = r"%(%|[aAbBdHIjmMpSUwWyY])"
_patternRE = r"%(%|[aAbBdfHIjmMpSUwWyYz])"
_patternName = {
'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day",
'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month",
'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek",
'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%"}
'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%",
'z': "Zone offset", 'f': "Microseconds" }
_patternRegex = {
'a': r"\w{3}", 'A': r"\w+", 'b': r"\w{3}", 'B': r"\w+",
'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)",
'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)",
'f': r"(?P<_f>\d{1,6})", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)",
'I': r"(?:1[0-2]|[ 0]?\d)",
'j': r"(?:36[0-6]3[0-5]\d|[1-2]\d\d|[ 0]?\d\d|[ 0]{0,2}\d)",
'm': r"(?:1[0-2]|[ 0]?[1-9])", 'M': r"[0-5]\d", 'p': r"[AP]M",
'S': r"(?:6[01]|[0-5]\d)", 'U': r"(?:5[0-3]|[1-4]\d|[ 0]?\d)",
'w': r"[0-6]", 'W': r"(?:5[0-3]|[ 0]?\d)", 'y': r"\d{2}",
'Y': r"\d{4}", '%': "%"}
'Y': r"\d{4}",
'z': r"(?P<_z>[+-]\d{4})", '%': "%"}
def __init__(self, pattern=None, **kwargs):
DateStrptime.__init__(self)
@ -205,7 +257,7 @@ class DatePatternRegex(DateStrptime):
self.setPattern(pattern, **kwargs)
def setPattern(self, pattern, anchor=False, **kwargs):
self.__pattern = pattern.strip()
DateStrptime.setPattern(self, pattern.strip())
name = re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName
DateStrptime.setName(self, name)
@ -218,15 +270,13 @@ class DatePatternRegex(DateStrptime):
regex = r"^" + regex
DateStrptime.setRegex(self, regex, **kwargs)
def getPattern(self):
return self.__pattern
def setRegex(self, line):
raise NotImplementedError("Regex derived from pattern")
def setName(self, line):
raise NotImplementedError("Name derived from pattern")
class DateTai64n(DateTemplate):
def __init__(self):
@ -236,32 +286,27 @@ class DateTai64n(DateTemplate):
self.setRegex("@[0-9a-f]{24}", wordBegin=False)
def getDate(self, line):
date = None
dateMatch = self.matchDate(line)
if dateMatch:
# extract part of format which represents seconds since epoch
value = dateMatch.group()
seconds_since_epoch = value[2:17]
# convert seconds from HEX into local time stamp
date = list(MyTime.localtime(int(seconds_since_epoch, 16)))
return date
return (int(seconds_since_epoch, 16), dateMatch)
return None
class DateISO8601(DateTemplate):
def __init__(self):
DateTemplate.__init__(self)
date_re = "[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}" \
".[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?" \
"(Z|(([-+])([0-9]{2}):([0-9]{2})))?"
self.setRegex(date_re)
self.setRegex(iso8601.ISO8601_REGEX_RAW)
def getDate(self, line):
date = None
dateMatch = self.matchDate(line)
if dateMatch:
# Parses the date.
value = dateMatch.group()
date = list(iso8601.parse_date(value, None).timetuple())
return date
return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch)
return None

@ -74,6 +74,7 @@ class Filter(JailThread):
self.__lineBuffer = []
## Store last time stamp, applicable for multi-line
self.__lastTimeLine = ""
self.__lastDate = None
self.dateDetector = DateDetector()
self.dateDetector.addDefaultTemplate()
@ -363,22 +364,7 @@ class Filter(JailThread):
line = line.rstrip('\r\n')
logSys.log(7, "Working on line %r", line)
timeMatch = self.dateDetector.matchTime(line)
if timeMatch:
# Lets split into time part and log part of the line
timeLine = timeMatch.group()
self.__lastTimeLine = timeLine
# Lets leave the beginning in as well, so if there is no
# anchore at the beginning of the time regexp, we don't
# at least allow injection. Should be harmless otherwise
logLine = line[:timeMatch.start()] + line[timeMatch.end():]
else:
timeLine = self.__lastTimeLine or line
logLine = line
self.__lineBuffer = ((self.__lineBuffer +
[logLine])[-self.__lineBufferSize:])
return self.findFailure(timeLine, "\n".join(self.__lineBuffer) + "\n",
returnRawHost, checkAllRegex)
return self.findFailure(line, returnRawHost, checkAllRegex)
def processLineAndAdd(self, line):
"""Processes the line for failures and populates failManager
@ -421,11 +407,38 @@ class Filter(JailThread):
# to find the logging time.
# @return a dict with IP and timestamp.
def findFailure(self, timeLine, logLine,
def findFailure(self, logLine,
returnRawHost=False, checkAllRegex=False):
logSys.log(5, "Date: %r, message: %r", timeLine, logLine)
failList = list()
date = self.dateDetector.getUnixTime(timeLine)
# Checks if we must ignore this line.
if self.ignoreLine(logLine) is not None:
# The ignoreregex matched. Return.
logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine)
return failList
dateTimeMatch = self.dateDetector.getTime(logLine)
if dateTimeMatch is not None:
# Lets split into time part and log part of the line
date = dateTimeMatch[0]
timeMatch = dateTimeMatch[1]
timeLine = timeMatch.group()
self.__lastTimeLine = timeLine
self.__lastDate = date
# Lets leave the beginning in as well, so if there is no
# anchore at the beginning of the time regexp, we don't
# at least allow injection. Should be harmless otherwise
logLine = logLine[:timeMatch.start()] + logLine[timeMatch.end():]
else:
timeLine = self.__lastTimeLine or logLine
date = self.__lastDate
self.__lineBuffer = (self.__lineBuffer + [logLine])[-self.__lineBufferSize:]
logLine = "\n".join(self.__lineBuffer) + "\n"
# Iterates over all the regular expressions.
for failRegexIndex, failRegex in enumerate(self.__failRegex):
failRegex.search(logLine)

@ -32,17 +32,17 @@ datetime.datetime(2007, 1, 25, 12, 0, tzinfo=<iso8601.iso8601.Utc ...>)
"""
from datetime import datetime, timedelta, tzinfo
from datetime import datetime, timedelta, tzinfo, time
import re
__all__ = ["parse_date", "ParseError"]
# Adapted from http://delete.me.uk/2005/03/iso8601.html
ISO8601_REGEX = re.compile(r"(?P<year>[0-9]{4})(-(?P<month>[0-9]{1,2})(-(?P<day>[0-9]{1,2})"
r"((?P<separator>.)(?P<hour>[0-9]{2}):(?P<minute>[0-9]{2})(:(?P<second>[0-9]{2})(\.(?P<fraction>[0-9]+))?)?"
r"(?P<timezone>Z|(([-+])([0-9]{2}):([0-9]{2})))?)?)?)?"
)
TIMEZONE_REGEX = re.compile("(?P<prefix>[+-])(?P<hours>[0-9]{2}).(?P<minutes>[0-9]{2})")
ISO8601_REGEX_RAW = "(?P<year>[0-9]{4})-(?P<month>[0-9]{1,2})-(?P<day>[0-9]{1,2})" \
"T(?P<hour>[0-9]{2}):(?P<minute>[0-9]{2})(:(?P<second>[0-9]{2})(\.(?P<fraction>[0-9]+))?)?" \
"(?P<timezone>Z|(([-+])([0-9]{2}):([0-9]{2})))?"
ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW)
TIMEZONE_REGEX = re.compile("(?P<prefix>[+-])(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})?")
class ParseError(Exception):
"""Raised when there is a problem parsing a date string"""
@ -67,8 +67,8 @@ class FixedOffset(tzinfo):
"""Fixed offset in hours and minutes from UTC
"""
def __init__(self, offset_hours, offset_minutes, name):
self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes)
def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0):
self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds)
self.__name = name
def utcoffset(self, dt):
@ -83,26 +83,30 @@ class FixedOffset(tzinfo):
def __repr__(self):
return "<FixedOffset %r>" % self.__name
def parse_timezone(tzstring, default_timezone=UTC):
def parse_timezone(tzstring):
"""Parses ISO 8601 time zone specs into tzinfo offsets
"""
if tzstring == "Z":
return default_timezone
# This isn't strictly correct, but it's common to encounter dates without
# timezones so I'll assume the default (which defaults to UTC).
# Addresses issue 4.
return UTC
if tzstring is None:
return default_timezone
zone_sec = -time.timezone
return FixedOffset(name=time.tzname[0],hours=(zone_sec / 3600),minutes=(zone_sec % 3600)/60,seconds=zone_sec % 60)
m = TIMEZONE_REGEX.match(tzstring)
prefix, hours, minutes = m.groups()
hours, minutes = int(hours), int(minutes)
if minutes is None:
minutes = 0
else:
minutes = int(minutes)
hours = int(hours)
if prefix == "-":
hours = -hours
minutes = -minutes
return FixedOffset(hours, minutes, tzstring)
return FixedOffset(tzstring, hours, minutes)
def parse_date(datestring, default_timezone=UTC):
def parse_date(datestring):
"""Parses ISO 8601 dates into datetime objects
The timezone is parsed from the date string. However it is quite common to
@ -116,7 +120,7 @@ def parse_date(datestring, default_timezone=UTC):
if not m:
raise ParseError("Unable to parse date string %r" % datestring)
groups = m.groupdict()
tz = parse_timezone(groups["timezone"], default_timezone=default_timezone)
tz = parse_timezone(groups["timezone"])
if groups["fraction"] is None:
groups["fraction"] = 0
else:

@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import time
import time, datetime
##
# MyTime class.
@ -74,6 +74,14 @@ class MyTime:
return time.gmtime(MyTime.myTime)
gmtime = staticmethod(gmtime)
#@staticmethod
def now():
if MyTime.myTime is None:
return datetime.datetime.now()
else:
return datetime.datetime.fromtimestamp(MyTime.myTime)
now = staticmethod(now)
def localtime(x=None):
if MyTime.myTime is None or x is not None:
return time.localtime(x)

@ -24,9 +24,10 @@ __author__ = "Cyril Jaquier"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import unittest, calendar, datetime, re, pprint
import unittest, calendar, time, datetime, re, pprint
from fail2ban.server.datedetector import DateDetector
from fail2ban.server.datetemplate import DateTemplate
from fail2ban.server.iso8601 import Utc
from fail2ban.tests.utils import setUpMyTime, tearDownMyTime
class DateDetectorTest(unittest.TestCase):
@ -43,11 +44,12 @@ class DateDetectorTest(unittest.TestCase):
def testGetEpochTime(self):
log = "1138049999 [sshd] error: PAM: Authentication failure"
date = [2006, 1, 23, 21, 59, 59, 0, 23, 0]
#date = [2006, 1, 23, 21, 59, 59, 0, 23, 0]
dateUnix = 1138049999.0
self.assertEqual(self.__datedetector.getTime(log), date)
self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
( datelog, matchlog ) = self.__datedetector.getTime(log)
self.assertEqual(datelog, dateUnix)
self.assertEqual(matchlog.group(), '1138049999')
def testGetTime(self):
log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure"
@ -57,8 +59,9 @@ class DateDetectorTest(unittest.TestCase):
# is not correctly determined atm, since year is not present
# in the log entry. Since this doesn't effect the operation
# of fail2ban -- we just ignore incorrect day of the week
self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6])
self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
( datelog, matchlog ) = self.__datedetector.getTime(log)
self.assertEqual(datelog, dateUnix)
self.assertEqual(matchlog.group(), 'Jan 23 21:59:59')
def testVariousTimes(self):
"""Test detection of various common date/time formats f2b should understand
@ -68,21 +71,23 @@ class DateDetectorTest(unittest.TestCase):
for sdate in (
"Jan 23 21:59:59",
"Sun Jan 23 21:59:59.011 2005",
"Sun Jan 23 21:59:59 2005",
"Sun Jan 23 21:59:59",
"2005/01/23 21:59:59",
"2005.01.23 21:59:59",
"23/01/2005 21:59:59",
"23/01/05 21:59:59",
"23/Jan/2005:21:59:59",
"23/Jan/2005:21:59:59 +0100",
"01/23/2005:21:59:59",
"2005-01-23 21:59:59",
"23-Jan-2005 21:59:59",
"23-Jan-2005 21:59:59.02",
"23-Jan-2005 21:59:59 +0100",
"23-01-2005 21:59:59",
"01-23-2005 21:59:59.252", # reported on f2b, causes Feb29 fix to break
"@4000000041f4104f00000000", # TAI64N
"2005-01-23T21:59:59.252Z", #ISO 8601
"2005-01-23T21:59:59-05:00Z", #ISO 8601 with TZ
"2005-01-23T20:59:59.252Z", #ISO 8601
"2005-01-23T15:59:59-05:00", #ISO 8601 with TZ
"<01/23/05@21:59:59>",
"050123 21:59:59", # MySQL
"Jan 23, 2005 9:59:59 PM", # Apache Tomcat
@ -94,8 +99,9 @@ class DateDetectorTest(unittest.TestCase):
# yoh: on [:6] see in above test
logtime = self.__datedetector.getTime(log)
self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s" % sdate)
self.assertEqual(logtime[:6], date[:6], "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6]))
self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix, "getUnixTime failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6]))
( logUnix, logMatch ) = logtime
self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix))
self.assertEqual(logMatch.group(), sdate)
def testStableSortTemplate(self):
old_names = [x.getName() for x in self.__datedetector.getTemplates()]
@ -112,21 +118,23 @@ class DateDetectorTest(unittest.TestCase):
# see https://github.com/fail2ban/fail2ban/pull/130
# yoh: unfortunately this test is not really effective to reproduce the
# situation but left in place to assure consistent behavior
m1 = [2012, 10, 11, 2, 37, 17]
self.assertEqual(
self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
m1)
mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).utctimetuple())
logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
self.assertNotEqual(logdate, None)
( logTime, logMatch ) = logdate
self.assertEqual(logTime, mu)
self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
self.__datedetector.sortTemplate()
# confuse it with year being at the end
for i in xrange(10):
self.assertEqual(
self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')[:6],
m1)
( logTime, logMatch ) = self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')
self.assertEqual(logTime, mu)
self.assertEqual(logMatch.group(), '11/10/2012 02:37:17')
self.__datedetector.sortTemplate()
# and now back to the original
self.assertEqual(
self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
m1)
( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')
self.assertEqual(logTime, mu)
self.assertEqual(logMatch.group(), '2012/10/11 02:37:17')
def testDateDetectorTemplateOverlap(self):
patterns = [template.getPattern()
@ -141,12 +149,13 @@ class DateDetectorTest(unittest.TestCase):
for minute in xrange(0, 60, 15):
for second in xrange(0, 60, 15): # Far enough?
yield datetime.datetime(
year, month, day, hour, minute, second)
year, month, day, hour, minute, second, 300, Utc())
overlapedTemplates = set()
for date in iterDates(year):
for pattern in patterns:
datestr = date.strftime(pattern)
datestr = re.sub(r'%f','300', datestr) # for python 2.5 where there is no %f
datestrs = set([
datestr,
re.sub(r"(\s)0", r"\1 ", datestr),
@ -160,12 +169,12 @@ class DateDetectorTest(unittest.TestCase):
matchedTemplates = [template
for template in self.__datedetector.getTemplates()
if template.getHits() > 0]
assert matchedTemplates != [] # Should match at least one
self.assertNotEqual(matchedTemplates, [], "Date %r should match at least one template" % pattern)
if len(matchedTemplates) > 1:
overlapedTemplates.add((pattern, tuple(sorted(template.getName()
for template in matchedTemplates))))
if overlapedTemplates:
print "WARNING: The following date templates overlap:"
print("WARNING: The following date templates overlap:")
pprint.pprint(overlapedTemplates)
# def testDefaultTempate(self):

@ -1,12 +1,12 @@
# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" }
# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "80.187.101.33" }
@400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=<martin@waschbuesch.de>, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS
# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" }
# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.224" }
@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224
# Above example with injected rhost into ruser -- should not match for 1.2.3.4
# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" }
# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "192.0.43.10" }
@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10
# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" }
# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.225" }
@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root
# failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" }

@ -1,2 +1,2 @@
# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" }
# failJSON: { "time": "2009-03-26T14:44:20", "match": true , "host": "66.185.212.172" }
66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"

@ -1,4 +1,4 @@
# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" }
# failJSON: { "time": "2013-01-22T21:28:21", "match": true , "host": "192.0.43.10" }
[22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10
# failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" }
May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47

@ -39,6 +39,7 @@ from fail2ban.server.filterpoll import FilterPoll
from fail2ban.server.filter import FileFilter, DNSUtils
from fail2ban.server.failmanager import FailManager
from fail2ban.server.failmanager import FailManagerEmpty
from fail2ban.server.mytime import MyTime
from fail2ban.tests.utils import setUpMyTime, tearDownMyTime
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
@ -89,8 +90,8 @@ def _assert_equal_entries(utest, found, output, count=None):
utest.assertEqual(found[0], output[0]) # IP
utest.assertEqual(found[1], count or output[1]) # count
found_time, output_time = \
time.localtime(found[2]),\
time.localtime(output[2])
MyTime.localtime(found[2]),\
MyTime.localtime(output[2])
utest.assertEqual(found_time, output_time)
if len(output) > 3 and count is None: # match matches
# do not check if custom count (e.g. going through them twice)
@ -737,7 +738,7 @@ class GetFailures(unittest.TestCase):
FILENAME_MULTILINE = os.path.join(TEST_FILES_DIR, "testcase-multiline.log")
# so that they could be reused by other tests
FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
FAILURES_01 = ('193.168.0.128', 3, 1124017199.0,
[u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
def setUp(self):
@ -783,7 +784,7 @@ class GetFailures(unittest.TestCase):
def testGetFailures02(self):
output = ('141.3.81.106', 4, 1124013539.0,
output = ('141.3.81.106', 4, 1124017139.0,
[u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
% m for m in 53, 54, 57, 58])
@ -793,7 +794,7 @@ class GetFailures(unittest.TestCase):
_assert_correct_last_attempt(self, self.filter, output)
def testGetFailures03(self):
output = ('203.162.223.135', 6, 1124013544.0)
output = ('203.162.223.135', 7, 1124017144.0)
self.filter.addLogPath(GetFailures.FILENAME_03)
self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown")
@ -801,8 +802,8 @@ class GetFailures(unittest.TestCase):
_assert_correct_last_attempt(self, self.filter, output)
def testGetFailures04(self):
output = [('212.41.96.186', 4, 1124013600.0),
('212.41.96.185', 4, 1124013598.0)]
output = [('212.41.96.186', 4, 1124017200.0),
('212.41.96.185', 4, 1124017198.0)]
self.filter.addLogPath(GetFailures.FILENAME_04)
self.filter.addFailRegex("Invalid user .* <HOST>")
@ -816,11 +817,11 @@ class GetFailures(unittest.TestCase):
def testGetFailuresUseDNS(self):
# We should still catch failures with usedns = no ;-)
output_yes = ('93.184.216.119', 2, 1124013539.0,
output_yes = ('93.184.216.119', 2, 1124017139.0,
[u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
output_no = ('93.184.216.119', 1, 1124013539.0,
output_no = ('93.184.216.119', 1, 1124017139.0,
[u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n'])
# Actually no exception would be raised -- it will be just set to 'no'
@ -842,7 +843,7 @@ class GetFailures(unittest.TestCase):
def testGetFailuresMultiRegex(self):
output = ('141.3.81.106', 8, 1124013541.0)
output = ('141.3.81.106', 8, 1124017141.0)
self.filter.addLogPath(GetFailures.FILENAME_02)
self.filter.addFailRegex("Failed .* from <HOST>")
@ -851,7 +852,7 @@ class GetFailures(unittest.TestCase):
_assert_correct_last_attempt(self, self.filter, output)
def testGetFailuresIgnoreRegex(self):
output = ('141.3.81.106', 8, 1124013541.0)
output = ('141.3.81.106', 8, 1124017141.0)
self.filter.addLogPath(GetFailures.FILENAME_02)
self.filter.addFailRegex("Failed .* from <HOST>")
@ -863,8 +864,8 @@ class GetFailures(unittest.TestCase):
self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
def testGetFailuresMultiLine(self):
output = [("192.0.43.10", 2, 1124013599.0),
("192.0.43.11", 1, 1124013598.0)]
output = [("192.0.43.10", 2, 1124017199.0),
("192.0.43.11", 1, 1124017198.0)]
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
self.filter.setMaxLines(100)
@ -882,7 +883,7 @@ class GetFailures(unittest.TestCase):
self.assertEqual(sorted(foundList), sorted(output))
def testGetFailuresMultiLineIgnoreRegex(self):
output = [("192.0.43.10", 2, 1124013599.0)]
output = [("192.0.43.10", 2, 1124017199.0)]
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
self.filter.addIgnoreRegex("rsync error: Received SIGINT")
@ -896,9 +897,9 @@ class GetFailures(unittest.TestCase):
self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
def testGetFailuresMultiLineMultiRegex(self):
output = [("192.0.43.10", 2, 1124013599.0),
("192.0.43.11", 1, 1124013598.0),
("192.0.43.15", 1, 1124013598.0)]
output = [("192.0.43.10", 2, 1124017199.0),
("192.0.43.11", 1, 1124017198.0),
("192.0.43.15", 1, 1124017198.0)]
self.filter.addLogPath(GetFailures.FILENAME_MULTILINE)
self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$")
self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$")

@ -22,7 +22,7 @@
__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
__license__ = "GPL"
import unittest, sys, os, fileinput, re, datetime, inspect
import unittest, sys, os, fileinput, re, time, datetime, inspect
if sys.version_info >= (2, 6):
import json
@ -119,15 +119,19 @@ def testSampleRegexsFactory(name):
(map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno()))
# Verify timestamp and host as expected
failregex, host, time = ret[0]
failregex, host, fail2banTime = ret[0]
self.assertEqual(host, faildata.get("host", None))
fail2banTime = datetime.datetime.fromtimestamp(time)
jsonTime = datetime.datetime.strptime(
faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")
t = faildata.get("time", None)
jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S")
jsonTime = time.mktime(jsonTimeLocal.utctimetuple())
self.assertEqual(fail2banTime, jsonTime,
"Time mismatch %s != %s on: %s:%i %r:" %
(fail2banTime, jsonTime, logFile.filename(), logFile.filelineno(), line ) )
"UTC Time mismatch fail2ban %s (%s) != failJson %s (%s) (diff %i seconds) on: %s:%i %r:" %
(fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)),
jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)),
fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) )
regexsUsed.add(failregex)

Loading…
Cancel
Save