From f2ddb3e3d099b212357f5b6cb0d2f373aae99e98 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 26 Jan 2014 22:03:55 +0000 Subject: [PATCH] RF: Refactor date detector and date template elements Changes include to use Python class properties, merge some date patterns, and change ISO8601 date template to DatePatternRegex class. --- fail2ban/server/datedetector.py | 145 ++++++---- fail2ban/server/datetemplate.py | 375 +++++++++++-------------- fail2ban/server/filter.py | 24 +- fail2ban/server/iso8601.py | 136 --------- fail2ban/server/strptime.py | 31 +- fail2ban/tests/datedetectortestcase.py | 12 +- fail2ban/tests/misctestcase.py | 50 ++-- fail2ban/tests/servertestcase.py | 2 - 8 files changed, 332 insertions(+), 443 deletions(-) delete mode 100644 fail2ban/server/iso8601.py diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index e5c57a31..c8e631ab 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -24,82 +24,94 @@ __license__ = "GPL" import sys, time, logging from threading import Lock -from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601 +from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch # Gets the instance of the logger. logSys = logging.getLogger(__name__) -class DateDetector: - +class DateDetector(object): + """Manages one or more date templates to find a date within a log line. + """ + def __init__(self): + """Initialise the date detector. + """ self.__lock = Lock() self.__templates = list() self.__known_names = set() def _appendTemplate(self, template): - name = template.getName() + name = template.name if name in self.__known_names: - raise ValueError("There is already a template with name %s" % name) + raise ValueError( + "There is already a template with name %s" % name) self.__known_names.add(name) self.__templates.append(template) - + def appendTemplate(self, template): + """Add a date template to manage and use in search of dates. + + Parameters + ---------- + template : DateTemplate or str + Can be either a `DateTemplate` instance, or a string which will + be used as the pattern for the `DatePatternRegex` template. The + template will then be added to the detector. + + Raises + ------ + ValueError + If a template already exists with the same name. + """ if isinstance(template, str): template = DatePatternRegex(template) - DateDetector._appendTemplate(self, template) + self._appendTemplate(template) def addDefaultTemplate(self): + """Add Fail2Ban's default set of date templates. + """ self.__lock.acquire() try: - # asctime with subsecond: Sun Jan 23 21:59:59.011 2005 - self.appendTemplate("%a %b %d %H:%M:%S\.%f %Y") - # asctime: Sun Jan 23 21:59:59 2005 - self.appendTemplate("%a %b %d %H:%M:%S %Y") - # asctime without year: Sun Jan 23 21:59:59 - self.appendTemplate("%a %b %d %H:%M:%S") - # standard: Jan 23 21:59:59 - self.appendTemplate("%b %d %H:%M:%S") - # proftpd date: 2005-01-23 21:59:59,333 - self.appendTemplate("%Y-%m-%d %H:%M:%S,%f") - # simple date: 2005-01-23 21:59:59 - self.appendTemplate("%Y-%m-%d %H:%M:%S") + # asctime with optional day, subsecond and/or year: + # Sun Jan 23 21:59:59.011 2005 + self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # simple date, optional subsecond (proftpd): + # 2005-01-23 21:59:59 + self.appendTemplate("%Y-%m-%d %H:%M:%S(?:,%f)?") # simple date: 2005/01/23 21:59:59 self.appendTemplate("%Y/%m/%d %H:%M:%S") # simple date too (from x11vnc): 23/01/2005 21:59:59 - self.appendTemplate("%d/%m/%Y %H:%M:%S") - # previous one but with year given by 2 digits: 23/01/05 21:59:59 + # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) - self.appendTemplate("%d/%m/%y %H:%M:%S") - # Apache format [31/Oct/2006:09:22:55 -0000] - self.appendTemplate("%d/%b/%Y:%H:%M:%S %z") - # [31/Oct/2006:09:22:55] - self.appendTemplate("%d/%b/%Y:%H:%M:%S") + self.appendTemplate("%d/%m/(?:%Y|%y) %H:%M:%S") + # Apache format optional time zone: + # [31/Oct/2006:09:22:55 -0000] + self.appendTemplate("%d/%b/%Y:%H:%M:%S(?: %z)?") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") # custom for syslog-ng 2006.12.21 06:43:20 self.appendTemplate("%Y\.%m\.%d %H:%M:%S") # named 26-Jul-2007 15:20:52.252 - self.appendTemplate("%d-%b-%Y %H:%M:%S\.%f") # roundcube 26-Jul-2007 15:20:52 +0200 - self.appendTemplate("%d-%b-%Y %H:%M:%S %z") + self.appendTemplate("%d-%b-%Y %H:%M:%S(?:\.%f)?(?: %z)?") # 26-Jul-2007 15:20:52 self.appendTemplate("%d-%b-%Y %H:%M:%S") # 17-07-2008 17:23:25 self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 + # subseconds explicit to avoid possible %m<->%d confusion + # with previous self.appendTemplate("%m-%d-%Y %H:%M:%S\.%f") # TAI64N template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" self.appendTemplate(template) # Epoch template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" self.appendTemplate(template) # ISO 8601 - template = DateISO8601() - template.setName("ISO 8601") - self.appendTemplate(template) + self.appendTemplate("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?(?:%z)?") # Only time information in the log self.appendTemplate("^%H:%M:%S") # <09/16/08@05:03:30> @@ -112,25 +124,60 @@ class DateDetector: self.appendTemplate("^%b-%d-%y %H:%M:%S") finally: self.__lock.release() - - def getTemplates(self): + + @property + def templates(self): + """List of template instances managed by the detector. + """ return self.__templates - - def matchTime(self, line, incHits=True): + + def matchTime(self, line): + """Attempts to find date on a log line using templates. + + This uses the templates' `matchDate` method in an attempt to find + a date. It also increments the match hit count for the winning + template. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + re.MatchObject + The regex match returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: match = template.matchDate(line) if not match is None: - logSys.debug("Matched time template %s" % template.getName()) - if incHits: - template.incHits() + logSys.debug("Matched time template %s" % template.name) + template.hits += 1 return match return None finally: self.__lock.release() def getTime(self, line): + """Attempts to return the date on a log line using templates. + + This uses the templates' `getDate` method in an attempt to find + a date. + + Parameters + ---------- + line : str + Line which is searched by the date templates. + + Returns + ------- + float + The Unix timestamp returned from the first successfully matched + template. + """ self.__lock.acquire() try: for template in self.__templates: @@ -138,7 +185,8 @@ class DateDetector: date = template.getDate(line) if date is None: continue - logSys.debug("Got time %f for \"%r\" using template %s" % (date[0], date[1].group(), template.getName())) + logSys.debug("Got time %f for \"%r\" using template %s" % + (date[0], date[1].group(), template.name)) return date except ValueError: pass @@ -146,16 +194,19 @@ class DateDetector: finally: self.__lock.release() - ## - # Sort the template lists using the hits score. This method is not called - # in this object and thus should be called from time to time. - def sortTemplate(self): + """Sort the date templates by number of hits + + Sort the template lists using the hits score. This method is not + called in this object and thus should be called from time to time. + This ensures the most commonly matched templates are checked first, + improving performance of matchTime and getTime. + """ self.__lock.acquire() try: logSys.debug("Sorting the template list") - self.__templates.sort(key=lambda x: x.getHits(), reverse=True) + self.__templates.sort(key=lambda x: x.hits, reverse=True) t = self.__templates[0] - logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits())) + logSys.debug("Winning template: %s with %d hits" % (t.name, t.hits)) finally: self.__lock.release() diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e728ef3f..f33a4a8a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -26,211 +26,132 @@ __license__ = "GPL" import re, time, calendar import logging +from abc import abstractmethod from datetime import datetime from datetime import timedelta from .mytime import MyTime -from . import iso8601 from .strptime import reGroupDictStrptime, timeRE logSys = logging.getLogger(__name__) class DateTemplate(object): - + """A template which searches for and returns a date from a log line. + + This is an not functional abstract class which other templates should + inherit from. + """ + def __init__(self): - self.__name = "" - self.__regex = "" - self.__cRegex = None - self.__hits = 0 - - def setName(self, name): - self.__name = name - - def getName(self): - return self.__name - + """Initialise the date template. + """ + self._name = "" + self._regex = "" + self._cRegex = None + self.hits = 0 + + @property + def name(self): + """Name assigned to template. + """ + return self._name + + @name.setter + def name(self, name): + self._name = name + + def getRegex(self): + return self._regex + def setRegex(self, regex, wordBegin=True): - #logSys.debug(u"setRegex for %s is %r" % (self.__name, regex)) + """Sets regex to use for searching for date in log line. + + Parameters + ---------- + regex : str + The regex the template will use for searching for a date. + wordBegin : bool + Defines whether the regex should be modified to search at + begining of a word, by adding "\\b" to start of regex. + Default True. + + Raises + ------ + re.error + If regular expression fails to compile + """ regex = regex.strip() if (wordBegin and not re.search(r'^\^', regex)): regex = r'\b' + regex - self.__regex = regex - self.__cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - - def getRegex(self): - return self.__regex - - def getHits(self): - return self.__hits + self._regex = regex + self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) - def incHits(self): - self.__hits += 1 + regex = property(getRegex, setRegex, doc= + """Regex used to search for date. + """) - def resetHits(self): - self.__hits = 0 - def matchDate(self, line): - dateMatch = self.__cRegex.search(line) + """Check if regex for date matches on a log line. + """ + dateMatch = self._cRegex.search(line) return dateMatch - + + @abstractmethod def getDate(self, line): - raise Exception("matchDate() is abstract") + """Abstract method, which should return the date for a log line + + This should return the date for a log line, typically taking the + date from the part of the line which matched the templates regex. + This requires abstraction, therefore just raises exception. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Raises + ------ + NotImplementedError + Abstract method, therefore always returns this. + """ + raise NotImplementedError("getDate() is abstract") class DateEpoch(DateTemplate): - + """A date template which searches for Unix timestamps. + + This includes Unix timestamps which appear at start of a line, optionally + within square braces (nsd), or on SELinux audit log lines. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) - self.setRegex("(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))") - + self.regex = "(?:^|(?P(?<=^\[))|(?P(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))" + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch return (float(dateMatch.group()), dateMatch) return None - -## -# Use strptime() to parse a date. Our current locale is the 'C' -# one because we do not set the locale explicitly. This is POSIX -# standard. - -class DateStrptime(DateTemplate): - - TABLE = dict() - TABLE["Jan"] = ["Sty"] - TABLE["Feb"] = [u"Fév", "Lut"] - TABLE["Mar"] = [u"Mär", "Mar"] - TABLE["Apr"] = ["Avr", "Kwi"] - TABLE["May"] = ["Mai", "Maj"] - TABLE["Jun"] = ["Lip"] - TABLE["Jul"] = ["Sie"] - TABLE["Aug"] = ["Aou", "Wrz"] - TABLE["Sep"] = ["Sie"] - TABLE["Oct"] = [u"Paź"] - TABLE["Nov"] = ["Lis"] - TABLE["Dec"] = [u"Déc", "Dez", "Gru"] - - def __init__(self): - DateTemplate.__init__(self) - self._pattern = "" - self._unsupportedStrptimeBits = False - - def setPattern(self, pattern): - self._unsupported_f = not DateStrptime._f and re.search('%f', pattern) - self._unsupported_z = not DateStrptime._z and re.search('%z', pattern) - self._pattern = pattern - - def getPattern(self): - return self._pattern - - #@staticmethod - def convertLocale(date): - for t in DateStrptime.TABLE: - for m in DateStrptime.TABLE[t]: - if date.find(m) >= 0: - logSys.debug(u"Replacing %r with %r in %r" % - (m, t, date)) - return date.replace(m, t) - return date - convertLocale = staticmethod(convertLocale) - - def getDate(self, line): - dateMatch = self.matchDate(line) - - if dateMatch: - datePattern = self.getPattern() - if self._unsupported_f: - if dateMatch.group('_f'): - datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern) - logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern)) - if self._unsupported_z: - if dateMatch.group('_z'): - datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern) - logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) - try: - # Try first with 'C' locale - date = datetime.strptime(dateMatch.group(), datePattern) - except ValueError: - # Try to convert date string to 'C' locale - conv = self.convertLocale(dateMatch.group()) - try: - date = datetime.strptime(conv, self.getPattern()) - except (ValueError, re.error), e: - # Try to add the current year to the pattern. Should fix - # the "Feb 29" issue. - opattern = self.getPattern() - # makes sense only if %Y is not in already: - if not '%Y' in opattern: - pattern = "%s %%Y" % opattern - conv += " %s" % MyTime.gmtime()[0] - date = datetime.strptime(conv, pattern) - else: - # we are helpless here - raise ValueError( - "Given pattern %r does not match. Original " - "exception was %r and Feb 29 workaround could not " - "be tested due to already present year mark in the " - "pattern" % (opattern, e)) - - if self._unsupported_z: - z = dateMatch.group('_z') - if z: - delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:])) - direction = z[0] - logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta)) - # here we reverse the effect of the timezone and force it to UTC - if direction == '+': - date -= delta - else: - date += delta - date = date.replace(tzinfo=iso8601.Utc()) - else: - logSys.warning("No _z group captured and %%z is not supported on current platform" - " - timezone ignored and assumed to be localtime. date: %s on line: %s" - % (date, line)) - - if date.year < 2000: - # There is probably no year field in the logs - # NOTE: Possibly makes week/year day incorrect - date = date.replace(year=MyTime.gmtime()[0]) - # Bug fix for #1241756 - # If the date is greater than the current time, we suppose - # that the log is not from this year but from the year before - if date > MyTime.now(): - logSys.debug( - u"Correcting deduced year by one since %s > now (%s)" % - (date, MyTime.time())) - date = date.replace(year=date.year-1) - elif date.month == 1 and date.day == 1: - # If it is Jan 1st, it is either really Jan 1st or there - # is neither month nor day in the log. - # NOTE: Possibly makes week/year day incorrect - date = date.replace( - month=MyTime.gmtime()[1], day=MyTime.gmtime()[2]) - - if date.tzinfo: - return ( calendar.timegm(date.utctimetuple()), dateMatch ) - else: - return ( time.mktime(date.utctimetuple()), dateMatch ) - - return None - -try: - time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") - DateStrptime._f = True -except (ValueError, KeyError): - DateTemplate._f = False - -try: - time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z") - DateStrptime._z = True -except ValueError: - DateStrptime._z = False - -class DatePatternRegex(DateStrptime): +class DatePatternRegex(DateTemplate): _patternRE = r"%%(%%|[%s])" % "".join(timeRE.keys()) _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", @@ -241,39 +162,97 @@ class DatePatternRegex(DateStrptime): for key in set(timeRE) - set(_patternName): # may not have them all... _patternName[key] = "%%%s" % key - def __init__(self, pattern=None, **kwargs): - super(DatePatternRegex, self).__init__() - if pattern: - self.setPattern(pattern, **kwargs) + def __init__(self, pattern=None): + """Initialise date template, with optional regex/pattern - def setPattern(self, pattern): - super(DatePatternRegex, self).setPattern(pattern) - super(DatePatternRegex, self).setName( - re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName) + Parameters + ---------- + pattern : str + Sets the date templates pattern. + """ + super(DatePatternRegex, self).__init__() + self._pattern = None + if pattern is not None: + self.pattern = pattern + + @property + def pattern(self): + """The pattern used for regex with strptime "%" time fields. + + This should be a valid regular expression, of which matching string + will be extracted from the log line. strptime style "%" fields will + be replaced by appropriate regular expressions, or custom regex + groups with names as per the strptime fields can also be used + instead. + """ + return self._pattern + + @pattern.setter + def pattern(self, pattern): + self._pattern = pattern + self._name = re.sub( + self._patternRE, r'%(\1)s', pattern) % self._patternName super(DatePatternRegex, self).setRegex( re.sub(self._patternRE, r'%(\1)s', pattern) % timeRE) - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - return reGroupDictStrptime(dateMatch.groupdict()), dateMatch - - def setRegex(self, line): + def setRegex(self, value): raise NotImplementedError("Regex derived from pattern") - def setName(self, line): + @DateTemplate.name.setter + def name(self, value): raise NotImplementedError("Name derived from pattern") + def getDate(self, line): + """Method to return the date for a log line. + + This uses a custom version of strptime, using the named groups + from the instances `pattern` property. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ + dateMatch = self.matchDate(line) + if dateMatch: + groupdict = dict( + (key, value) + for key, value in dateMatch.groupdict().iteritems() + if value is not None) + return reGroupDictStrptime(groupdict), dateMatch class DateTai64n(DateTemplate): - + """A date template which matches TAI64N formate timestamps. + """ + def __init__(self): + """Initialise the date template. + """ DateTemplate.__init__(self) # We already know the format for TAI64N # yoh: we should not add an additional front anchor self.setRegex("@[0-9a-f]{24}", wordBegin=False) - + def getDate(self, line): + """Method to return the date for a log line. + + Parameters + ---------- + line : str + Log line, of which the date should be extracted from. + + Returns + ------- + (float, str) + Tuple containing a Unix timestamp, and the string of the date + which was matched and in turned used to calculated the timestamp. + """ dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch @@ -282,19 +261,3 @@ class DateTai64n(DateTemplate): # convert seconds from HEX into local time stamp return (int(seconds_since_epoch, 16), dateMatch) return None - - -class DateISO8601(DateTemplate): - - def __init__(self): - DateTemplate.__init__(self) - self.setRegex(iso8601.ISO8601_REGEX_RAW) - - def getDate(self, line): - dateMatch = self.matchDate(line) - if dateMatch: - # Parses the date. - value = dateMatch.group() - return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch) - return None - diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 9d801afe..eae79bac 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -27,7 +27,7 @@ from .failmanager import FailManagerEmpty, FailManager from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector -from .datetemplate import DatePatternRegex, DateISO8601, DateEpoch, DateTai64n +from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction @@ -202,24 +202,20 @@ class Filter(JailThread): if pattern is None: self.dateDetector = None return - elif pattern.upper() == "ISO8601": - template = DateISO8601() - template.setName("ISO8601") elif pattern.upper() == "EPOCH": template = DateEpoch() - template.setName("Epoch") + template.name = "Epoch" elif pattern.upper() == "TAI64N": template = DateTai64n() - template.setName("TAI64N") + template.name = "TAI64N" else: - template = DatePatternRegex() - template.setPattern(pattern) + template = DatePatternRegex(pattern) self.dateDetector = DateDetector() self.dateDetector.appendTemplate(template) logSys.info("Date pattern set to `%r`: `%s`" % - (pattern, template.getName())) + (pattern, template.name)) logSys.debug("Date pattern regex for %r: %s" % - (pattern, template.getRegex())) + (pattern, template.regex)) ## # Get the date detector pattern, or Default Detectors if not changed @@ -228,15 +224,15 @@ class Filter(JailThread): def getDatePattern(self): if self.dateDetector is not None: - templates = self.dateDetector.getTemplates() + templates = self.dateDetector.templates if len(templates) > 1: return None, "Default Detectors" elif len(templates) == 1: - if hasattr(templates[0], "getPattern"): - pattern = templates[0].getPattern() + if hasattr(templates[0], "pattern"): + pattern = templates[0].pattern else: pattern = None - return pattern, templates[0].getName() + return pattern, templates[0].name ## # Set the maximum retry value. diff --git a/fail2ban/server/iso8601.py b/fail2ban/server/iso8601.py deleted file mode 100644 index a28ce13b..00000000 --- a/fail2ban/server/iso8601.py +++ /dev/null @@ -1,136 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: - -# Copyright (c) 2007 Michael Twomey -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""ISO 8601 date time string parsing - -Basic usage: ->>> import iso8601 ->>> iso8601.parse_date("2007-01-25T12:00:00Z") -datetime.datetime(2007, 1, 25, 12, 0, tzinfo=) ->>> - -""" - -from datetime import datetime, timedelta, tzinfo -import time -import re - -__all__ = ["parse_date", "ParseError"] - -# Adapted from http://delete.me.uk/2005/03/iso8601.html -ISO8601_REGEX_RAW = "(?P[0-9]{4})-(?P[0-9]{1,2})-(?P[0-9]{1,2})" \ - "T(?P[0-9]{2}):(?P[0-9]{2})(:(?P[0-9]{2})(\.(?P[0-9]+))?)?" \ - "(?PZ|[-+][0-9]{2}(:?[0-9]{2})?)?" -ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW) -TIMEZONE_REGEX = re.compile("(?P[+-])(?P[0-9]{2}):?(?P[0-9]{2})?") - -class ParseError(Exception): - """Raised when there is a problem parsing a date string""" - -# Yoinked from python docs -ZERO = timedelta(0) -class Utc(tzinfo): - """UTC - - """ - def utcoffset(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return ZERO -UTC = Utc() - -class FixedOffset(tzinfo): - """Fixed offset in hours and minutes from UTC - - """ - def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0): - self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds) - self.__name = name - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return self.__name - - def dst(self, dt): - return ZERO - - def __repr__(self): - return "" % self.__name - -def parse_timezone(tzstring): - """Parses ISO 8601 time zone specs into tzinfo offsets - - """ - if tzstring == "Z": - return UTC - - if tzstring is None: - zone_sec = -time.timezone - return FixedOffset(name=time.tzname[0],offset_hours=(zone_sec / 3600), offset_minutes=(zone_sec % 3600)/60, offset_seconds=zone_sec % 60) - - m = TIMEZONE_REGEX.match(tzstring) - prefix, hours, minutes = m.groups() - if minutes is None: - minutes = 0 - else: - minutes = int(minutes) - hours = int(hours) - if prefix == "-": - hours = -hours - minutes = -minutes - return FixedOffset(tzstring, hours, minutes) - -def parse_date(datestring): - """Parses ISO 8601 dates into datetime objects - - The timezone is parsed from the date string. However it is quite common to - have dates without a timezone (not strictly correct). In this case the - default timezone specified in default_timezone is used. This is UTC by - default. - """ - if not isinstance(datestring, basestring): - raise ValueError("Expecting a string %r" % datestring) - m = ISO8601_REGEX.match(datestring) - if not m: - raise ParseError("Unable to parse date string %r" % datestring) - groups = m.groupdict() - tz = parse_timezone(groups["timezone"]) - if groups["fraction"] is None: - groups["fraction"] = 0 - else: - groups["fraction"] = int(float("0.%s" % groups["fraction"]) * 1e6) - - try: - return datetime(int(groups["year"]), int(groups["month"]), int(groups["day"]), - int(groups["hour"]), int(groups["minute"]), int(groups["second"]), - int(groups["fraction"]), tz) - except Exception, e: - raise ParseError("Failed to create a valid datetime record due to: %s" - % e) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index da04495f..b23a424e 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -26,11 +26,24 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() -if 'z' not in timeRE: # python2.6 not present - timeRE['z'] = r"(?P[+-]\d{2}[0-5]\d)" +timeRE['z'] = r"(?PZ|[+-]\d{2}(?::?[0-5]\d)?)" def reGroupDictStrptime(found_dict): - """This is tweaked from python built-in _strptime""" + """Return time from dictionary of strptime fields + + This is tweaked from python built-in _strptime. + + Parameters + ---------- + found_dict : dict + Dictionary where keys represent the strptime fields, and values the + respective value. + + Returns + ------- + float + Unix time stamp. + """ now = MyTime.now() year = month = day = hour = minute = None @@ -119,9 +132,14 @@ def reGroupDictStrptime(found_dict): week_of_year_start = 0 elif group_key == 'z': z = found_dict['z'] - tzoffset = int(z[1:3]) * 60 + int(z[3:5]) - if z.startswith("-"): - tzoffset = -tzoffset + if z == "Z": + tzoffset = 0 + else: + tzoffset = int(z[1:3]) * 60 # Hours... + if len(z)>3: + tzoffset += int(z[-2:]) # ...and minutes + if z.startswith("-"): + tzoffset = -tzoffset elif group_key == 'Z': # Since -1 is default value only need to worry about setting tz if # it can be something other than -1. @@ -158,7 +176,6 @@ def reGroupDictStrptime(found_dict): month = datetime_result.month day = datetime_result.day # Add timezone info - tzname = found_dict.get("Z") if tzoffset is not None: gmtoff = tzoffset * 60 else: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 79723e31..45e1bbf5 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -27,7 +27,6 @@ __license__ = "GPL" import unittest, calendar, time, datetime, re, pprint from ..server.datedetector import DateDetector from ..server.datetemplate import DateTemplate -from ..server.iso8601 import Utc from .utils import setUpMyTime, tearDownMyTime class DateDetectorTest(unittest.TestCase): @@ -88,8 +87,9 @@ class DateDetectorTest(unittest.TestCase): (False, "23-01-2005 21:59:59"), (False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break (False, "@4000000041f4104f00000000"), # TAI64N - (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 + (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC) (False, "2005-01-23T15:59:59-05:00"), #ISO 8601 with TZ + (False, "2005-01-23T21:59:59"), #ISO 8601 no TZ, assume local (True, "<01/23/05@21:59:59>"), (True, "050123 21:59:59"), # MySQL (True, "Jan-23-05 21:59:59"), # ASSP like @@ -116,15 +116,15 @@ class DateDetectorTest(unittest.TestCase): self.assertEqual(logtime, None, "getTime should have not matched for %r Got: %s" % (sdate, logtime)) def testStableSortTemplate(self): - old_names = [x.getName() for x in self.__datedetector.getTemplates()] + old_names = [x.name for x in self.__datedetector.templates] self.__datedetector.sortTemplate() # If there were no hits -- sorting should not change the order - for old_name, n in zip(old_names, self.__datedetector.getTemplates()): - self.assertEqual(old_name, n.getName()) # "Sort must be stable" + for old_name, n in zip(old_names, self.__datedetector.templates): + self.assertEqual(old_name, n.name) # "Sort must be stable" def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.__datedetector.appendTemplate, - self.__datedetector.getTemplates()[0]) + self.__datedetector.templates[0]) def testFullYearMatch_gh130(self): # see https://github.com/fail2ban/fail2ban/pull/130 diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 64b4355c..e9209b42 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -170,46 +170,46 @@ class TestsUtilsTest(unittest.TestCase): self.assertTrue(pindex > 10) # we should have some traceback self.assertEqual(s[:pindex], s[pindex+1:pindex*2 + 1]) -from ..server import iso8601 import datetime import time +from ..server.datetemplate import DatePatternRegex + +iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") + class CustomDateFormatsTest(unittest.TestCase): def testIso8601(self): - date = iso8601.parse_date("2007-01-25T12:00:00Z") + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00Z")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 12, 0, tzinfo=iso8601.Utc())) - self.assertRaises(ValueError, iso8601.parse_date, None) - self.assertRaises(ValueError, iso8601.parse_date, date) + datetime.datetime(2007, 1, 25, 12, 0)) + self.assertRaises(TypeError, iso8601.getDate, None) + self.assertRaises(TypeError, iso8601.getDate, date) - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "") - self.assertRaises(iso8601.ParseError, iso8601.parse_date, "Z") + self.assertEqual(iso8601.getDate(""), None) + self.assertEqual(iso8601.getDate("Z"), None) - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-01-01T120:00:00Z") - self.assertRaises(iso8601.ParseError, - iso8601.parse_date, "2007-13-01T12:00:00Z") - date = iso8601.parse_date("2007-01-25T12:00:00+0400") + self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) + self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00+04:00") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 8, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-0400") + datetime.datetime(2007, 1, 25, 8, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-0400")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - date = iso8601.parse_date("2007-01-25T12:00:00-04") + datetime.datetime(2007, 1, 25, 16, 0)) + date = datetime.datetime.fromtimestamp( + iso8601.getDate("2007-01-25T12:00:00-04")[0]) self.assertEqual( date, - datetime.datetime(2007, 1, 25, 16, 0, tzinfo=iso8601.Utc())) - - def testTimeZone(self): - # Just verify consistent operation and improve coverage ;) - self.assertEqual((iso8601.parse_timezone(None).tzname(False), iso8601.parse_timezone(None).tzname(True)), time.tzname) - self.assertEqual(iso8601.parse_timezone('Z').tzname(True), "UTC") - self.assertEqual(iso8601.parse_timezone('Z').dst(True), datetime.timedelta(0)) + datetime.datetime(2007, 1, 25, 16, 0)) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 648a6bf9..0d1c52d1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -278,8 +278,6 @@ class Transmitter(TransmitterBase): "datepattern", "Epoch", (None, "Epoch"), jail=self.jailName) self.setGetTest( "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) - self.setGetTest( - "datepattern", "ISO8601", (None, "ISO8601"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) def testJailUseDNS(self):