Code review and small optimizations, prepared to provide offset-based time zones for date-detectors (parsing of input-string)

pull/1792/head
sebres 2017-06-09 13:55:30 +02:00
parent e8f2173904
commit 8cb4ae0242
4 changed files with 46 additions and 27 deletions

View File

@ -27,6 +27,7 @@ import time
from threading import Lock
from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
from .strptime import validateTimeZone
from .utils import Utils
from ..helpers import getLogger
@ -222,6 +223,8 @@ class DateDetector(object):
self.__firstUnused = 0
# pre-match pattern:
self.__preMatch = None
# default TZ (if set, treat log lines without explicit time zone to be in this time zone):
self.__default_tz = None
def _appendTemplate(self, template, ignoreDup=False):
name = template.name
@ -423,7 +426,15 @@ class DateDetector(object):
logSys.log(logLevel, " no template.")
return (None, None)
def getTime(self, line, timeMatch=None, default_tz=None):
@property
def default_tz(self):
return self.__default_tz
@default_tz.setter
def default_tz(self, value):
self.__default_tz = validateTimeZone(value)
def getTime(self, line, timeMatch=None):
"""Attempts to return the date on a log line using templates.
This uses the templates' `getDate` method in an attempt to find
@ -449,7 +460,7 @@ class DateDetector(object):
template = timeMatch[1]
if template is not None:
try:
date = template.getDate(line, timeMatch[0], default_tz=default_tz)
date = template.getDate(line, timeMatch[0], default_tz=self.__default_tz)
if date is not None:
if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
logSys.log(logLevel, " got time %f for %r using template %s",

View File

@ -34,13 +34,12 @@ from .failmanager import FailManagerEmpty, FailManager
from .ipdns import DNSUtils, IPAddr
from .ticket import FailTicket
from .jailthread import JailThread
from .datedetector import DateDetector
from .datedetector import DateDetector, validateTimeZone
from .mytime import MyTime
from .failregex import FailRegex, Regex, RegexException
from .action import CommandAction
from .utils import Utils
from ..helpers import getLogger, PREFER_ENC
from .strptime import validateTimeZone
# Gets the instance of the logger.
logSys = getLogger(__name__)
@ -88,6 +87,8 @@ class Filter(JailThread):
## Store last time stamp, applicable for multi-line
self.__lastTimeText = ""
self.__lastDate = None
## if set, treat log lines without explicit time zone to be in this time zone
self.__logtimezone = None
## External command
self.__ignoreCommand = False
## Default or preferred encoding (to decode bytes from file or journal):
@ -103,8 +104,6 @@ class Filter(JailThread):
self.checkAllRegex = False
## if true ignores obsolete failures (failure time < now - findTime):
self.checkFindTime = True
## if set, treat log lines without explicit time zone to be in this time zone
self.logtimezone = None
## Ticks counter
self.ticks = 0
@ -285,6 +284,7 @@ class Filter(JailThread):
return
else:
dd = DateDetector()
dd.default_tz = self.__logtimezone
if not isinstance(pattern, (list, tuple)):
pattern = filter(bool, map(str.strip, re.split('\n+', pattern)))
for pattern in pattern:
@ -316,7 +316,9 @@ class Filter(JailThread):
# @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM)
def setLogTimeZone(self, tz):
self.logtimezone = validateTimeZone(tz)
validateTimeZone(tz); # avoid setting of wrong value, but hold original
self.__logtimezone = tz
if self.dateDetector: self.dateDetector.default_tz = self.__logtimezone
##
# Get the log default timezone
@ -324,7 +326,7 @@ class Filter(JailThread):
# @return symbolic timezone (a string)
def getLogTimeZone(self):
return self.logtimezone
return self.__logtimezone
##
# Set the maximum retry value.
@ -640,8 +642,7 @@ class Filter(JailThread):
self.__lastDate = date
elif timeText:
dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3],
default_tz=self.logtimezone)
dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
if dateTimeMatch is None:
logSys.error("findFailure failed to parse timeText: %s", timeText)
@ -994,8 +995,7 @@ class FileFilter(Filter):
if timeMatch:
dateTimeMatch = self.dateDetector.getTime(
line[timeMatch.start():timeMatch.end()],
(timeMatch, template),
default_tz=self.logtimezone)
(timeMatch, template))
else:
nextp = container.tell()
if nextp > maxp:

View File

@ -27,7 +27,7 @@ from .mytime import MyTime
locale_time = LocaleTime()
timeRE = TimeRE()
FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$')
FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$')
def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
""" Build century regex for last year and the next years (distance).
@ -84,30 +84,33 @@ def getTimePatternRE():
def validateTimeZone(tz):
"""Validate a timezone.
For now this accepts only the UTC[+-]hhmm format.
For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
In the future, it may be extended for named time zones (such as Europe/Paris)
present on the system, if a suitable tz library is present.
"""
if tz is None:
return None
m = FIXED_OFFSET_TZ_RE.match(tz)
if m is None:
raise ValueError("Unknown or unsupported time zone: %r" % tz)
return tz
tz = m.group(1)
if tz is None or tz == '': # UTC/GMT
return 0; # fixed zero offzet
return zone2offset(tz, 0)
def zone2offset(tz, dt):
"""Return the proper offset, in minutes according to given timezone at a given time.
Parameters
----------
tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have
been validated already)
dt: datetime instance for offset computation
tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have
been validated already)
dt: datetime instance for offset computation
"""
if tz == 'UTC':
return 0
unsigned = int(tz[4:6])*60 + int(tz[6:])
if tz[3] == '-':
return -unsigned
return unsigned
if isinstance(tz, basestring):
# [+-]1 * (hh*60 + mm)
return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
return tz
def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
"""Return time from dictionary of strptime fields

View File

@ -91,19 +91,24 @@ class DateDetectorTest(LogCaptureTestCase):
def testDefaultTimeZone(self):
log = "2017-01-23 15:00:00"
datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300')
dd = self.datedetector
dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log)
# so in UTC, it was noon
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 12, 0, 0))
datelog, _ = self.datedetector.getTime(log, default_tz='UTC')
dd.default_tz='UTC'; datelog, _ = dd.getTime(log)
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 15, 0, 0))
self.assertEqual(dd.default_tz, 0); # utc == 0
datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430')
dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log)
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 19, 30, 0))
self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
dd.default_tz = None
def testVariousTimes(self):
"""Test detection of various common date/time formats f2b should understand
"""