|
|
|
|
@ -30,7 +30,7 @@ import datetime
|
|
|
|
|
|
|
|
|
|
from ..server.datedetector import DateDetector
|
|
|
|
|
from ..server import datedetector
|
|
|
|
|
from ..server.datetemplate import DateTemplate
|
|
|
|
|
from ..server.datetemplate import DatePatternRegex, DateTemplate
|
|
|
|
|
from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase
|
|
|
|
|
from ..helpers import getLogger
|
|
|
|
|
|
|
|
|
|
@ -89,6 +89,10 @@ class DateDetectorTest(LogCaptureTestCase):
|
|
|
|
|
"""
|
|
|
|
|
dateUnix = 1106513999.0
|
|
|
|
|
|
|
|
|
|
# anchored - matching expression (pattern) is anchored
|
|
|
|
|
# bound - pattern can be tested using word boundary (e.g. False if contains in front some optional part)
|
|
|
|
|
# sdate - date string used in test log-line
|
|
|
|
|
# rdate - if specified, the result match, which differs from sdate
|
|
|
|
|
for anchored, bound, sdate, rdate in (
|
|
|
|
|
(False, True, "Jan 23 21:59:59", None),
|
|
|
|
|
(False, False, "Sun Jan 23 21:59:59 2005", None),
|
|
|
|
|
@ -113,15 +117,15 @@ class DateDetectorTest(LogCaptureTestCase):
|
|
|
|
|
(False, True, "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC)
|
|
|
|
|
(False, True, "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ
|
|
|
|
|
(False, True, "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local
|
|
|
|
|
(False, True, "20050123T215959", None), #Short ISO
|
|
|
|
|
(False, True, "20050123 215959", None), #Short ISO
|
|
|
|
|
(False, True, "20050123T215959", None), #Short ISO with T
|
|
|
|
|
(False, True, "20050123 215959", None), #Short ISO with space
|
|
|
|
|
(True, True, "<01/23/05@21:59:59>", None),
|
|
|
|
|
(False, True, "050123 21:59:59", None), # MySQL
|
|
|
|
|
(True, True, "Jan-23-05 21:59:59", None), # ASSP like
|
|
|
|
|
(False, True, "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat
|
|
|
|
|
(True, True, "1106513999", None), # Regular epoch
|
|
|
|
|
(True, True, "1106513999.000", None), # Regular epoch with millisec
|
|
|
|
|
(True, True, "[1106513999.000]", "1106513999.000"), # epoch squared
|
|
|
|
|
(True, True, "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match)
|
|
|
|
|
(False, True, "audit(1106513999.000:987)", "1106513999.000"), # SELinux
|
|
|
|
|
):
|
|
|
|
|
logSys.debug('== test %r', (anchored, bound, sdate))
|
|
|
|
|
@ -195,6 +199,141 @@ class DateDetectorTest(LogCaptureTestCase):
|
|
|
|
|
self.assertEqual(t.matchDate('aaaac').group(), 'aaaac')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z")
|
|
|
|
|
|
|
|
|
|
class CustomDateFormatsTest(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
def testIso8601(self):
|
|
|
|
|
date = datetime.datetime.utcfromtimestamp(
|
|
|
|
|
iso8601.getDate("2007-01-25T12:00:00Z")[0])
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
date,
|
|
|
|
|
datetime.datetime(2007, 1, 25, 12, 0))
|
|
|
|
|
self.assertRaises(TypeError, iso8601.getDate, None)
|
|
|
|
|
self.assertRaises(TypeError, iso8601.getDate, date)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(iso8601.getDate(""), None)
|
|
|
|
|
self.assertEqual(iso8601.getDate("Z"), None)
|
|
|
|
|
|
|
|
|
|
self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None)
|
|
|
|
|
self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None)
|
|
|
|
|
date = datetime.datetime.utcfromtimestamp(
|
|
|
|
|
iso8601.getDate("2007-01-25T12:00:00+0400")[0])
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
date,
|
|
|
|
|
datetime.datetime(2007, 1, 25, 8, 0))
|
|
|
|
|
date = datetime.datetime.utcfromtimestamp(
|
|
|
|
|
iso8601.getDate("2007-01-25T12:00:00+04:00")[0])
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
date,
|
|
|
|
|
datetime.datetime(2007, 1, 25, 8, 0))
|
|
|
|
|
date = datetime.datetime.utcfromtimestamp(
|
|
|
|
|
iso8601.getDate("2007-01-25T12:00:00-0400")[0])
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
date,
|
|
|
|
|
datetime.datetime(2007, 1, 25, 16, 0))
|
|
|
|
|
date = datetime.datetime.utcfromtimestamp(
|
|
|
|
|
iso8601.getDate("2007-01-25T12:00:00-04")[0])
|
|
|
|
|
self.assertEqual(
|
|
|
|
|
date,
|
|
|
|
|
datetime.datetime(2007, 1, 25, 16, 0))
|
|
|
|
|
|
|
|
|
|
def testAmbiguousDatePattern(self):
|
|
|
|
|
defDD = DateDetector()
|
|
|
|
|
defDD.addDefaultTemplate()
|
|
|
|
|
for (matched, dp, line) in (
|
|
|
|
|
# positive case:
|
|
|
|
|
('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'),
|
|
|
|
|
# ambiguous "unbound" patterns (missed):
|
|
|
|
|
(False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'),
|
|
|
|
|
(False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'),
|
|
|
|
|
# ambiguous "no optional year" patterns (matched):
|
|
|
|
|
('Aug 8 11:25:50', None, 'Aug 8 11:25:50 20030f2329b8 Authentication failed from 192.0.2.1'),
|
|
|
|
|
('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 20030f2329b8 Authentication failed from 192.0.2.1'),
|
|
|
|
|
('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 20030f2329b8 Authentication failed from 192.0.2.1'),
|
|
|
|
|
# direct specified patterns:
|
|
|
|
|
('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'),
|
|
|
|
|
('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'),
|
|
|
|
|
('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
|
|
|
|
|
('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'),
|
|
|
|
|
('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'),
|
|
|
|
|
('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'),
|
|
|
|
|
('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'),
|
|
|
|
|
('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'),
|
|
|
|
|
# direct specified patterns (begin/end, missed):
|
|
|
|
|
(False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'),
|
|
|
|
|
(False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'),
|
|
|
|
|
# direct specified unbound patterns (no begin/end boundary):
|
|
|
|
|
('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '192.0.2.1x20:00:00 01.02.2003'),
|
|
|
|
|
('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '20:00:00 01.02.2003x192.0.2.1'),
|
|
|
|
|
# pattern enclosed with stars (in comparison to example above):
|
|
|
|
|
('*20:00:00 01.02.2003*', r'\**%H:%M:%S %d.%m.%Y\**', 'test*20:00:00 01.02.2003*test'),
|
|
|
|
|
# direct specified patterns (begin/end, matched):
|
|
|
|
|
('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'),
|
|
|
|
|
('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'),
|
|
|
|
|
# wrong year in 1st date, so failed by convert using not precise year (filter used last known date),
|
|
|
|
|
# in the 2nd and 3th tests (with precise year) it should find correct the 2nd date:
|
|
|
|
|
(None, r'%Y-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
|
|
|
|
|
('2003-12-30 00:00:00', r'%ExY-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
|
|
|
|
|
('2003-12-30 00:00:00', None, "0000-12-30 00:00:00 - 2003-12-30 00:00:00"),
|
|
|
|
|
# wrong date recognized short month/day (unbounded date pattern without separator between parts),
|
|
|
|
|
# in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date:
|
|
|
|
|
('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
('20031230 010203', None, "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
# Explicit bound in start of the line using %ExLB key,
|
|
|
|
|
# (negative) in the 1st case without line begin boundary - wrong date may be found,
|
|
|
|
|
# (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found)
|
|
|
|
|
# (positive) and in 3th case with line begin boundary - find the correct date
|
|
|
|
|
("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
|
|
|
|
|
(None, "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"),
|
|
|
|
|
("20031230 010203", "%ExLB%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"),
|
|
|
|
|
# Explicit bound in start of the line using %ExLB key,
|
|
|
|
|
# up to 2 non-alphanumeric chars front, ** - no word boundary on the right
|
|
|
|
|
("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"),
|
|
|
|
|
("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"),
|
|
|
|
|
("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"),
|
|
|
|
|
("20031230010203", "%ExLB%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"),
|
|
|
|
|
):
|
|
|
|
|
logSys.debug('== test: %r', (matched, dp, line))
|
|
|
|
|
if dp is None:
|
|
|
|
|
dd = defDD
|
|
|
|
|
else:
|
|
|
|
|
dp = DatePatternRegex(dp)
|
|
|
|
|
dd = DateDetector()
|
|
|
|
|
dd.appendTemplate(dp)
|
|
|
|
|
date = dd.getTime(line)
|
|
|
|
|
if matched:
|
|
|
|
|
self.assertTrue(date)
|
|
|
|
|
self.assertEqual(matched, date[1].group())
|
|
|
|
|
else:
|
|
|
|
|
self.assertEqual(date, None)
|
|
|
|
|
|
|
|
|
|
# def testAmbiguousUsingOrderedTemplates(self):
|
|
|
|
|
# defDD = DateDetector()
|
|
|
|
|
# defDD.addDefaultTemplate()
|
|
|
|
|
# for (matched, dp, line) in (
|
|
|
|
|
# # wrong date recognized short month/day (unbounded date pattern without separator),
|
|
|
|
|
# # in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date:
|
|
|
|
|
# ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
# ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
# ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"),
|
|
|
|
|
# ):
|
|
|
|
|
# logSys.debug('== test: %r', (matched, dp, line))
|
|
|
|
|
# if dp is None:
|
|
|
|
|
# dd = defDD
|
|
|
|
|
# else:
|
|
|
|
|
# dp = DatePatternRegex(dp)
|
|
|
|
|
# dd = DateDetector()
|
|
|
|
|
# dd.appendTemplate(dp)
|
|
|
|
|
# date = dd.getTime(line)
|
|
|
|
|
# if matched:
|
|
|
|
|
# self.assertTrue(date)
|
|
|
|
|
# self.assertEqual(matched, date[1].group())
|
|
|
|
|
# else:
|
|
|
|
|
# self.assertEqual(date, None)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# def testDefaultTempate(self):
|
|
|
|
|
# self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
|
|
|
|
|
# self.__datedetector.setDefaultPattern("%b %d %H:%M:%S")
|
|
|
|
|
|
sebres
8 years ago