From d17b415371d14f00c9b24fc3672733b6666fe345 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Tue, 5 Mar 2013 00:02:39 +0100
Subject: [PATCH 1/2] invalid date recognition, irregular because of sorting
template list (sometimes not reproducible by fail2ban-regex, cause will be
not sorted)
date in following log line (from nginx) will be wrong detected:
2012/10/11 02:37:17 [error] 18434#0: *947 user "test" was not found in "/www/...", client: 192.168.1.5, ...
sometimes it is [correct] - 2012/10/11 (%Y/%m/%d) = 1349919861.71
sometimes it is [invalid] - 12/10/11 (%d/%m/%y) = 1349915838.06
and older as now - 1800 seconds (therefore will be not found)
solution: regular expression fixed, cause date in log used always after non symbol (\W) character, so r"\d{2}/\d{2}/\d{2}" will be r"(?<!\w)\d{2}/\d{2}/\d{2}".
---
server/datedetector.py | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/server/datedetector.py b/server/datedetector.py
index c013d551..6e484449 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -55,80 +55,80 @@ class DateDetector:
# standard
template = DateStrptime()
template.setName("MONTH Day Hour:Minute:Second")
- template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
template.setPattern("%b %d %H:%M:%S")
self._appendTemplate(template)
# asctime
template = DateStrptime()
template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year")
- template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
+ template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
template.setPattern("%a %b %d %H:%M:%S %Y")
self._appendTemplate(template)
# asctime without year
template = DateStrptime()
template.setName("WEEKDAY MONTH Day Hour:Minute:Second")
- template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
template.setPattern("%a %b %d %H:%M:%S")
self._appendTemplate(template)
# simple date
template = DateStrptime()
template.setName("Year/Month/Day Hour:Minute:Second")
- template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y/%m/%d %H:%M:%S")
self._appendTemplate(template)
# simple date too (from x11vnc)
template = DateStrptime()
template.setName("Day/Month/Year Hour:Minute:Second")
- template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d/%m/%Y %H:%M:%S")
self._appendTemplate(template)
# previous one but with year given by 2 digits
# (See http://bugs.debian.org/537610)
template = DateStrptime()
template.setName("Day/Month/Year2 Hour:Minute:Second")
- template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%d/%m/%y %H:%M:%S")
self._appendTemplate(template)
# Apache format [31/Oct/2006:09:22:55 -0000]
template = DateStrptime()
template.setName("Day/MONTH/Year:Hour:Minute:Second")
- template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
template.setPattern("%d/%b/%Y:%H:%M:%S")
self._appendTemplate(template)
# CPanel 05/20/2008:01:57:39
template = DateStrptime()
template.setName("Month/Day/Year:Hour:Minute:Second")
- template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
template.setPattern("%m/%d/%Y:%H:%M:%S")
self._appendTemplate(template)
# Exim 2006-12-21 06:43:20
template = DateStrptime()
template.setName("Year-Month-Day Hour:Minute:Second")
- template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y-%m-%d %H:%M:%S")
self._appendTemplate(template)
# custom for syslog-ng 2006.12.21 06:43:20
template = DateStrptime()
template.setName("Year.Month.Day Hour:Minute:Second")
- template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y.%m.%d %H:%M:%S")
self._appendTemplate(template)
# named 26-Jul-2007 15:20:52.252
template = DateStrptime()
template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]")
- template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d-%b-%Y %H:%M:%S")
self._appendTemplate(template)
# 17-07-2008 17:23:25
template = DateStrptime()
template.setName("Day-Month-Year Hour:Minute:Second")
- template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d-%m-%Y %H:%M:%S")
self._appendTemplate(template)
# 01-27-2012 16:22:44.252
template = DateStrptime()
template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]")
- template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%m-%d-%Y %H:%M:%S")
self._appendTemplate(template)
# TAI64N
From b6bb2f88c1dbb111647269590d80d95f72c81c3e Mon Sep 17 00:00:00 2001
From: sebres <sergey.brester@W7-DEHBG0189.wincor-nixdorf.com>
Date: Mon, 11 Mar 2013 13:52:31 +0100
Subject: [PATCH 2/2] invalid date recognition, irregular because of sorting
template list, now via setRegex
---
server/datedetector.py | 26 +++++++++++++-------------
server/datetemplate.py | 7 +++++--
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/server/datedetector.py b/server/datedetector.py
index 6e484449..c013d551 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -55,80 +55,80 @@ class DateDetector:
# standard
template = DateStrptime()
template.setName("MONTH Day Hour:Minute:Second")
- template.setRegex("(?<!\w)\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
template.setPattern("%b %d %H:%M:%S")
self._appendTemplate(template)
# asctime
template = DateStrptime()
template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year")
- template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
+ template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
template.setPattern("%a %b %d %H:%M:%S %Y")
self._appendTemplate(template)
# asctime without year
template = DateStrptime()
template.setName("WEEKDAY MONTH Day Hour:Minute:Second")
- template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
template.setPattern("%a %b %d %H:%M:%S")
self._appendTemplate(template)
# simple date
template = DateStrptime()
template.setName("Year/Month/Day Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y/%m/%d %H:%M:%S")
self._appendTemplate(template)
# simple date too (from x11vnc)
template = DateStrptime()
template.setName("Day/Month/Year Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d/%m/%Y %H:%M:%S")
self._appendTemplate(template)
# previous one but with year given by 2 digits
# (See http://bugs.debian.org/537610)
template = DateStrptime()
template.setName("Day/Month/Year2 Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%d/%m/%y %H:%M:%S")
self._appendTemplate(template)
# Apache format [31/Oct/2006:09:22:55 -0000]
template = DateStrptime()
template.setName("Day/MONTH/Year:Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
template.setPattern("%d/%b/%Y:%H:%M:%S")
self._appendTemplate(template)
# CPanel 05/20/2008:01:57:39
template = DateStrptime()
template.setName("Month/Day/Year:Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
template.setPattern("%m/%d/%Y:%H:%M:%S")
self._appendTemplate(template)
# Exim 2006-12-21 06:43:20
template = DateStrptime()
template.setName("Year-Month-Day Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y-%m-%d %H:%M:%S")
self._appendTemplate(template)
# custom for syslog-ng 2006.12.21 06:43:20
template = DateStrptime()
template.setName("Year.Month.Day Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
template.setPattern("%Y.%m.%d %H:%M:%S")
self._appendTemplate(template)
# named 26-Jul-2007 15:20:52.252
template = DateStrptime()
template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]")
- template.setRegex("(?<!\w)\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d-%b-%Y %H:%M:%S")
self._appendTemplate(template)
# 17-07-2008 17:23:25
template = DateStrptime()
template.setName("Day-Month-Year Hour:Minute:Second")
- template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%d-%m-%Y %H:%M:%S")
self._appendTemplate(template)
# 01-27-2012 16:22:44.252
template = DateStrptime()
template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]")
- template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+ template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
template.setPattern("%m-%d-%Y %H:%M:%S")
self._appendTemplate(template)
# TAI64N
diff --git a/server/datetemplate.py b/server/datetemplate.py
index f663862e..a25a42ea 100644
--- a/server/datetemplate.py
+++ b/server/datetemplate.py
@@ -50,8 +50,11 @@ class DateTemplate:
def getName(self):
return self.__name
- def setRegex(self, regex):
- self.__regex = regex.strip()
+ def setRegex(self, regex, wordBegin = True):
+ regex = regex.strip()
+ if (wordBegin and not re.search(r'^\^', regex)):
+ regex = r'\b' + regex
+ self.__regex = regex
self.__cRegex = re.compile(regex)
def getRegex(self):