From d17b415371d14f00c9b24fc3672733b6666fe345 Mon Sep 17 00:00:00 2001
From: sebres <info@sebres.de>
Date: Tue, 5 Mar 2013 00:02:39 +0100
Subject: [PATCH 1/2] invalid date recognition, irregular because of sorting
 template list (sometimes not reproducible by fail2ban-regex, cause will be
 not sorted)

date in following log line (from nginx) will be wrong detected:
2012/10/11 02:37:17 [error] 18434#0: *947 user "test" was not found in "/www/...", client: 192.168.1.5, ...

sometimes it is [correct] - 2012/10/11 (%Y/%m/%d) = 1349919861.71
sometimes it is [invalid] -   12/10/11 (%d/%m/%y) = 1349915838.06
and older as now - 1800 seconds (therefore will be not found)

solution: regular expression fixed, cause date in log used always after non symbol (\W) character, so r"\d{2}/\d{2}/\d{2}" will be r"(?<!\w)\d{2}/\d{2}/\d{2}".
---
 server/datedetector.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/server/datedetector.py b/server/datedetector.py
index c013d551..6e484449 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -55,80 +55,80 @@ class DateDetector:
 			# standard
 			template = DateStrptime()
 			template.setName("MONTH Day Hour:Minute:Second")
-			template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%b %d %H:%M:%S")
 			self._appendTemplate(template)
 			# asctime
 			template = DateStrptime()
 			template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year")
-			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
+			template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
 			template.setPattern("%a %b %d %H:%M:%S %Y")
 			self._appendTemplate(template)
 			# asctime without year
 			template = DateStrptime()
 			template.setName("WEEKDAY MONTH Day Hour:Minute:Second")
-			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%a %b %d %H:%M:%S")
 			self._appendTemplate(template)
 			# simple date
 			template = DateStrptime()
 			template.setName("Year/Month/Day Hour:Minute:Second")
-			template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y/%m/%d %H:%M:%S")
 			self._appendTemplate(template)
 			# simple date too (from x11vnc)
 			template = DateStrptime()
 			template.setName("Day/Month/Year Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%m/%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# previous one but with year given by 2 digits
 			# (See http://bugs.debian.org/537610)
 			template = DateStrptime()
 			template.setName("Day/Month/Year2 Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%m/%y %H:%M:%S")
 			self._appendTemplate(template)
 			# Apache format [31/Oct/2006:09:22:55 -0000]
 			template = DateStrptime()
 			template.setName("Day/MONTH/Year:Hour:Minute:Second")
-			template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%b/%Y:%H:%M:%S")
 			self._appendTemplate(template)
 			# CPanel 05/20/2008:01:57:39
 			template = DateStrptime()
 			template.setName("Month/Day/Year:Hour:Minute:Second")
-			template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
 			template.setPattern("%m/%d/%Y:%H:%M:%S")
 			self._appendTemplate(template)
 			# Exim 2006-12-21 06:43:20
 			template = DateStrptime()
 			template.setName("Year-Month-Day Hour:Minute:Second")
-			template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y-%m-%d %H:%M:%S")
 			self._appendTemplate(template)
 			# custom for syslog-ng 2006.12.21 06:43:20
 			template = DateStrptime()
 			template.setName("Year.Month.Day Hour:Minute:Second")
-			template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y.%m.%d %H:%M:%S")
 			self._appendTemplate(template)
 			# named 26-Jul-2007 15:20:52.252 
 			template = DateStrptime()
 			template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d-%b-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# 17-07-2008 17:23:25
 			template = DateStrptime()
 			template.setName("Day-Month-Year Hour:Minute:Second")
-			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d-%m-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# 01-27-2012 16:22:44.252
 			template = DateStrptime()
 			template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%m-%d-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# TAI64N

From b6bb2f88c1dbb111647269590d80d95f72c81c3e Mon Sep 17 00:00:00 2001
From: sebres <sergey.brester@W7-DEHBG0189.wincor-nixdorf.com>
Date: Mon, 11 Mar 2013 13:52:31 +0100
Subject: [PATCH 2/2] invalid date recognition, irregular because of sorting
 template list, now via setRegex

---
 server/datedetector.py | 26 +++++++++++++-------------
 server/datetemplate.py |  7 +++++--
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/server/datedetector.py b/server/datedetector.py
index 6e484449..c013d551 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -55,80 +55,80 @@ class DateDetector:
 			# standard
 			template = DateStrptime()
 			template.setName("MONTH Day Hour:Minute:Second")
-			template.setRegex("(?<!\w)\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%b %d %H:%M:%S")
 			self._appendTemplate(template)
 			# asctime
 			template = DateStrptime()
 			template.setName("WEEKDAY MONTH Day Hour:Minute:Second Year")
-			template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
+			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}")
 			template.setPattern("%a %b %d %H:%M:%S %Y")
 			self._appendTemplate(template)
 			# asctime without year
 			template = DateStrptime()
 			template.setName("WEEKDAY MONTH Day Hour:Minute:Second")
-			template.setRegex("(?<!\w)\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%a %b %d %H:%M:%S")
 			self._appendTemplate(template)
 			# simple date
 			template = DateStrptime()
 			template.setName("Year/Month/Day Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y/%m/%d %H:%M:%S")
 			self._appendTemplate(template)
 			# simple date too (from x11vnc)
 			template = DateStrptime()
 			template.setName("Day/Month/Year Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%m/%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# previous one but with year given by 2 digits
 			# (See http://bugs.debian.org/537610)
 			template = DateStrptime()
 			template.setName("Day/Month/Year2 Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%m/%y %H:%M:%S")
 			self._appendTemplate(template)
 			# Apache format [31/Oct/2006:09:22:55 -0000]
 			template = DateStrptime()
 			template.setName("Day/MONTH/Year:Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}")
 			template.setPattern("%d/%b/%Y:%H:%M:%S")
 			self._appendTemplate(template)
 			# CPanel 05/20/2008:01:57:39
 			template = DateStrptime()
 			template.setName("Month/Day/Year:Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}/\d{2}/\d{4}:\d{2}:\d{2}:\d{2}")
 			template.setPattern("%m/%d/%Y:%H:%M:%S")
 			self._appendTemplate(template)
 			# Exim 2006-12-21 06:43:20
 			template = DateStrptime()
 			template.setName("Year-Month-Day Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y-%m-%d %H:%M:%S")
 			self._appendTemplate(template)
 			# custom for syslog-ng 2006.12.21 06:43:20
 			template = DateStrptime()
 			template.setName("Year.Month.Day Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{4}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%Y.%m.%d %H:%M:%S")
 			self._appendTemplate(template)
 			# named 26-Jul-2007 15:20:52.252 
 			template = DateStrptime()
 			template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("(?<!\w)\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d-%b-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# 17-07-2008 17:23:25
 			template = DateStrptime()
 			template.setName("Day-Month-Year Hour:Minute:Second")
-			template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%d-%m-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# 01-27-2012 16:22:44.252
 			template = DateStrptime()
 			template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]")
-			template.setRegex("(?<!\w)\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
+			template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}")
 			template.setPattern("%m-%d-%Y %H:%M:%S")
 			self._appendTemplate(template)
 			# TAI64N
diff --git a/server/datetemplate.py b/server/datetemplate.py
index f663862e..a25a42ea 100644
--- a/server/datetemplate.py
+++ b/server/datetemplate.py
@@ -50,8 +50,11 @@ class DateTemplate:
 	def getName(self):
 		return self.__name
 	
-	def setRegex(self, regex):
-		self.__regex = regex.strip()
+	def setRegex(self, regex, wordBegin = True):
+		regex = regex.strip()
+		if (wordBegin and not re.search(r'^\^', regex)):
+			regex = r'\b' + regex
+		self.__regex = regex
 		self.__cRegex = re.compile(regex)
 		
 	def getRegex(self):