Merge pull request #1421 from sebres/fix-1405

filter.d/common.conf: fixes unexpected extra regex-space in generic `__prefix_line` (gh-1405)

ChangeLog

@@ -18,6 +18,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * filter.d/common.conf
+     - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
+     - all optional spaces normalized in `common.conf`, test covered now
+     - generic `__prefix_line` extended with optional brackets for the date ambit (gh-1421),
+       added new parameter `__date_ambit`
    * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, 
      not argument of fail2ban (see gh-1434)
 

config/filter.d/asterisk.conf

@@ -16,17 +16,17 @@ __pid_re = (?:\[\d+\])
 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 
 # All Asterisk log messages begin like this:
-log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)?
+log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*( in \w+:)?
 
-failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
+failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
+            ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^%(__prefix_line)s%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
+            ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@<HOST>\S*$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s hacking attempt detected '<HOST>'$
+            ^%(__prefix_line)s%(log_prefix)s hacking attempt detected '<HOST>'$
-            ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
+            ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|<unknown>)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$
-            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )Ext\. s: "Rejecting unknown SIP connection from <HOST>"$
+            ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from <HOST>"$
 
 ignoreregex =
 

config/filter.d/common.conf

@@ -26,7 +26,7 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
 
 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
-__daemon_extra_re = (?:\[ID \d+ \S+\])
+__daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
@@ -44,14 +44,18 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 
 # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
-__bsd_syslog_verbose = (<[^.]+\.[^.]+>)
+__bsd_syslog_verbose = <[^.]+\.[^.]+>
+
+__vserver = @vserver_\S+
+
+__date_ambit = (?:\[\])
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s*
+__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap

config/filter.d/nsd.conf

@@ -22,7 +22,7 @@ _daemon = nsd
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 
-failregex =  ^\[\]%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
+failregex =  ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
-              ^\[\]%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
+             ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$
 
 ignoreregex =

fail2ban/tests/config/filter.d/zzz-generic-example.conf

@@ -0,0 +1,17 @@
+# Fail2Ban generic example resp. test filter 
+#
+# Author: Serg G. Brester (sebres)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = ../../../../config/filter.d/common.conf
+
+[Definition]
+
+_daemon = test-demo
+
+failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
+ignoreregex = 

fail2ban/tests/files/logs/zzz-generic-example

@@ -0,0 +1,31 @@
+# -- _daemon with __pid_re, without __hostname --
+# failJSON: { "time": "2005-06-21T16:47:46", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:46 machine test-demo[13709]: F2B: failure from 192.0.2.1
+# -- _daemon with __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:48 test-demo[13709]: F2B: failure from 192.0.2.1
+
+# -- __kernel_prefix --
+# failJSON: { "time": "2005-06-21T16:47:50", "match": true , "host": "192.0.2.2" }
+Jun 21 16:47:50 machine kernel: [  970.699396] F2B: failure from 192.0.2.2
+
+# -- _daemon_re with and without __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:52", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:52 machine [test-demo] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:47:53", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:53 machine [test-demo][13709] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:00", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:00 machine test-demo(pam_unix) F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:02 machine test-demo(pam_unix)[13709] F2B: failure from 192.0.2.3
+
+
+# -- all common definitions together (bsdverbose hostname kernel_prefix vserver tag daemon_id space) --
+# failJSON: { "time": "2005-06-21T16:55:01", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
+# -- the same as above with additional spaces around --
+# failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3
+# -- the same as above with brackets as date ambit --
+# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" }
+[Jun 21 16:55:03] <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3

fail2ban/tests/samplestestcase.py

@@ -35,6 +35,7 @@ from ..server.filter import Filter
 from ..client.filterreader import FilterReader
 from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR
 
+TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config")
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 
 
@@ -60,11 +61,11 @@ class FilterSamplesRegex(unittest.TestCase):
 			"Expected more FilterSampleRegexs tests")
 
 
-def testSampleRegexsFactory(name):
+def testSampleRegexsFactory(name, basedir):
 	def testFilter(self):
 
 		# Check filter exists
-		filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR)
+		filterConf = FilterReader(name, "jail", {}, basedir=basedir)
 		self.assertEqual(filterConf.getFile(), name)
 		self.assertEqual(filterConf.getJailName(), "jail")
 		filterConf.read()
@@ -147,11 +148,15 @@ def testSampleRegexsFactory(name):
 
 	return testFilter
 
-for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'),
+for basedir_, filter_ in (
-					  os.listdir(os.path.join(CONFIG_DIR, "filter.d"))):
+	(CONFIG_DIR, lambda x: not x.endswith('common.conf') and x.endswith('.conf')),
+	(TEST_CONFIG_DIR, lambda x: x.startswith('zzz-') and x.endswith('.conf')),
+):
+	for filter_ in filter(filter_,
+						  os.listdir(os.path.join(basedir_, "filter.d"))):
 		filterName = filter_.rpartition(".")[0]
 		if not filterName.startswith('.'):
 			setattr(
 				FilterSamplesRegex,
 				"testSampleRegexs%s" % filterName.upper(),
-			testSampleRegexsFactory(filterName))
+				testSampleRegexsFactory(filterName, basedir_))