fixes unexpected extra regex-space in generic `__prefix_line` (gh-1405, misleadingly committed in d2a9537568);

all optional spaces normalized in generic include `common.conf` + test cases are extended (using new example pseudo-filter and test log `zzz-generic-example`);

ChangeLog

@@ -18,6 +18,9 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
    * fixed a grave bug within tags substitutions because of incorrect detection of recursion 
      in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). 
      Now tracks the actual list of the already substituted tags (per tag instead of single list)
+   * filter.d/common.conf
+     - unexpected extra regex-space in generic `__prefix_line` (gh-1405)
+     - all optional spaces normalized in `common.conf`, test covered now
 
 - New Features:
    * New Actions:

config/filter.d/common.conf

@@ -26,11 +26,11 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
 
 # extra daemon info
 # EXAMPLE: [ID 800047 auth.info]
-__daemon_extra_re = (?:\[ID \d+ \S+\])
+__daemon_extra_re = \[ID \d+ \S+\]
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
+__daemon_combs_re = %(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
@@ -44,14 +44,14 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2}
 
 # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or
 # <auth.info> appearing before the host as per testcases/files/logs/bsd/*.
-__bsd_syslog_verbose = (<[^.]+\.[^.]+>)
+__bsd_syslog_verbose = <[^.]+\.[^.]+>
 
 # Common line prefixes (beginnings) which could be used in filters
 #
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s*
+__prefix_line = \s*(?:(?:%(__bsd_syslog_verbose)s)\s*)?(?:(?:%(__hostname)s)\s*)?(?:(?:%(__kernel_prefix)s)\s*)?(?:(?:@vserver_\S+)\s*)?(?:(?:%(__daemon_combs_re)s)\s*)?(?:(?:%(__daemon_extra_re)s)\s*)?
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap

config/filter.d/zzz-generic-example.conf

@@ -0,0 +1,17 @@
+# Fail2Ban generic example resp. test filter 
+#
+# Author: Serg G. Brester (sebres)
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+[Definition]
+
+_daemon = test-demo
+
+failregex = ^%(__prefix_line)sF2B: failure from <HOST>$
+ignoreregex = 

fail2ban/tests/clientreadertestcase.py

@@ -600,9 +600,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertTrue(jails.read())             # opens fine
 			self.assertTrue(jails.getOptions())       # reads fine
 			# grab all filter names
-			filters = set(os.path.splitext(os.path.split(a)[1])[0]
+			filters = (os.path.splitext(os.path.split(flt)[1])[0]
-				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+				for flt in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
-					if not a.endswith('common.conf'))
+					if not flt.endswith('common.conf'))
+			filters = set(filter(lambda flt: not flt.startswith('zzz-'), filters))
 			# get filters of all jails (filter names without options inside filter[...])
 			filters_jail = set(
 				JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails

fail2ban/tests/files/logs/zzz-generic-example

@@ -0,0 +1,28 @@
+# -- _daemon with __pid_re, without __hostname --
+# failJSON: { "time": "2005-06-21T16:47:46", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:46 machine test-demo[13709]: F2B: failure from 192.0.2.1
+# -- _daemon with __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.0.2.1" }
+Jun 21 16:47:48 test-demo[13709]: F2B: failure from 192.0.2.1
+
+# -- __kernel_prefix --
+# failJSON: { "time": "2005-06-21T16:47:50", "match": true , "host": "192.0.2.2" }
+Jun 21 16:47:50 machine kernel: [  970.699396] F2B: failure from 192.0.2.2
+
+# -- _daemon_re with and without __pid_re --
+# failJSON: { "time": "2005-06-21T16:47:52", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:52 machine [test-demo] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:47:53", "match": true , "host": "192.0.2.3" }
+Jun 21 16:47:53 machine [test-demo][13709] F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:00", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:00 machine test-demo(pam_unix) F2B: failure from 192.0.2.3
+# failJSON: { "time": "2005-06-21T16:50:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:50:02 machine test-demo(pam_unix)[13709] F2B: failure from 192.0.2.3
+
+
+# -- all common definitions together (bsdverbose hostname kernel_prefix vserver tag daemon_id space) --
+# failJSON: { "time": "2005-06-21T16:55:01", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:01 <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3
+# -- the same as above with additional spaces around --
+# failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" }
+Jun 21 16:55:02   <auth.info> machine kernel: [  970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test]    F2B: failure from 192.0.2.3