merge of change log and apache-auth differences

2013-08-25 16:51:35 +10:00 · 2013-08-25 16:51:35 +10:00 · a9eb8a76c6
parent aad7d08451 4e5feed7fc
commit a9eb8a76c6
85 changed files with 1189 additions and 490 deletions
--- a/49
+++ b/49
@ -7,24 +7,69 @@
 Fail2Ban (version 0.8.10)                                             2013/06/12
 ================================================================================

-ver. 0.8.11 (2013/XX/XXX) - wanna-be-released
+ver. 0.8.11 (2013/XX/XXX) - loves-unittests
 -----------

 - Fixes:
  Daniel Black & Marcel Dopita
   * filter.d/apache-auth -- fixed and apache auth samples provide. closes #286
+  Yaroslav Halchenko
+   * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267
+   * filter.d/apache-common.conf -- support apache 2.4 more detailed error
+     log format.  Closes gh-268
+   * Backends changes detection and parsing. Close gh-223 and gh-103:
+     - Polling backend: detect changes in the files not only based on
+       mtime, but also on the size and inode.  It should allow for
+       better detection of changes and log rotations on busy servers,
+       older python 2.4, and file systems with precision of mtime only
+       up to a second (e.g. ext3).
+     - All backends, possible race condition: do not read from a file
+       initially reported empty.  Originally could have lead to
+       accounting for detected log lines multiple times.
+  Daniel Black & Мернов Георгий
+   * filter.d/dovecot.conf -- Fix when no TLS enabled - line doesn't end in ,
+  Daniel Black
+  * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across
+    all platforms to ensure permissions are the same before and after a ban -
+    closes gh-266
 - New Features:
  Daniel Black & ykimon
   * filter.d/3proxy.conf -- filter added
+  Daniel Black
+   * filter.d/exim-spam.conf -- a splitout of exim's spam regexes
+     with additions for greater control over filtering spam.
 - Enhancements:
  Daniel Black
   * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening
     and extra failure examples in sample logs
 	 * filter.d/apache-auth - added expressions for mod_authz, mod_auth and
     mod_auth_digest failures.
-  Daniel Black & Georgiy Mernov
+  Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий
   * filter.d/exim.conf -- regex hardening and extra failure examples in
     sample logs
+  Daniel Black & Sebastian Arcus
+   * filter.d/asterisk -- more regexes
+  Yaroslav Halchenko
+   * fail2ban-regex -- refactored to provide more details (missing and
+     ignored lines, control over logging, etc) while maintaining look&feel
+   * fail2ban-client -- log to standard error. Closes gh-264
+   * Fail to configure if not a single log file was found for an
+     enabled jail. Closes gh-63
+   * <HOST> is now enforced to end with an alphanumeric
+   * filter.d/roundcube-auth.conf -- anchored version
+  Alexander Dietrich
+   * action.d/sendmail-common.conf -- added common sendmail settings file
+     and made the sender display name configurable
+  Steven Hiscocks
+   * filter.d/dovecot - Addition of session, time values and possible blank
+     user
+   * filter.d/{courier{login,smtp},proftpd,sieve,wuftpd,xinetd} - General
+     regex impovements
+   * IMPORTANT: 'lighttpd-fastcgi' filter has been renamed to 'suhosin', which
+     will require changing in jail.{conf,local} if using this filter.
+  Zurd
+   * filter.d/postfix - add filter for VRFY failures. closes gh-322.
+

 ver. 0.8.10 (2013/06/12) - wanna-be-secure
 -----------
--- a/9
+++ b/9
@ -39,8 +39,9 @@ Filters

 * Include sample logs with 1.2.3.4 used for IP addresses and 
  example.com/example.org used for DNS names
-* Ensure ./fail2ban-regex testcases/files/logs/{samplelog} config/filter.d/{filter}.conf
-  has matches for EVERY regex
+* Ensure sample log is provided in testcases/files/logs/ with same name as the
+  filter. Each log line should include match meta data for time & IP above
+  every line (see other sample log files for examples)
 * Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if
  \d+ is sufficient
 * Use the functionality of regexs http://docs.python.org/2/library/re.html
@ -252,6 +253,10 @@ Use the text "closes #333"/"resolves #333 "/"fixes #333" where 333 represents
 an issue that is closed. Other text and details in link below.
 See: https://help.github.com/articles/closing-issues-via-commit-messages

+If merge resulted in conflicts, clarify what changes were done to
+corresponding files in the 'Conflicts:' section of the merge commit
+message.  See e.g. https://github.com/fail2ban/fail2ban/commit/f5a8a8ac
+
 Adding Actions
 --------------

--- a/2
+++ b/2
@ -101,7 +101,7 @@ config/filter.d/couriersmtp.conf
 config/filter.d/cyrus-imap.conf
 config/filter.d/exim.conf
 config/filter.d/gssftpd.conf
-config/filter.d/lighttpd-fastcgi.conf
+config/filter.d/suhosin.conf
 config/filter.d/named-refused.conf
 config/filter.d/postfix.conf
 config/filter.d/proftpd.conf
--- a/README.Solaris
+++ b/README.Solaris
@ -69,27 +69,10 @@ FAIL2BAN CONFIGURATION

 OPT: Create /etc/fail2ban/fail2ban.local containing:

-# Fail2Ban main configuration file
-#
-# Comments: use '#' for comment lines and ';' (following a space) for inline comments
-#
-# Changes:  in most of the cases you should not modify this
-#           file, but provide customizations in fail2ban.local file, e.g.:
-#
-# [Definition]
-# loglevel = 4
+# Fail2Ban configuration file for logging fail2ban on Solaris
 #
 [Definition]

-# Option:  logtarget
-# Notes.:  Set the log target. This could be a file, SYSLOG, STDERR or STDOUT.
-#          Only one log target can be specified.
-#          If you change logtarget from the default value and you are
-#          using logrotate -- also adjust or disable rotation in the
-#          corresponding configuration file
-#          (e.g. /etc/logrotate.d/fail2ban on Debian systems)
-# Values:  STDOUT STDERR SYSLOG file  Default:  /var/log/fail2ban.log
-#
 logtarget = /var/adm/fail2ban.log


@ -105,7 +88,7 @@ ignoreregex = for myuser from
 logpath     = /var/adm/auth.log

 Set the sendmail dest address to something useful or drop the line to stop it spamming you.
-Set 'myuser' to your username to avoid banning yourself or drop it.
+Set 'myuser' to your username to avoid banning yourself or remove the line.

 START (OR RESTART) FAIL2BAN

@ -128,7 +111,7 @@ GOTCHAS AND FIXMES
    svcadm enable fail2ban

 * If svcs -xv says that fail2ban failed to start or svcs says it's in maintenance mode
-  chcek /var/svc/log/network-fail2ban:default.log for clues. 
+  check /var/svc/log/network-fail2ban:default.log for clues. 
  Check permissions on /var/adm, /var/adm/auth.log /var/adm/fail2ban.log and /var/run/fail2ban
  You may need to:

--- a/3
+++ b/3
@ -18,6 +18,7 @@ Daniel Black
 David Nutter
 Eric Gerbier
 Enrico Labedzki
+ftoppi
 Georgiy Mernov
 Guillaume Delvit
 Hanno 'Rince' Wagner
@ -33,6 +34,7 @@ Mark Edgington
 Markus Hoffmann
 Marvin Rouge
 mEDI
+Мернов Георгий
 Michael C. Haller
 Michael Hanselmann
 NickMunger
@ -41,6 +43,7 @@ Raphaël Marichez
 René Berber
 Robert Edeker
 Russell Odom
+Sebastian Arcus
 Sireyessire
 silviogarbes
 Stephen Gildea
--- a/client/jailreader.py
+++ b/client/jailreader.py
@ -103,16 +103,30 @@ class JailReader(ConfigReader):
 				logSys.warn("No actions were defined for %s" % self.__name)
 		return True
 	
-	def convert(self):
+	def convert(self, allow_no_files=False):
+		"""Convert read before __opts to the commands stream
+
+		Parameters
+		----------
+		allow_missing : bool
+		  Either to allow log files to be missing entirely.  Primarily is
+		  used for testing
+		 """
+
 		stream = []
 		for opt in self.__opts:
 			if opt == "logpath":
+				found_files = 0
 				for path in self.__opts[opt].split("\n"):
 					pathList = glob.glob(path)
 					if len(pathList) == 0:
-						logSys.error("No file found for " + path)
+						logSys.error("No file(s) found for glob %s" % path)
 					for p in pathList:
+						found_files += 1
 						stream.append(["set", self.__name, "addlogpath", p])
+				if not (found_files or allow_no_files):
+					raise ValueError(
+						"Have not found any log file for %s jail" % self.__name)
 			elif opt == "backend":
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
--- a/client/jailsreader.py
+++ b/client/jailsreader.py
@ -79,14 +79,23 @@ class JailsReader(ConfigReader):
 					return False
 		return True
 	
-	def convert(self):
+	def convert(self, allow_no_files=False):
+		"""Convert read before __opts and jails to the commands stream
+
+		Parameters
+		----------
+		allow_missing : bool
+		  Either to allow log files to be missing entirely.  Primarily is
+		  used for testing
+		"""
+
 		stream = list()
 		for opt in self.__opts:
 			if opt == "":
 				stream.append([])
 		# Convert jails
 		for jail in self.__jails:
-			stream.extend(jail.convert())
+			stream.extend(jail.convert(allow_no_files=allow_no_files))
 		# Start jails
 		for jail in self.__jails:
 			stream.append(["start", jail.getName()])
--- a/common/init.py
+++ b/common/init.py
@ -23,3 +23,8 @@
 __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
+
+import logging
+
+# Custom debug level
+logging.HEAVYDEBUG = 5
--- a/config/action.d/hostsdeny.conf
+++ b/config/action.d/hostsdeny.conf
@ -39,7 +39,7 @@ actionban = IP=<ip> &&
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = IP=<ip> && sed  /ALL:\ $IP/d <file> > <file>.new && mv <file>.new <file>
+actionunban = echo "/ALL: <ip>$/<br>d<br>w<br>q" | ed <file>

 [Init]

--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@ -4,6 +4,10 @@
 #
 #

+[INCLUDES]
+
+before = sendmail-common.conf
+
 [Definition]

 # Option:  actionstart
@ -11,7 +15,7 @@
 # Values:  CMD
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
-              From: Fail2Ban <<sender>>
+              From: <sendername> <<sender>>
              To: <dest>\n
              Hi,\n
              The jail <name> has been started successfully.\n
@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
 #
 actionstop = if [ -f <tmpfile> ]; then
                 printf %%b "Subject: [Fail2Ban] <name>: summary
-                 From: Fail2Ban <<sender>>
+                 From: <sendername> <<sender>>
                 To: <dest>\n
                 Hi,\n
                 These hosts have been banned by Fail2Ban.\n
@ -58,7 +62,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
            LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
            if [ $LINE -ge <lines> ]; then
                printf %%b "Subject: [Fail2Ban] <name>: summary
-                From: Fail2Ban <<sender>>
+                From: <sendername> <<sender>>
                To: <dest>\n
                Hi,\n
                These hosts have been banned by Fail2Ban.\n
@ -82,14 +86,6 @@ actionunban =
 #
 name = default

-# Destination/Addressee of the mail
-#
-dest = root
-
-# Sender of the mail
-#
-sender = fail2ban
-
 # Default number of lines that are buffered
 #
 lines = 5
--- a/config/action.d/sendmail-common.conf
+++ b/config/action.d/sendmail-common.conf
@ -0,0 +1,23 @@
+# Fail2Ban configuration file
+#
+# Common settings for sendmail actions
+#
+# Users can override the defaults in sendmail-common.local
+
+[INCLUDES]
+
+after = sendmail-common.local
+
+[Init]
+
+# Recipient mail address
+#
+dest = root
+
+# Sender mail address
+#
+sender = fail2ban
+
+# Sender display name
+#
+sendername = Fail2Ban
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@ -4,6 +4,10 @@
 #
 #

+[INCLUDES]
+
+before = sendmail-common.conf
+
 [Definition]

 # Option:  actionstart
@ -12,7 +16,7 @@
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: Fail2Ban <<sender>>
+              From: <sendername> <<sender>>
              To: <dest>\n
              Hi,\n
              The jail <name> has been started successfully.\n
@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
 #
 actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped
             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: Fail2Ban <<sender>>
+             From: <sendername> <<sender>>
             To: <dest>\n
             Hi,\n
             The jail <name> has been stopped.\n
@ -46,7 +50,7 @@ actioncheck =
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-            From: Fail2Ban <<sender>>
+            From: <sendername> <<sender>>
            To: <dest>\n
            Hi,\n
            The IP <ip> has just been banned by Fail2Ban after
@ -72,14 +76,6 @@ actionunban =
 #
 name = default

-# Destination/Addressee of the mail
-#
-dest = root
-
-# Sender of the mail
-#
-sender = fail2ban
-
 # Path to the log files which contain relevant lines for the abuser IP
 #
 logpath = /dev/null
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@ -4,6 +4,10 @@
 #
 #

+[INCLUDES]
+
+before = sendmail-common.conf
+
 [Definition]

 # Option:  actionstart
@ -12,7 +16,7 @@
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: Fail2Ban <<sender>>
+              From: <sendername> <<sender>>
              To: <dest>\n
              Hi,\n
              The jail <name> has been started successfully.\n
@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
 #
 actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped
             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: Fail2Ban <<sender>>
+             From: <sendername> <<sender>>
             To: <dest>\n
             Hi,\n
             The jail <name> has been stopped.\n
@ -46,7 +50,7 @@ actioncheck =
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-            From: Fail2Ban <<sender>>
+            From: <sendername> <<sender>>
            To: <dest>\n
            Hi,\n
            The IP <ip> has just been banned by Fail2Ban after
@ -70,11 +74,3 @@ actionunban =
 #
 name = default

-# Destination/Addressee of the mail
-#
-dest = root
-
-# Sender of the mail
-#
-sender = fail2ban
-
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@ -4,6 +4,10 @@
 #
 #

+[INCLUDES]
+
+before = sendmail-common.conf
+
 [Definition]

 # Option:  actionstart
@ -12,7 +16,7 @@
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
              Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-              From: Fail2Ban <<sender>>
+              From: <sendername> <<sender>>
              To: <dest>\n
              Hi,\n
              The jail <name> has been started successfully.\n
@ -25,7 +29,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started
 #
 actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped
             Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-             From: Fail2Ban <<sender>>
+             From: <sendername> <<sender>>
             To: <dest>\n
             Hi,\n
             The jail <name> has been stopped.\n
@ -46,7 +50,7 @@ actioncheck =
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
            Date: `LC_TIME=C date -u +"%%a, %%d %%h %%Y %%T +0000"`
-            From: Fail2Ban <<sender>>
+            From: <sendername> <<sender>>
            To: <dest>\n
            Hi,\n
            The IP <ip> has just been banned by Fail2Ban after
@ -68,11 +72,3 @@ actionunban =
 #
 name = default

-# Destination/Addressee of the mail
-#
-dest = root
-
-# Sender of the mail
-#
-sender = fail2ban
-
--- a/config/filter.d/apache-common.conf
+++ b/config/filter.d/apache-common.conf
@ -14,4 +14,8 @@ after = apache-common.local
 [DEFAULT]

 # Common prefix for [error] apache messages which also would include <HOST>
-_apache_error_client = \[[^]]+\] \[(\w+:)?error\] (\[pid \d+:tid \d+\] )?\[client <HOST>(:\d+)?\]
+# Depending on the version it could be
+# 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]
+# 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652]
+# Reference: https://github.com/fail2ban/fail2ban/issues/268
+_apache_error_client = \[[^]]+\] \[(error|core:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]( \S+:)?
--- a/config/filter.d/asterisk.conf
+++ b/config/filter.d/asterisk.conf
@ -17,19 +17,22 @@ before = common.conf
 # Notes.:  regex to match the password failures messages in the logfile.
 # Values:  TEXT
 #
-failregex = ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - Wrong password$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - No matching peer found$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - Username/auth name mismatch$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - Device does not match ACL$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - Peer is not supposed to register$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - ACL error \(permit/deny\)$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '<HOST>(:\d+)?' - Not a local domain$
-            ^\[\]\s*NOTICE%(__pid_re)s\[\S+\] \S+: Call from '[^']*' \(<HOST>:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Host <HOST> failed to authenticate as '[^']*'$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: No registration for peer '[^']*' \(from <HOST>\)$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
-            ^\[\]\s*NOTICE%(__pid_re)s \S+: Failed to authenticate user [^@]+@<HOST>\S*$
-            ^\[\]\s*SECURITY%(__pid_re)s \S+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P/<HOST>/\d+"$
+log_prefix= \[\]\s*(?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[\S+\d*\])? \S+:\d*
+
+failregex = ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - Wrong password$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - No matching peer found$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - Username/auth name mismatch$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - Device does not match ACL$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - Peer is not supposed to register$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - ACL error \(permit/deny\)$
+            ^%(log_prefix)s Registration from '[^']*' failed for '<HOST>(:\d+)?' - Not a local domain$
+            ^%(log_prefix)s Call from '[^']*' \(<HOST>:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$
+            ^%(log_prefix)s Host <HOST> failed to authenticate as '[^']*'$
+            ^%(log_prefix)s No registration for peer '[^']*' \(from <HOST>\)$
+            ^%(log_prefix)s Host <HOST> failed MD5 authentication for '[^']*' \([^)]+\)$
+            ^%(log_prefix)s Failed to authenticate user [^@]+@<HOST>\S*$
+            ^%(log_prefix)s (?:handle_request_subscribe: )?Sending fake auth rejection for (device|user) \d*<sip:[^@]+@<HOST>>;tag=\w+\S*$
+            ^%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P/<HOST>/\d+"(,Challenge="\w+",ReceivedChallenge="\w+")?(,ReceivedHash="[\da-f]+")?$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/common.conf
+++ b/config/filter.d/common.conf
@ -33,7 +33,7 @@ __daemon_extra_re = (?:\[ID \d+ \S+\])

 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:)
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)

 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]
--- a/config/filter.d/courierlogin.conf
+++ b/config/filter.d/courierlogin.conf
@ -5,8 +5,17 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)?
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -14,7 +23,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = LOGIN FAILED, .*, ip=\[<HOST>\]$
+failregex = ^%(__prefix_line)sLOGIN FAILED, user=.*, ip=\[<HOST>\]$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/couriersmtp.conf
+++ b/config/filter.d/couriersmtp.conf
@ -4,8 +4,17 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_daemon = courieresmtpd
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -13,7 +22,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = error,relay=<HOST>,.*550 User unknown
+failregex = ^%(__prefix_line)serror,relay=<HOST>,.*: 550 User unknown\.$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/cyrus-imap.conf
+++ b/config/filter.d/cyrus-imap.conf
@ -4,8 +4,17 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_daemon = (?:cyrus/)?(?:imapd?|pop3d?)
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -13,10 +22,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = : badlogin: .*\[<HOST>\] plaintext .*SASL\(-13\): authentication failure: checkpass failed$
-	    : badlogin: .*\[<HOST>\] LOGIN \[SASL\(-13\): authentication failure: checkpass failed\]$
-	    : badlogin: .*\[<HOST>\] (?:CRAM-MD5|NTLM) \[SASL\(-13\): authentication failure: incorrect (?:digest|NTLM) response\]$
-	    : badlogin: .*\[<HOST>\] DIGEST-MD5 \[SASL\(-13\): authentication failure: client response doesn't match what we generated\]$
+failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[<HOST>\] \S+ .*?\[?SASL\(-13\): authentication failure: .*\]?$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@ -16,8 +16,8 @@ _daemon = dovecot(-auth)?
 #          first regex is essentially a copy of pam-generic.conf
 # Values:  TEXT
 #
-failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(\s+user=\S*)?\s*$
-            ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=<HOST>, lip=(\d{1,3}\.){3}\d{1,3},( TLS( handshaking)?(: Disconnected)?)?\s*$
+failregex = ^%(__prefix_line)s(pam_unix(\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(\s+user=\S*)?\s*$
+            ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use disabled \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=<HOST>, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/exim-common.conf
+++ b/config/filter.d/exim-common.conf
@ -0,0 +1,17 @@
+# Fail2Ban configuration file for exim
+#
+# Author:  Daniel Black
+#
+
+[INCLUDES]
+
+# Load customizations if any available
+#
+after = exim-common.local
+
+[Definition]
+
+# From exim source code: ./src/receive.c:add_host_info_for_log
+host_info = H=([\w.-]+ )?(\(\S+\) )?\[<HOST>\](:\d+)? (I=\[\S+\]:\d+ )?(U=\S+ )?(P=e?smtp )?
+pid = ( \[\d+\])?
+
--- a/config/filter.d/exim-spam.conf
+++ b/config/filter.d/exim-spam.conf
@ -0,0 +1,29 @@
+# Fail2Ban configuration file
+#
+# Author: Cyril Jaquier
+#         Daniel Black (rewrote with strong regexs)
+#
+
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# exim-common.local
+before = exim-common.conf
+
+
+[Definition]
+
+# Option:  failregex
+# Notes.:  This includes the spam rejection messages of exim.
+#          Note the %(host_info) defination contains a <HOST> match
+
+failregex =  ^%(pid)s \S+ F=(<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$
+             ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$
+             ^%(pid)s \S+ %(host_info)sF=(<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$
+
+# Option:  ignoreregex
+# Notes.:  regex to ignore. If this regex matches, the line is ignored.
+# Values:  TEXT
+#
+ignoreregex = 
--- a/config/filter.d/exim.conf
+++ b/config/filter.d/exim.conf
@ -1,20 +1,29 @@
 # Fail2Ban configuration file
 #
 # Author: Cyril Jaquier
+#         Daniel Black (rewrote with strong regexs)
 #
-#
+
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# exim-common.local
+before = exim-common.conf
+

 [Definition]

 # Option:  failregex
-# Notes.:  regex to match the password failures messages in the logfile. The
-#          host must be matched by a group named "host". The tag "<HOST>" can
-#          be used for standard IP/hostname matching and is only an alias for
-#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
-# Values:  TEXT
-#
-failregex = ^ H=\S+ \(\S+\) \[<HOST>\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$
-             ^ login authenticator failed for (\S+ )?\(\S+\) \[<HOST>\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+# Notes.:  This includes the rejection messages of exim. For spam and filter
+#          related bans use the exim-spam.conf
+#          Note the %(host_info) defination contains a <HOST> match
+
+failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
+             ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[<HOST>\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+             ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (relay not permitted|Sender verify failed|Unknown user)\s*$
+             ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
+             ^%(pid)s SMTP call from \S+ \[<HOST>\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/postfix.conf
+++ b/config/filter.d/postfix.conf
@ -4,8 +4,17 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_daemon = postfix/smtpd
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -13,8 +22,9 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = reject: RCPT from (.*)\[<HOST>\]: 554
-            reject: RCPT from (.*)\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 554 5\.7\.1 .*$
+            ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
+            ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[<HOST>\]: 550 5\.1\.1 .*$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/proftpd.conf
+++ b/config/filter.d/proftpd.conf
@ -13,6 +13,8 @@ before = common.conf

 [Definition]

+_deamon = proftpd
+
 # Option: failregex
 # Notes.: regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -20,10 +22,12 @@ before = common.conf
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values: TEXT
 #
-failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
-            ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .* \(Login failed\): .*$
-            ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[<HOST>\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$
-            ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[<HOST>\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$
+
+__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).?
+failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$
+            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$
+            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$
+            ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/roundcube-auth.conf
+++ b/config/filter.d/roundcube-auth.conf
@ -4,6 +4,10 @@
 #
 #

+[INCLUDES]
+
+before = common.conf
+
 [Definition]

 # Option:  failregex
@ -13,7 +17,7 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = (FAILED login|Login failed) for .* from <HOST>\s*$
+failregex = ^\s*(\[(\s\+[0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from <HOST>(\. AUTHENTICATE .*)?\s*$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/sieve.conf
+++ b/config/filter.d/sieve.conf
@ -4,15 +4,24 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_deamon = (?:cyrus/)?(?:tim)?sieved?
+
 # Option: failregex
 # Notes.: regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
 #          be used for standard IP/hostname matching.
 # Values: TEXT
 #
-failregex = : badlogin: .*\[<HOST>\] (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failure$
+failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[<HOST>\] \S+ authentication failure$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/lighttpd-fastcgi.conf
+++ b/config/filter.d/lighttpd-fastcgi.conf
@ -9,7 +9,8 @@
 # Notes.:  regex to match ALERTS as notified by lighttpd's FastCGI Module
 # Values:  TEXT
 #
-failregex = .*ALERT\ -\ .*attacker\ \'<HOST>\'
+# https://github.com/stefanesser/suhosin/blob/1fba865ab73cc98a3109f88d85eb82c1bfc29b37/log.c#L161
+failregex = ALERT - .* \(attacker '<HOST>', file '.*'(?:, line \d+)?\)$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/wuftpd.conf
+++ b/config/filter.d/wuftpd.conf
@ -4,14 +4,21 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
 [Definition]

+_daemon = wu-ftpd
+
 # Option: failregex
 # Notes.: regex to match the password failures messages in the logfile.
 # Values: TEXT
 #
-failregex = wu-ftpd(?:\[\d+\])?:\s+\(pam_unix\)\s+authentication failure.* rhost=<HOST>$
-            wu-ftpd(?:\[\d+\])?: *failed login from .*\[<HOST>\] *$
+failregex = ^%(__prefix_line)sfailed login from \S+ \[<HOST>\]\s*$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/filter.d/xinetd-fail.conf
+++ b/config/filter.d/xinetd-fail.conf
@ -4,8 +4,17 @@
 #
 #

+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
 [Definition]

+_daemon = xinetd
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@ -19,8 +28,8 @@
 # load => xinetd: max_load (temporary problem)
 #

-failregex = xinetd(?:\[\d{1,5}\])?: FAIL: \S+ address from=<HOST>$
-            xinetd(?:\[\d{1,5}\])?: FAIL: \S+ libwrap from=<HOST>$
+failregex = ^%(__prefix_line)sFAIL: \S+ address from=<HOST>$
+            ^%(__prefix_line)sFAIL: \S+ libwrap from=<HOST>$

 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
--- a/config/jail.conf
+++ b/config/jail.conf
@ -65,7 +65,7 @@ usedns = warn
 enabled  = false
 filter   = sshd
 action   = iptables[name=SSH, port=ssh, protocol=tcp]
-           sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com]
+           sendmail-whois[name=SSH, dest=you@example.com, sender=fail2ban@example.com, sendername="Fail2Ban"]
 logpath  = /var/log/sshd.log
 maxretry = 5

@ -229,7 +229,7 @@ logpath  = /var/log/apache2/error_log

 enabled  = false
 filter   = roundcube-auth
-action   = iptables[name=RoundCube, port="http,https"]
+action   = iptables-multiport[name=RoundCube, port="http,https"]
 logpath  = /var/log/roundcube/userlogins


@ -241,7 +241,7 @@ enabled  = false
 filter   = sogo-auth
 # without proxy this would be:
 # port    = 20000
-action   = iptables[name=SOGo, port="http,https"]
+action   = iptables-multiport[name=SOGo, port="http,https"]
 logpath  = /var/log/sogo/sogo.log

 # Ban attackers that try to use PHP's URL-fopen() functionality
@ -251,35 +251,25 @@ logpath  = /var/log/sogo/sogo.log
 [php-url-fopen]

 enabled = false
-action  = iptables[name=php-url-open, port="http,https"]
+action  = iptables-multiport[name=php-url-open, port="http,https"]
 filter  = php-url-fopen
 logpath = /var/www/*/logs/access_log
 maxretry = 1

-# A simple PHP-fastcgi jail which works with lighttpd.
-# If you run a lighttpd server, then you probably will
-# find these kinds of messages in your error_log:
-# ALERT – tried to register forbidden variable ‘GLOBALS’
-# through GET variables (attacker '1.2.3.4', file '/var/www/default/htdocs/index.php')
-# This jail would block the IP 1.2.3.4.
-
-[lighttpd-fastcgi]
+[suhosin]

 enabled = false
-filter  = lighttpd-fastcgi
-action  = iptables[name=lighttpd-fastcgi, port="http,https"]
+filter  = suhosin
+action  = iptables-multiport[name=suhosin, port="http,https"]
 # adapt the following two items as needed
 logpath = /var/log/lighttpd/error.log
 maxretry = 2

-# Same as above for mod_auth
-# It catches wrong authentications
-
 [lighttpd-auth]

 enabled = false
 filter  = lighttpd-auth
-action  = iptables[name=lighttpd-auth, port="http,https"]
+action  = iptables-multiport[name=lighttpd-auth, port="http,https"]
 # adapt the following two items as needed
 logpath = /var/log/lighttpd/error.log
 maxretry = 2
@ -405,6 +395,17 @@ maxretry=5

 enabled = false
 filter  = 3proxy
-action  = iptables-multiport[name=3proxy, port=3128, protocol=tcp]
+action  = iptables[name=3proxy, port=3128, protocol=tcp]
 logpath = /var/log/3proxy.log

+[exim]
+enabled = false
+filter = exim
+action = iptables-multiport[name=exim,port="25,465,587"]
+logpath = /var/log/exim/mainlog
+
+[exim-spam]
+enabled = false
+filter = exim-spam
+action = iptables-multiport[name=exim-spam,port="25,465,587"]
+logpath = /var/log/exim/mainlog
--- a/33
+++ b/33
@ -229,7 +229,7 @@ class Fail2banClient:
 		elif len(cmd) == 2 and cmd[0] == "reload":
 			if self.__ping():
 				jail = cmd[1]
-				ret = self.__readJailConfig(jail)
+				ret = self.__readConfig(jail)
 				# Do not continue if configuration is not 100% valid
 				if not ret:
 					return False
@ -336,13 +336,13 @@ class Fail2banClient:
 			logSys.setLevel(logging.INFO)
 		else:
 			logSys.setLevel(logging.DEBUG)
-		# Add the default logging handler
-		stdout = logging.StreamHandler(sys.stdout)
+		# Add the default logging handler to dump to stderr
+		logout = logging.StreamHandler(sys.stderr)
 		# set a format which is simpler for console use
 		formatter = logging.Formatter('%(levelname)-6s %(message)s')
 		# tell the handler to use this format
-		stdout.setFormatter(formatter)
-		logSys.addHandler(stdout)
+		logout.setFormatter(formatter)
+		logSys.addHandler(logout)

 		# Set the configuration path
 		self.__configurator.setBaseDir(self.__conf["conf"])
@ -394,19 +394,18 @@ class Fail2banClient:
 				return False
 			return self.__processCommand(args)

-	def __readConfig(self):
+	def __readConfig(self, jail=None):
 		# Read the configuration
-		self.__configurator.readAll()
-		ret = self.__configurator.getOptions()
-		self.__configurator.convertToProtocol()
-		self.__stream = self.__configurator.getConfigStream()
-		return ret
-
-	def __readJailConfig(self, jail):
-		self.__configurator.readAll()
-		ret = self.__configurator.getOptions(jail)
-		self.__configurator.convertToProtocol()
-		self.__stream = self.__configurator.getConfigStream()
+		# TODO: get away from stew of return codes and exception
+		# handling -- handle via exceptions
+		try:
+			self.__configurator.readAll()
+			ret = self.__configurator.getOptions(jail)
+			self.__configurator.convertToProtocol()
+			self.__stream = self.__configurator.getConfigStream()
+		except Exception, e:
+			logSys.error("Failed during configuration: %s" % e)
+			ret = False
 		return ret

 	#@staticmethod
--- a/531
+++ b/531
@ -17,9 +17,17 @@
 # You should have received a copy of the GNU General Public License
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Fail2Ban  reads log file that contains password failure report
+and bans the corresponding IP addresses using firewall rules.
+
+This tools can test regular expressions for "fail2ban".
+
+Report bugs to https://github.com/fail2ban/fail2ban/issues
+"""

 __author__ = "Cyril Jaquier, Yaroslav Halchenko"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko"
+__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko"
 __license__ = "GPL"

 import getopt, sys, time, logging, os
@ -32,231 +40,238 @@ except ImportError, e:
 	sys.path.insert(1, "/usr/share/fail2ban")
 	from common.version import version

+from optparse import OptionParser, Option
+
 from client.configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 from server.filter import Filter
 from server.failregex import RegexException

+from testcases.utils import FormatterWithTraceBack
 # Gets the instance of the logger.
-logSys = logging.getLogger("fail2ban.regex")
+logSys = logging.getLogger("fail2ban")

-class RegexStat:
+def shortstr(s, l=53):
+	"""Return shortened string
+	"""
+	if len(s) > l:
+		return s[:l-3] + '...'
+	return s
+
+def pprint_list(l, header=None):
+	if not len(l):
+		return
+	if header:
+		s = "|- %s\n" % header
+	else:
+		s = ''
+	print s + "|  " + "\n|  ".join(l) + '\n`-'
+
+def get_opt_parser():
+	# use module docstring for help output
+	p = OptionParser(
+				usage="%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]\n" % sys.argv[0] + __doc__
+				+ """
+LOG:
+    string                  a string representing a log line
+    filename                path to a log file (/var/log/auth.log)
+
+REGEX:
+    string                  a string representing a 'failregex'
+    filename                path to a filter file (filter.d/sshd.conf)
+
+IGNOREREGEX:
+    string                  a string representing an 'ignoreregex'
+    filename                path to a filter file (filter.d/sshd.conf)
+""",
+				version="%prog " + version)
+
+	p.add_options([
+		Option('-l', "--log-level", type="choice",
+			   dest="log_level",
+			   choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'),
+			   default=None,
+			   help="Log level for the Fail2Ban logger to use"),
+		Option("-v", "--verbose", action='store_true',
+			   help="Be verbose in output"),
+		Option("--print-all-missed", action='store_true',
+			   help="Either to print all missed lines"),
+		Option("--print-all-ignored", action='store_true',
+			   help="Either to print all ignored lines"),
+		Option("-t", "--log-traceback", action='store_true',
+			   help="Enrich log-messages with compressed tracebacks"),
+		Option("--full-traceback", action='store_true',
+			   help="Either to make the tracebacks full, not compressed (as by default)"),
+
+		])
+
+	return p
+
+
+class RegexStat(object):

 	def __init__(self, failregex):
-		self.__stats = 0
-		self.__failregex = failregex
-		self.__ipList = list()
+		self._stats = 0
+		self._failregex = failregex
+		self._ipList = list()

 	def __str__(self):
 		return "%s(%r) %d failed: %s" \
-		  % (self.__class__, self.__failregex, self.__stats, self.__ipList)
+		  % (self.__class__, self._failregex, self._stats, self._ipList)

 	def inc(self):
-		self.__stats += 1
+		self._stats += 1

 	def getStats(self):
-		return self.__stats
+		return self._stats

 	def getFailRegex(self):
-		return self.__failregex
+		return self._failregex

 	def appendIP(self, value):
-		self.__ipList.extend(value)
+		self._ipList.append(value)

 	def getIPList(self):
-		return self.__ipList
+		return self._ipList

-class Fail2banRegex:

-	test = None
+class LineStats(object):
+	"""Just a convenience container for stats
+	"""
+	def __init__(self):
+		self.tested = self.matched = 0
+		self.missed_lines = []
+		self.ignored_lines = []
+
+	def __str__(self):
+		return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self
+
+	@property
+	def ignored(self):
+		return len(self.ignored_lines)
+
+	@property
+	def missed(self):
+		return self.tested - (self.ignored + self.matched)
+
+	# just for convenient str
+	def __getitem__(self, key):
+		return getattr(self, key)
+
+
+class Fail2banRegex(object):

 	CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"}

-	def __init__(self):
-		self.__filter = Filter(None)
-		self.__ignoreregex = list()
-		self.__failregex = list()
-		self.__verbose = False
-		# Setup logging
-		logging.getLogger("fail2ban").handlers = []
-		self.__hdlr = logging.StreamHandler(Fail2banRegex.test)
-		# set a format which is simpler for console use
-		formatter = logging.Formatter("%(message)s")
-		# tell the handler to use this format
-		self.__hdlr.setFormatter(formatter)
-		self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN
-		logging.getLogger("fail2ban").addHandler(self.__hdlr)
-		logging.getLogger("fail2ban").setLevel(logging.ERROR)
+	def __init__(self, opts):
+		self._verbose = opts.verbose
+		self._print_all_missed = opts.print_all_missed
+		self._print_all_ignored = opts.print_all_ignored

-	#@staticmethod
-	def dispVersion():
-		print "Fail2Ban v" + version
-		print
-		print "Copyright (c) 2004-2008 Cyril Jaquier"
-		print "Copyright of modifications held by their respective authors."
-		print "Licensed under the GNU General Public License v2 (GPL)."
-		print
-		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
-		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
-	dispVersion = staticmethod(dispVersion)
+		self._filter = Filter(None)
+		self._ignoreregex = list()
+		self._failregex = list()
+		self._line_stats = LineStats()

-	#@staticmethod
-	def dispUsage():
-		print "Usage: "+sys.argv[0]+" [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]"
-		print
-		print "Fail2Ban v" + version + " reads log file that contains password failure report"
-		print "and bans the corresponding IP addresses using firewall rules."
-		print
-		print "This tools can test regular expressions for \"fail2ban\"."
-		print
-		print "Options:"
-		print "    -h, --help              display this help message"
-		print "    -V, --version           print the version"
-		print "    -v, --verbose           verbose output"
-		print
-		print "Log:"
-		print "    string                  a string representing a log line"
-		print "    filename                path to a log file (/var/log/auth.log)"
-		print
-		print "Regex:"
-		print "    string                  a string representing a 'failregex'"
-		print "    filename                path to a filter file (filter.d/sshd.conf)"
-		print
-		print "IgnoreRegex:"
-		print "    string                  a string representing an 'ignoreregex'"
-		print "    filename                path to a filter file (filter.d/sshd.conf)"
-		print
-		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
-	dispUsage = staticmethod(dispUsage)

-	def getCmdLineOptions(self, optList):
-		""" Gets the command line options
-		"""
-		for opt in optList:
-			if opt[0] in ["-h", "--help"]:
-				self.dispUsage()
-				sys.exit(0)
-			elif opt[0] in ["-V", "--version"]:
-				self.dispVersion()
-				sys.exit(0)
-			elif opt[0] in ["-v", "--verbose"]:
-				self.__verbose = True
-
-	#@staticmethod
-	def logIsFile(value):
-		return os.path.isfile(value)
-	logIsFile = staticmethod(logIsFile)
-
-	def readIgnoreRegex(self, value):
+	def readRegex(self, value, regextype):
+		assert(regextype in ('fail', 'ignore'))
+		regex = regextype + 'regex'
 		if os.path.isfile(value):
 			reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS)
 			try:
 				reader.read(value)
-				print "Use ignoreregex file : " + value
-				self.__ignoreregex = [RegexStat(m)
-									for m in reader.get("Definition", "ignoreregex").split('\n')]
+				print "Use %11s file : %s" % (regex, value)
+				# TODO: reuse functionality in client
+				regex_values = [
+					RegexStat(m)
+					for m in reader.get("Definition", regex).split('\n')
+					if m != ""]
 			except NoSectionError:
-				print "No [Definition] section in " + value
-				print
+				print "No [Definition] section in %s" % value
 				return False
 			except NoOptionError:
-				print "No failregex option in " + value
-				print
+				print "No %s option in %s" % (regex, value)
 				return False
 			except MissingSectionHeaderError:
-				print "No section headers in " + value
-				print
+				print "No section headers in %s" % value
 				return False
 		else:
-			if len(value) > 53:
-				stripReg = value[0:50] + "..."
-			else:
-				stripReg = value
-			print "Use ignoreregex line : " + stripReg
-			self.__ignoreregex = [RegexStat(value)]
-		return True
+			print "Use %11s line : %s" % (regex, shortstr(value))
+			regex_values = [RegexStat(value)]

-	def readRegex(self, value):
-		if os.path.isfile(value):
-			reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS)
-			try:
-				reader.read(value)
-				print "Use regex file : " + value
-				self.__failregex = [RegexStat(m)
-									for m in reader.get("Definition", "failregex").split('\n')]
-			except NoSectionError:
-				print "No [Definition] section in " + value
-				print
-				return False
-			except NoOptionError:
-				print "No failregex option in " + value
-				print
-				return False
-			except MissingSectionHeaderError:
-				print "No section headers in " + value
-				print
-				return False
-		else:
-			if len(value) > 53:
-				stripReg = value[0:50] + "..."
-			else:
-				stripReg = value
-			print "Use regex line : " + stripReg
-			self.__failregex = [RegexStat(value)]
+		setattr(self, "_" + regex, regex_values)
+		for regex in regex_values:
+			getattr(
+				self._filter,
+				'add%sRegex' % regextype.title())(regex.getFailRegex())
 		return True

 	def testIgnoreRegex(self, line):
 		found = False
-		for regex in self.__ignoreregex:
-			logging.getLogger("fail2ban").setLevel(self.__logging_level)
-			try:
-				self.__filter.addIgnoreRegex(regex.getFailRegex())
-				try:
-					ret = self.__filter.ignoreLine(line)
-					if ret:
-						regex.inc()
-				except RegexException, e:
-					print e
-					return False
-			finally:
-				self.__filter.delIgnoreRegex(0)
-				logging.getLogger("fail2ban").setLevel(self.__logging_level)
+		try:
+			ret = self._filter.ignoreLine(line)
+			if ret is not None:
+				found = True
+				regex = self._ignoreregex[ret].inc()
+		except RegexException, e:
+			print e
+			return False
+		return found

 	def testRegex(self, line):
-		found = False
-		for regex in self.__ignoreregex:
-			self.__filter.addIgnoreRegex(regex.getFailRegex())
-		for regex in self.__failregex:
-			logging.getLogger("fail2ban").setLevel(logging.DEBUG)
-			try:
-				self.__filter.addFailRegex(regex.getFailRegex())
-				try:
-					ret = self.__filter.processLine(line)
-					if not len(ret) == 0:
-						if found == True:
-							ret[0].append(True)
-						else:
-							found = True
-							ret[0].append(False)
-						regex.inc()
-						regex.appendIP(ret)
-				except RegexException, e:
-					print e
-					return False
-				except IndexError:
-					print "Sorry, but no <host> found in regex"
-					return False
-			finally:
-				self.__filter.delFailRegex(0)
-				logging.getLogger("fail2ban").setLevel(logging.CRITICAL)
-		for regex in self.__ignoreregex:
-					self.__filter.delIgnoreRegex(0)
+		try:
+			ret = self._filter.processLine(line, checkAllRegex=True)
+			for match in ret:
+				# Append True/False flag depending if line was matched by
+				# more than one regex
+				match.append(len(ret)>1)
+				regex = self._failregex[match[0]]
+				regex.inc()
+				regex.appendIP(match)
+		except RegexException, e:
+			print e
+			return False
+		except IndexError:
+			print "Sorry, but no <host> found in regex"
+			return False
+		return len(ret) > 0
+
+
+	def process(self, test_lines):
+
+		for line in test_lines:
+			if line.startswith('#') or not line.strip():
+				# skip comment and empty lines
+				continue
+			is_ignored = fail2banRegex.testIgnoreRegex(line)
+			if is_ignored:
+				self._line_stats.ignored_lines.append(line)
+
+			if fail2banRegex.testRegex(line):
+				assert(not is_ignored)
+				self._line_stats.matched += 1
+			else:
+				if not is_ignored:
+					self._line_stats.missed_lines.append(line)
+			self._line_stats.tested += 1
+
+	def printLines(self, ltype):
+		lstats = self._line_stats
+		assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored))
+		l = lstats[ltype + '_lines']
+		if len(l):
+			header = "%s line(s):" % (ltype.capitalize(),)
+			if len(l) < 20 or getattr(self, '_print_all_' + ltype):
+				pprint_list([x.rstrip() for x in l], header)
+			else:
+				print "%s: too many to print.  Use --print-all-%s " \
+					  "to print all %d lines" % (header, ltype, len(l))

 	def printStats(self):
 		print
 		print "Results"
 		print "======="
-		print

 		def print_failregexes(title, failregexes):
 			# Print title
@ -264,106 +279,110 @@ class Fail2banRegex:
 			for cnt, failregex in enumerate(failregexes):
 				match = failregex.getStats()
 				total += match
-				if (match or self.__verbose):
-					out.append("|  %d) [%d] %s" % (cnt+1, match, failregex.getFailRegex()))
-			print "%s: %d total" % (title, total)
-			if len(out):
-				print "|- #) [# of hits] regular expression"
-				print '\n'.join(out)
-				print '`-'
-			print
+				if (match or self._verbose):
+					out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex()))
+
+				if self._verbose and len(failregex.getIPList()):
+					for ip in failregex.getIPList():
+						timeTuple = time.localtime(ip[2])
+						timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple)
+						out.append(
+							"    %s  %s%s" % (
+								ip[1],
+								timeString,
+								ip[3] and " (multiple regex matched)" or ""))
+
+			print "\n%s: %d total" % (title, total)
+			pprint_list(out, " #) [# of hits] regular expression")
 			return total

 		# Print title
-		total = print_failregexes("Failregex", self.__failregex)
-		_ = print_failregexes("Ignoreregex", self.__ignoreregex)
+		total = print_failregexes("Failregex", self._failregex)
+		_ = print_failregexes("Ignoreregex", self._ignoreregex)

-		print "Summary"
-		print "======="
-		print

-		if total == 0:
-			print "Sorry, no match"
-			print
-			print "Look at the above section 'Running tests' which could contain important"
-			print "information."
-			return False
-		else:
-			# Print stats
-			print "Addresses found:"
-			for cnt, failregex in enumerate(self.__failregex):
-				if self.__verbose or len(failregex.getIPList()):
-					print "[%d]" % (cnt+1)
-					for ip in failregex.getIPList():
-						timeTuple = time.localtime(ip[1])
-						timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple)
-						print "    %s (%s)%s" % (
-							ip[0], timeString, ip[2] and " (already matched)" or "")
-			print
+		print "\nDate template hits:"
+		out = []
+		for template in self._filter.dateDetector.getTemplates():
+			if self._verbose or template.getHits():
+				out.append("[%d] %s" % (template.getHits(), template.getName()))
+		pprint_list(out, "[# of hits] date format")

-			print "Date template hits:"
-			for template in self.__filter.dateDetector.getTemplates():
-				if self.__verbose or template.getHits():
-					print `template.getHits()` + " hit(s): " + template.getName()
-			print
+		print "\nLines: %s" % self._line_stats

-			print "Success, the total number of match is " + str(total)
-			print
-			print "However, look at the above section 'Running tests' which could contain important"
-			print "information."
-			return True
+		self.printLines('ignored')
+		self.printLines('missed')
+
+		return True


 if __name__ == "__main__":
-	fail2banRegex = Fail2banRegex()
-	# Reads the command line options.
-	try:
-		cmdOpts = 'hVcv'
-		cmdLongOpts = ['help', 'version', 'verbose']
-		optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts)
-	except getopt.GetoptError:
-		fail2banRegex.dispUsage()
-		sys.exit(-1)
-	# Process command line
-	fail2banRegex.getCmdLineOptions(optList)
+
+	parser = get_opt_parser()
+	(opts, args) = parser.parse_args()
+
+	fail2banRegex = Fail2banRegex(opts)

 	# We need 2 or 3 parameters
 	if not len(args) in (2, 3):
-		fail2banRegex.dispUsage()
+		sys.stderr.write("ERROR: provide both <LOG> and <REGEX>.\n\n")
+		parser.print_help()
 		sys.exit(-1)
+
+	# TODO: taken from -testcases -- move common functionality somewhere
+	if opts.log_level is not None: # pragma: no cover
+		# so we had explicit settings
+		logSys.setLevel(getattr(logging, opts.log_level.upper()))
+	else: # pragma: no cover
+		# suppress the logging but it would leave unittests' progress dots
+		# ticking, unless like with '-l fatal' which would be silent
+		# unless error occurs
+		logSys.setLevel(getattr(logging, 'FATAL'))
+
+	# Add the default logging handler
+	stdout = logging.StreamHandler(sys.stdout)
+
+	fmt = 'D: %(message)s'
+
+	if opts.log_traceback:
+		Formatter = FormatterWithTraceBack
+		fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt
 	else:
-		print
-		print "Running tests"
-		print "============="
-		print
+		Formatter = logging.Formatter

-		cmd_log, cmd_regex = args[:2]
+	# Custom log format for the verbose tests runs
+	if opts.verbose > 1: # pragma: no cover
+		stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt))
+	else: # pragma: no cover
+		# just prefix with the space
+		stdout.setFormatter(Formatter(fmt))
+	logSys.addHandler(stdout)

-		if len(args) == 3:
-			fail2banRegex.readIgnoreRegex(args[2]) or sys.exit(-1)
+	print
+	print "Running tests"
+	print "============="
+	print

-		fail2banRegex.readRegex(cmd_regex) or sys.exit(-1)
+	cmd_log, cmd_regex = args[:2]

-		if fail2banRegex.logIsFile(cmd_log):
-			try:
-				hdlr = open(cmd_log)
-				print "Use log file   : " + cmd_log
-				print
-				for line in hdlr:
-					fail2banRegex.testIgnoreRegex(line)
-					fail2banRegex.testRegex(line)
-			except IOError, e:
-				print e
-				print
-				sys.exit(-1)
-		else:
-			if len(sys.argv[1]) > 53:
-				stripLog = cmd_log[0:50] + "..."
-			else:
-				stripLog = cmd_log
-			print "Use single line: " + stripLog
-			print
-			fail2banRegex.testIgnoreRegex(cmd_log)
-			fail2banRegex.testRegex(cmd_log)
+	if len(args) == 3:
+		fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1)

-		fail2banRegex.printStats() or sys.exit(-1)
+	fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1)
+
+	if os.path.isfile(cmd_log):
+		try:
+			hdlr = open(cmd_log)
+			print "Use         log file : %s" % cmd_log
+			test_lines = hdlr.readlines()
+		except IOError, e:
+			print e
+			sys.exit(-1)
+	else:
+		print "Use      single line : %s" % shortstr(cmd_log)
+		test_lines = [ cmd_log ]
+	print
+
+	fail2banRegex.process(test_lines)
+
+	fail2banRegex.printStats() or sys.exit(-1)
--- a/22
+++ b/22
@ -27,6 +27,14 @@ __license__ = "GPL"

 import unittest, logging, sys, time, os

+if sys.version_info >= (2, 6):
+	import json
+else:
+	try:
+		import simplejson as json
+	except ImportError:
+		json = None
+
 from common.version import version
 from testcases import banmanagertestcase
 from testcases import clientreadertestcase
@ -37,6 +45,8 @@ from testcases import datedetectortestcase
 from testcases import actiontestcase
 from testcases import sockettestcase
 from testcases import misctestcase
+if json:
+	from testcases import samplestestcase

 from testcases.utils import FormatterWithTraceBack
 from server.mytime import MyTime
@ -52,7 +62,7 @@ def get_opt_parser():
 	p.add_options([
 		Option('-l', "--log-level", type="choice",
 			   dest="log_level",
-			   choices=('debug', 'info', 'warn', 'error', 'fatal'),
+			   choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'),
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
 		Option('-n', "--no-network", action="store_true",
@ -76,9 +86,10 @@ parser = get_opt_parser()
 logSys = logging.getLogger("fail2ban")

 # Numerical level of verbosity corresponding to a log "level"
-verbosity = {'debug': 3,
+verbosity = {'heavydebug': 4,
+			 'debug': 3,
 			 'info': 2,
-			 'warn': 1,
+			 'warning': 1,
 			 'error': 1,
 			 'fatal': 0,
 			 None: 1}[opts.log_level]
@ -169,6 +180,11 @@ tests.addTest(unittest.makeSuite(filtertestcase.JailTests))

 # DateDetector
 tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest))
+if json:
+	# Filter Regex tests with sample logs
+	tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex))
+else:
+	print "I: Skipping filter samples testing. No simplejson/json module"

 #
 # Extensive use-tests of different available filters backends
--- a/server/datedetector.py
+++ b/server/datedetector.py
@ -174,6 +174,7 @@ class DateDetector:
 				match = template.matchDate(line)
 				if not match is None:
 					logSys.debug("Matched time template %s" % template.getName())
+					template.incHits()
 					return match
 			return None
 		finally:
--- a/server/datetemplate.py
+++ b/server/datetemplate.py
@ -59,11 +59,12 @@ class DateTemplate:
 	
 	def getHits(self):
 		return self.__hits
+
+	def incHits(self):
+		self.__hits += 1
 	
 	def matchDate(self, line):
 		dateMatch = self.__cRegex.search(line)
-		if not dateMatch is None:
-			self.__hits += 1
 		return dateMatch
 	
 	def getDate(self, line):
--- a/server/failregex.py
+++ b/server/failregex.py
@ -41,7 +41,7 @@ class Regex:
 		self._matchCache = None
 		# Perform shortcuts expansions.
 		# Replace "<HOST>" with default regular expression for host.
-		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]+)")
+		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]*\w)")
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
 		try:
@ -50,7 +50,8 @@ class Regex:
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
 								 regex)
-	
+	def __str__(self):
+		return "%s(%r)" % (self.__class__.__name__, self._regex)
 	##
 	# Gets the regular expression.
 	#
--- a/server/filter.py
+++ b/server/filter.py
@ -21,6 +21,8 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
 __license__ = "GPL"

+import sys
+
 from failmanager import FailManagerEmpty
 from failmanager import FailManager
 from ticket import FailTicket
@ -282,7 +284,7 @@ class Filter(JailThread):
 		return False


-	def processLine(self, line):
+	def processLine(self, line, returnRawHost=False, checkAllRegex=False):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
 		try:
@ -291,6 +293,8 @@ class Filter(JailThread):
 		except UnicodeDecodeError:
 			l = line
 		l = l.rstrip('\r\n')
+
+		logSys.log(7, "Working on line %r", l)
 		timeMatch = self.dateDetector.matchTime(l)
 		if timeMatch:
 			# Lets split into time part and log part of the line
@ -302,14 +306,15 @@ class Filter(JailThread):
 		else:
 			timeLine = l
 			logLine = l
-		return self.findFailure(timeLine, logLine)
+		return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex)

 	def processLineAndAdd(self, line):
 		"""Processes the line for failures and populates failManager
 		"""
 		for element in self.processLine(line):
-			ip = element[0]
-			unixTime = element[1]
+			failregex = element[0]
+			ip = element[1]
+			unixTime = element[2]
 			logSys.debug("Processing line with time:%s and ip:%s"
 						 % (unixTime, ip))
 			if unixTime < MyTime.time() - self.getFindTime():
@ -320,6 +325,7 @@ class Filter(JailThread):
 				logSys.debug("Ignore %s" % ip)
 				continue
 			logSys.debug("Found %s" % ip)
+			## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),)
 			self.failManager.addFailure(FailTicket(ip, unixTime, [line]))

 	##
@ -330,11 +336,11 @@ class Filter(JailThread):
 	# @return: a boolean

 	def ignoreLine(self, line):
-		for ignoreRegex in self.__ignoreRegex:
+		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
 			ignoreRegex.search(line)
 			if ignoreRegex.hasMatched():
-				return True
-		return False
+				return ignoreRegexIndex
+		return None

 	##
 	# Finds the failure in a line given split into time and log parts.
@ -343,18 +349,22 @@ class Filter(JailThread):
 	# to find the logging time.
 	# @return a dict with IP and timestamp.

-	def findFailure(self, timeLine, logLine):
+	def findFailure(self, timeLine, logLine,
+			returnRawHost=False, checkAllRegex=False):
+		logSys.log(5, "Date: %r, message: %r", timeLine, logLine)
 		failList = list()
 		# Checks if we must ignore this line.
-		if self.ignoreLine(logLine):
+		if self.ignoreLine(logLine) is not None:
 			# The ignoreregex matched. Return.
+			logSys.log(7, "Matched ignoreregex and was ignored")
 			return failList
+		date = self.dateDetector.getUnixTime(timeLine)
 		# Iterates over all the regular expressions.
-		for failRegex in self.__failRegex:
+		for failRegexIndex, failRegex in enumerate(self.__failRegex):
 			failRegex.search(logLine)
 			if failRegex.hasMatched():
 				# The failregex matched.
-				date = self.dateDetector.getUnixTime(timeLine)
+				logSys.log(7, "Matched %s", failRegex)
 				if date is None:
 					logSys.debug("Found a match for %r but no valid date/time "
 								 "found for %r. Please file a detailed issue on"
@ -364,12 +374,17 @@ class Filter(JailThread):
 				else:
 					try:
 						host = failRegex.getHost()
-						ipMatch = DNSUtils.textToIp(host, self.__useDns)
-						if ipMatch:
-							for ip in ipMatch:
-								failList.append([ip, date])
-							# We matched a regex, it is enough to stop.
-							break
+						if returnRawHost:
+							failList.append([failRegexIndex, host, date])
+							if not checkAllRegex:
+								break
+						else:
+							ipMatch = DNSUtils.textToIp(host, self.__useDns)
+							if ipMatch:
+								for ip in ipMatch:
+									failList.append([failRegexIndex, ip, date])
+								if not checkAllRegex:
+									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
 						logSys.error(e)
 		return failList
@ -473,7 +488,7 @@ class FileFilter(Filter):
 			return False
 		# Try to open log file.
 		try:
-			container.open()
+			has_content = container.open()
 		# see http://python.org/dev/peps/pep-3151/
 		except IOError, e:
 			logSys.error("Unable to open %s" % filename)
@ -488,7 +503,12 @@ class FileFilter(Filter):
 			logSys.exception(e)
 			return False

-		while True:
+		# yoh: has_content is just a bool, so do not expect it to
+		# change -- loop is exited upon break, and is not entered at
+		# all if upon container opening that one was empty.  If we
+		# start reading tested to be empty container -- race condition
+		# might occur leading at least to tests failures.
+		while has_content:
 			line = container.readline()
 			if (line == "") or not self._isActive():
 				# The jail reached the bottom or has been stopped
@ -551,10 +571,20 @@ class FileContainer:
 		fd = self.__handler.fileno()
 		flags = fcntl.fcntl(fd, fcntl.F_GETFD)
 		fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
+		# Stat the file before even attempting to read it
+		stats = os.fstat(self.__handler.fileno())
+		if not stats.st_size:
+			# yoh: so it is still an empty file -- nothing should be
+			#      read from it yet
+			# print "D: no content -- return"
+			return False
 		firstLine = self.__handler.readline()
 		# Computes the MD5 of the first line.
 		myHash = md5sum(firstLine).digest()
-		stats = os.fstat(self.__handler.fileno())
+		## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % (
+		## 	self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos,
+		## 	self.__hash != myHash or self.__ino != stats.st_ino)
+		## sys.stdout.flush()
 		# Compare hash and inode
 		if self.__hash != myHash or self.__ino != stats.st_ino:
 			logSys.debug("Log rotation detected for %s" % self.__filename)
@ -563,6 +593,7 @@ class FileContainer:
 			self.__pos = 0
 		# Sets the file pointer to the last position.
 		self.__handler.seek(self.__pos)
+		return True

 	def readline(self):
 		if self.__handler is None:
@ -576,6 +607,8 @@ class FileContainer:
 			# Closes the file.
 			self.__handler.close()
 			self.__handler = None
+		## print "D: Closed %s with pos %d" % (handler, self.__pos)
+		## sys.stdout.flush()



@ -598,9 +631,9 @@ class DNSUtils:
 		"""
 		try:
 			return socket.gethostbyname_ex(dns)[2]
-		except socket.gaierror:
-			logSys.warn("Unable to find a corresponding IP address for %s"
-						% dns)
+		except socket.error, e:
+			logSys.warn("Unable to find a corresponding IP address for %s: %s"
+						% (dns, e))
 			return list()
 	dnsToIp = staticmethod(dnsToIp)

--- a/server/filterpoll.py
+++ b/server/filterpoll.py
@ -52,7 +52,7 @@ class FilterPoll(FileFilter):
 		FileFilter.__init__(self, jail)
 		self.__modified = False
 		## The time of the last modification of the file.
-		self.__lastModTime = dict()
+		self.__prevStats = dict()
 		self.__file404Cnt = dict()
 		logSys.debug("Created FilterPoll")

@ -62,7 +62,7 @@ class FilterPoll(FileFilter):
 	# @param path log file path

 	def _addLogPath(self, path):
-		self.__lastModTime[path] = 0
+		self.__prevStats[path] = (0, None, None)	 # mtime, ino, size
 		self.__file404Cnt[path] = 0

 	##
@ -71,7 +71,7 @@ class FilterPoll(FileFilter):
 	# @param path the log file to delete

 	def _delLogPath(self, path):
-		del self.__lastModTime[path]
+		del self.__prevStats[path]
 		del self.__file404Cnt[path]

 	##
@ -84,6 +84,9 @@ class FilterPoll(FileFilter):
 	def run(self):
 		self.setActive(True)
 		while self._isActive():
+			if logSys.getEffectiveLevel() <= 6:
+				logSys.log(6, "Woke up idle=%s with %d files monitored",
+						   self.getIdle(), len(self.getLogPath()))
 			if not self.getIdle():
 				# Get file modification
 				for container in self.getLogPath():
@ -118,12 +121,20 @@ class FilterPoll(FileFilter):
 	def isModified(self, filename):
 		try:
 			logStats = os.stat(filename)
+			stats = logStats.st_mtime, logStats.st_ino, logStats.st_size
+			pstats = self.__prevStats[filename]
 			self.__file404Cnt[filename] = 0
-			if self.__lastModTime[filename] == logStats.st_mtime:
+			if logSys.getEffectiveLevel() <= 7:
+				# we do not want to waste time on strftime etc if not necessary
+				dt = logStats.st_mtime - pstats[0]
+				logSys.log(7, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s",
+				           filename, pstats, stats, dt)
+				# os.system("stat %s | grep Modify" % filename)
+			if pstats == stats:
 				return False
 			else:
-				logSys.debug(filename + " has been modified")
-				self.__lastModTime[filename] = logStats.st_mtime
+				logSys.debug("%s has been modified", filename)
+				self.__prevStats[filename] = stats
 				return True
 		except OSError, e:
 			logSys.error("Unable to get stat on %s because of: %s"
--- a/server/jailthread.py
+++ b/server/jailthread.py
@ -56,7 +56,7 @@ class JailThread(Thread):
 	
 	def setSleepTime(self, value):
 		self.__sleepTime = value
-		logSys.info("Set sleeptime = " + value)
+		logSys.info("Set sleeptime %s" % value)
 	
 	##
 	# Get the time that the thread sleeps.
--- a/testcases/actiontestcase.py
+++ b/testcases/actiontestcase.py
@ -69,11 +69,11 @@ class ExecuteAction(unittest.TestCase):
 		self.assertFalse(Action.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
 		self.assertFalse(Action.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# missing tags are ok
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
 		# rest is just cool
-		self.assertEquals(Action.substituteRecursiveTags(aInfo),
+		self.assertEqual(Action.substituteRecursiveTags(aInfo),
 								{ 'HOST': "192.0.2.0",
 									'ABC': '123 192.0.2.0',
 									'xyz': '890 123 192.0.2.0',
--- a/testcases/clientreadertestcase.py
+++ b/testcases/clientreadertestcase.py
@ -108,7 +108,7 @@ class JailReaderTest(unittest.TestCase):
 		action = "mail-whois[name=SSH]"
 		expected = ['mail-whois', {'name': 'SSH'}]
 		result = JailReader.splitAction(action)
-		self.assertEquals(expected, result)
+		self.assertEqual(expected, result)

 class JailsReaderTest(unittest.TestCase):

@ -132,7 +132,7 @@ class JailsReaderTest(unittest.TestCase):
 		jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm
 		self.assertTrue(jails.read())		  # opens fine
 		self.assertTrue(jails.getOptions())	  # reads fine
-		comm_commands = jails.convert()
+		comm_commands = jails.convert(allow_no_files=True)

 		# by default we have lots of jails ;)
 		self.assertTrue(len(comm_commands))
--- a/testcases/files/logs/3proxy
+++ b/testcases/files/logs/3proxy
@ -1,3 +1,6 @@
+# failJSON: { "time": "2013-06-11T02:09:40", "match": true , "host": "1.2.3.4" }
 11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1
+# failJSON: { "time": "2013-06-11T02:09:43", "match": true , "host": "1.2.3.4" }
 11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1
+# failJSON: { "time": "2013-06-13T01:39:34", "match": true , "host": "1.2.3.4" }
 13-06-2013 01:39:34 +0300 PROXY.3128 00508 - 1.2.3.4:28938 0.0.0.0:0 0 0 0 
--- a/testcases/files/logs/apache-auth
+++ b/testcases/files/logs/apache-auth
@ -1,4 +1,5 @@
 # Should not match -- DoS vector https://vndh.net/note:fail2ban-089-denial-service
+# failJSON: { "match": false }
 [Sat Jun 01 02:17:42 2013] [error] [client 192.168.33.1] File does not exist: /srv/http/site/[client 192.168.0.1] user root not found

 # should match
@ -105,7 +106,8 @@
 # failJSON: { "time": "2013-07-29T02:15:26", "match": true , "host": "127.0.0.1" }
 [Mon Jul 29 02:15:26 2013] [error] [client 127.0.0.1] Digest: invalid nonce LWEDr5TiBAA=ceddd011628c30e3646f7acda4f1a0ab6b7c5ae6 received - user attempted time travel

-
-
 # failJSON: { "time": "2013-07-29T02:12:55", "match": true , "host": "127.0.0.1" }
 [Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel
+
+# failJSON: { "time": "2005-06-01T02:17:42", "match": true , "host": "192.168.0.2" }
+[Sat Jun 01 02:17:42 2013] [error] [client 192.168.0.2] user root not found
--- a/testcases/files/logs/apache-badbots
+++ b/testcases/files/logs/apache-badbots
@ -0,0 +1,2 @@
+# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" }
+1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02"
--- a/testcases/files/logs/apache-nohome
+++ b/testcases/files/logs/apache-nohome
@ -0,0 +1,6 @@
+# Apache 2.2
+# failJSON: { "time": "2005-06-01T11:23:08", "match": true , "host": "1.2.3.4" }
+[Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] File does not exist: /xxx/~
+# Apache 2.4
+# failJSON: { "time": "2005-06-27T11:55:44", "match": true , "host": "192.0.2.12" }
+[Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 192.0.2.12:46652] AH00128: File does not exist: /xxx/~
--- a/testcases/files/logs/apache-noscript
+++ b/testcases/files/logs/apache-noscript
@ -1 +1,4 @@
+# failJSON: { "time": "2005-06-09T07:57:47", "match": true , "host": "192.0.43.10" }
 [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat
+# failJSON: { "time": "2005-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php
--- a/testcases/files/logs/apache-overflows
+++ b/testcases/files/logs/apache-overflows
@ -1,2 +1,4 @@
+# failJSON: { "time": "2005-03-16T15:39:29", "match": true , "host": "58.179.109.179" }
 [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8
+# failJSON: { "time": "2005-03-15T15:44:47", "match": true , "host": "121.222.2.133" }
 [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9
--- a/testcases/files/logs/assp
+++ b/testcases/files/logs/assp
@ -1,13 +1,25 @@
+# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" }
 Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" }
 Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+# failJSON: { "time": "2013-04-07T07:10:37", "match": true , "host": "68.171.223.68" }
 Apr-07-13 07:10:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+# failJSON: { "time": "2013-04-07T07:12:37", "match": true , "host": "68.171.223.68" }
 Apr-07-13 07:12:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+# failJSON: { "time": "2013-04-07T07:14:36", "match": true , "host": "68.171.223.68" }
 Apr-07-13 07:14:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol;
+# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (8);
+# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (9);
+# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (10);
+# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;
+# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;
+# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;
+# failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" }
 Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6;

--- a/testcases/files/logs/asterisk
+++ b/testcases/files/logs/asterisk
@ -1,15 +1,40 @@
 # Sample log files for asterisk 
+# failJSON: { "time": "2012-02-13T17:21:54", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:21:54] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - Wrong password
+# failJSON: { "time": "2012-02-13T17:18:22", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:18:22] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - No matching peer found
+# failJSON: { "time": "2012-02-13T17:21:21", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:21:21] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - Username/auth name mismatch
+# failJSON: { "time": "2012-02-13T17:32:01", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:32:01] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - Device does not match ACL
+# failJSON: { "time": "2012-02-13T17:34:10", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:34:10] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - Peer is not supposed to register
+# failJSON: { "time": "2012-02-13T17:36:23", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:36:23] NOTICE[1638] chan_sip.c: Registration from '<sip:301@example.com>' failed for '1.2.3.4' - ACL error (permit/deny)
+# failJSON: { "time": "2012-02-13T17:53:59", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:53:59] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed to authenticate as 'Fail2ban'
+# failJSON: { "time": "2012-02-13T17:39:20", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:39:20] NOTICE[1638] chan_iax2.c: No registration for peer 'Fail2ban' (from 1.2.3.4)
+# failJSON: { "time": "2012-02-13T17:44:26", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247)
+# failJSON: { "time": "2012-02-13T17:37:07", "match": true , "host": "1.2.3.4" }
 [2012-02-13 17:37:07] NOTICE[1638] chan_sip.c: Failed to authenticate user "Fail2ban" <sip:301@1.2.3.4>;tag=1r698745234
+# failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" }
 [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'.
+# failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" }
 [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found
+# failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" }
 [2013-05-13 07:10:53] SECURITY[1204] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="1368439853-500975",Severity="Error",Service="SIP",EventVersion="1",AccountID="00972599580679",SessionID="0x7f8ecc0421f8",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/1.2.3.4/5070"
+# failJSON: { "time": "2013-06-10T18:15:03", "match": true , "host": "1.2.3.4" }
 [2013-06-10 18:15:03] NOTICE[2723] chan_sip.c: Registration from '"100"<sip:100@192.168.0.2:5060>' failed for '1.2.3.4' - Not a local domain
+# http://forum.4psa.com/showthread.php?t=6601
+# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" }
+[2009-12-22 16:35:24] NOTICE[6163] chan_sip.c: Sending fake auth rejection for device <sip:0004*001@192.168.2.102>;tag=e3793a95e1acbc69o
+# http://www.freepbx.org/forum/general-help/fake-auth-rejection
+# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" }
+[2009-12-22 16:35:24] NOTICE[1570][C-00000086] chan_sip.c: Sending fake auth rejection for device 1022<sip:1022@192.168.2.102>;tag=5d8b6f92
+# http://www.spinics.net/lists/asterisk/msg127381.html
+# failJSON: { "time": "2009-12-22T16:35:24", "match": true , "host": "192.168.2.102" }
+[2009-12-22 16:35:24] NOTICE[14916]: chan_sip.c:15644 handle_request_subscribe: Sending fake auth rejection for user <sip:CS@192.168.2.102>;tag=6pwd6erg54
+# failJSON: { "time": "2013-07-06T09:09:25", "match": true , "host": "141.255.164.106" }
+[2013-07-06 09:09:25] SECURITY[3308] res_security_log.c: SecurityEvent="InvalidPassword",EventTV="1373098165-824497",Severity="Error",Service="SIP",EventVersion="2",AccountID="972592891005",SessionID="0x88aab6c",LocalAddress="IPV4/UDP/92.28.73.180/5060",RemoteAddress="IPV4/UDP/141.255.164.106/5084",Challenge="41d26de5",ReceivedChallenge="41d26de5",ReceivedHash="7a6a3a2e95a05260aee612896e1b4a39"
--- a/testcases/files/logs/courierlogin
+++ b/testcases/files/logs/courierlogin
@ -0,0 +1,8 @@
+# failJSON: { "time": "2005-04-23T21:59:01", "match": true , "host": "1.2.3.4" }
+Apr 23 21:59:01 dns2 imapd: LOGIN FAILED, user=sales@example.com, ip=[::ffff:1.2.3.4]
+# failJSON: { "time": "2005-04-23T21:59:38", "match": true , "host": "198.51.100.76" }
+Apr 23 21:59:38 dns2 pop3d: LOGIN FAILED, user=info@example.com, ip=[::ffff:198.51.100.76]
+# failJSON: { "time": "2004-11-13T08:11:53", "match": true , "host": "198.51.100.33" }
+Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff:198.51.100.33]
+# failJSON: { "time": "2005-04-17T19:17:11", "match": true , "host": "1.2.3.4" }
+Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4]
--- a/testcases/files/logs/couriersmtp
+++ b/testcases/files/logs/couriersmtp
@ -0,0 +1,8 @@
+# failJSON: { "time": "2005-04-10T03:47:57", "match": true , "host": "1.2.3.4" }
+Apr 10 03:47:57 web courieresmtpd: error,relay=::ffff:1.2.3.4,ident=tmf,from=<tmf@example.com>,to=<mailman-subscribe@example.com>: 550 User unknown.
+# failJSON: { "time": "2005-07-06T03:42:28", "match": true , "host": "1.2.3.4" }
+Jul  6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<admin at memcpy>: 550 User unknown.
+# failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" }
+Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown.
+# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" }
+Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=<firozquarl@aclunc.org>,to=<BOGUSUSER@HOSTEDDOMAIN.org>: 550 User unknown.
--- a/testcases/files/logs/cyrus-imap
+++ b/testcases/files/logs/cyrus-imap
@ -0,0 +1,13 @@
+# failJSON: { "time": "2005-01-04T21:51:05", "match": true , "host": "127.0.0.1" }
+Jan 4 21:51:05 hostname cyrus/imap[5355]: badlogin: localhost.localdomain [127.0.0.1] plaintext cyrus@localdomain SASL(-13): authentication failure: checkpass failed
+# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "198.51.100.23" }
+Feb 20 17:23:32 domain cyrus/pop3[18635]: badlogin: localhost [198.51.100.23] plaintext administrator SASL(-13): authentication failure: checkpass failed
+# failJSON: { "time": "2005-02-20T17:23:32", "match": true , "host": "1.2.3.4" }
+Feb 20 17:23:32 cyrus/pop3[4297]: badlogin: example.com [1.2.3.4] plaintext mail0001 SASL(-13): authentication failure: checkpass failed
+# failJSON: { "time": "2005-06-08T18:11:13", "match": true , "host": "198.51.100.45" }
+Jun 8 18:11:13 lampserver imap[4480]: badlogin: example.com [198.51.100.45] DIGEST-MD5 [SASL(-13): authentication failure: client response doesn't match what we generated]
+# failJSON: { "time": "2004-12-21T10:01:57", "match": true , "host": "198.51.100.57" }
+Dec 21 10:01:57 hostname imapd[18454]: badlogin: example.com [198.51.100.57] CRAM-MD5 [SASL(-13): authentication failure: incorrect digest response]
+# failJSON: { "time": "2004-12-30T16:03:27", "match": true , "host": "1.2.3.4" }
+Dec 30 16:03:27 somehost imapd[2517]: badlogin: local-somehost[1.2.3.4] OTP [SASL(-13): authentication failure: External SSF not good enough]
+
--- a/testcases/files/logs/dovecot
+++ b/testcases/files/logs/dovecot
@ -1,15 +1,31 @@
+# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" }
@400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=<martin@waschbuesch.de>, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS

-@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224
+# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" }
+@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224
 # Above example with injected rhost into ruser -- should not match for 1.2.3.4
-@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10
-@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225  user=root
+# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" }
+@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10
+# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" }
+@400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225  user=root

+# failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" }
 Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193

+# failJSON: { "time": "2005-06-13T16:30:54", "match": true , "host": "49.176.98.87" }
 Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=<username.bob>, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS
+# failJSON: { "time": "2005-06-14T00:48:21", "match": true , "host": "59.167.242.100" }
 Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected
+# failJSON: { "time": "2005-06-13T20:48:11", "match": true , "host": "121.44.24.254" }
 Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected
+# failJSON: { "time": "2005-06-13T21:48:06", "match": true , "host": "180.200.180.81" }
 Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS
+# failJSON: { "time": "2005-06-13T20:20:21", "match": true , "host": "180.189.168.166" }
 Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected
+# failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" }
+Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=<info>, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210

+# failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" }
+Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=<test>, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session=<YADINsQCDs5BH8Pg>
+# failJSON: { "time": "2005-07-02T13:49:32", "match": true , "host": "192.51.100.13" }
+Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempts in 58 secs): user=<>, rip=192.51.100.13, lip=203.0.113.17, session=<LgDINsQCkttVIMPg>
--- a/testcases/files/logs/dropbear
+++ b/testcases/files/logs/dropbear
@ -0,0 +1,9 @@
+# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" }
+Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543
+# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" }
+Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495
+# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" }
+Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543
+# failJSON: { "time": "2005-02-11T15:23:17", "match": true , "host": "198.51.100.215" }
+Feb 11 15:23:17 dropbear[1252]: login attempt for nonexistent user from ::ffff:198.51.100.215:60495
+
--- a/testcases/files/logs/exim
+++ b/testcases/files/logs/exim
@ -1,6 +1,31 @@
 # From IRC 2013-01-04
+# failJSON: { "time": "2013-01-04T17:03:46", "match": true , "host": "24.106.174.74" }
 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian)
 # From IRC 2013-06-13 XATRIX (Georgiy Mernov)
+# failJSON: { "time": "2013-06-12T03:57:58", "match": true , "host": "120.196.140.45" }
 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s)
+# failJSON: { "time": "2013-06-12T13:18:11", "match": true , "host": "101.66.165.86" }
 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data
+# failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "193.169.56.211" }
 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for <user@example.com>: Unrouteable address
+# http://forum.lissyara.su/viewtopic.php?f=20&t=29857
+# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem
+# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html
+# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory
+# failJSON: { "time": "2013-06-10T18:33:32", "match": true , "host": "202.132.70.178" }
+2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted
+# failJSON: { "time": "2013-06-02T06:54:20", "match": true , "host": "211.148.195.192" }
+2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n"
+# failJSON: { "time": "2013-06-02T09:05:48", "match": true , "host": "82.96.160.77" }
+2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n"
+# failJSON: { "time": "2013-06-02T09:22:05", "match": true , "host": "163.14.21.161" }
+2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET")
+# failJSON: { "time": "2013-06-02T15:06:18", "match": true , "host": "46.20.35.114" }
+2013-06-02 15:06:18 H=(VM-WIN2K3-1562) [46.20.35.114] sender verify fail for <usfh@technological-systems.com>: Unknown user
+# failJSON: { "time": "2013-06-07T02:02:09", "match": true , "host": "91.232.21.92" }
+2013-06-07 02:02:09 H=treeladders.kiev.ua [91.232.21.92] sender verify fail for <mailer@treeladders.kiev.ua>: all relevant MX records point to non-existent hosts
+# failJSON: { "time": "2013-06-15T16:34:55", "match": true , "host": "182.18.24.93" }
+2013-06-15 16:34:55 H=mx.tillions.com [182.18.24.93] F=<buh@caladan.ebay.sun.com> rejected RCPT <ruslan@maslovskiy.com.ua>: Sender verify failed
+# failJSON: { "time": "2013-06-15T16:36:49", "match": true , "host": "111.67.203.116" }
+2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F=<kadrofutcheti@mail.ru> rejected RCPT <oksana@birzhatm.ua>: Unknown user
+
--- a/testcases/files/logs/exim-spam
+++ b/testcases/files/logs/exim-spam
@ -0,0 +1,17 @@
+# http://forum.lissyara.su/viewtopic.php?f=20&t=29857
+# 2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem
+# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html
+# 2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory
+# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html
+# failJSON: { "time": "2004-01-18T07:15:35", "match": true , "host": "4.47.28.40" }
+2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0
+# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875
+# failJSON: { "time": "2013-06-15T11:19:33", "match": true , "host": "2.181.148.95" }
+2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org
+# failJSON: { "time": "2013-06-09T10:21:28", "match": true , "host": "46.254.240.82" }
+2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected
+# failJSON: { "time": "2013-06-15T11:20:36", "match": true , "host": "83.235.177.148" }
+2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL).
+# failJSON: { "time": "2013-06-16T02:50:43", "match": true , "host": "111.67.203.114" }
+2013-06-16 02:50:43 H=dbs.marsukov.com [111.67.203.114] F=<trudofspiori@mail.ru> rejected RCPT <info@nanomedtech.ua>: rejected because 111.67.203.114 is in a black list at dnsbl.sorbs.net\nCurrently Sending Spam See: http://www.sorbs.net/lookup.shtml?111.67.203.114
+
--- a/testcases/files/logs/gssftpd
+++ b/testcases/files/logs/gssftpd
@ -0,0 +1,2 @@
+# failJSON: { "time": "2005-01-22T18:09:46", "match": true , "host": "198.51.100.23" }
+Jan 22 18:09:46 host ftpd[132]: repeated login failures from 198.51.100.23 (example.com)
--- a/testcases/files/logs/lighttpd-auth
+++ b/testcases/files/logs/lighttpd-auth
@ -1,3 +1,5 @@
 #authentification failure (mod_auth)
+# failJSON: { "time": "2011-12-25T17:09:20", "match": true , "host": "4.4.4.4" }
 2011-12-25 17:09:20: (http_auth.c.875) password doesn't match for /gitweb/ username: francois, IP: 4.4.4.4
+# failJSON: { "time": "2012-09-26T10:24:35", "match": true , "host": "4.4.4.4" }
 2012-09-26 10:24:35: (http_auth.c.1136) digest: auth failed for  xxx : wrong password, IP: 4.4.4.4
--- a/testcases/files/logs/mysqld-auth
+++ b/testcases/files/logs/mysqld-auth
@ -1,6 +1,12 @@
+# failJSON: { "time": "2013-03-24T00:04:00", "match": true , "host": "192.168.1.35" }
 130324  0:04:00 [Warning] Access denied for user 'root'@'192.168.1.35' (using password: NO)
+# failJSON: { "time": "2013-03-24T08:24:09", "match": true , "host": "220.95.238.171" }
 130324  8:24:09 [Warning] Access denied for user 'root'@'220.95.238.171' (using password: NO)
+# failJSON: { "time": "2013-03-24T17:56:13", "match": true , "host": "61.160.223.112" }
 130324 17:56:13 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: NO)
+# failJSON: { "time": "2013-03-24T17:56:14", "match": true , "host": "61.160.223.112" }
 130324 17:56:14 [Warning] Access denied for user 'root'@'61.160.223.112' (using password: YES)
+# failJSON: { "time": "2013-03-24T19:01:39", "match": true , "host": "61.147.108.35" }
 130324 19:01:39 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: NO)
+# failJSON: { "time": "2013-03-24T19:01:40", "match": true , "host": "61.147.108.35" }
 130324 19:01:40 [Warning] Access denied for user 'root'@'61.147.108.35' (using password: YES)
--- a/testcases/files/logs/named-refused
+++ b/testcases/files/logs/named-refused
@ -1,6 +1,12 @@
+# failJSON: { "time": "2005-07-24T14:16:55", "match": true , "host": "194.145.196.18" }
 Jul 24 14:16:55 raid5 named[3935]: client 194.145.196.18#4795: query 'ricreig.com/NS/IN' denied
+# failJSON: { "time": "2005-07-24T14:16:56", "match": true , "host": "62.123.164.113" }
 Jul 24 14:16:56 raid5 named[3935]: client 62.123.164.113#32768: query 'ricreig.com/NS/IN' denied
+# failJSON: { "time": "2005-07-24T14:17:13", "match": true , "host": "148.160.29.6" }
 Jul 24 14:17:13 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'geo-mueller.de/NS/IN' denied
+# failJSON: { "time": "2005-07-24T14:20:25", "match": true , "host": "148.160.29.6" }
 Jul 24 14:20:25 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'shivaree.de/NS/IN' denied
+# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "148.160.29.6" }
 Jul 24 14:23:36 raid5 named[3935]: client 148.160.29.6#33081: query (cache) 'mietberatung.de/NS/IN' denied
+# failJSON: { "time": "2005-07-24T14:23:36", "match": true , "host": "62.109.4.89" }
 Jul 24 14:23:36 raid5 named[3935]: client 62.109.4.89#9334: view external: query (cache) './NS/IN' denied
--- a/testcases/files/logs/pam-generic
+++ b/testcases/files/logs/pam-generic
@ -1,7 +1,14 @@
+# failJSON: { "time": "2005-02-07T15:10:42", "match": true , "host": "192.168.1.1" }
 Feb  7 15:10:42 example pure-ftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=pure-ftpd ruser=sample-user rhost=192.168.1.1 
+# failJSON: { "time": "2005-05-12T09:47:54", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" }
 May 12 09:47:54 vaio sshd[16004]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com  user=root
+# failJSON: { "time": "2005-05-12T09:48:03", "match": true , "host": "71-13-115-12.static.mdsn.wi.charter.com" }
 May 12 09:48:03 vaio sshd[16021]: (pam_unix) authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=71-13-115-12.static.mdsn.wi.charter.com
+# failJSON: { "time": "2005-05-15T18:02:12", "match": true , "host": "66.232.129.62" }
 May 15 18:02:12 localhost proftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=66.232.129.62  user=mark
+# failJSON: { "time": "2004-11-25T17:12:13", "match": true , "host": "192.168.10.3" }
 Nov 25 17:12:13 webmail pop(pam_unix)[4920]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.168.10.3 user=mailuser
+# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" }
 Jul 19 18:11:26 srv2 vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com
+# failJSON: { "time": "2005-07-19T18:11:26", "match": true , "host": "www3.google.com" }
 Jul 19 18:11:26 srv2 vsftpd: pam_unix: authentication failure; logname= uid=0 euid=0 tty=ftp ruser=an8767 rhost=www3.google.com
--- a/testcases/files/logs/php-url-fopen
+++ b/testcases/files/logs/php-url-fopen
@ -0,0 +1,2 @@
+# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" }
+66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)"
--- a/testcases/files/logs/postfix
+++ b/testcases/files/logs/postfix
@ -1,3 +1,12 @@
 # per https://github.com/fail2ban/fail2ban/issues/125
 # and https://github.com/fail2ban/fail2ban/issues/126
+# failJSON: { "time": "2005-02-21T09:21:54", "match": true , "host": "192.0.43.10" }
 Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo=
+# failJSON: { "time": "2005-07-12T07:47:48", "match": true , "host": "1.2.3.4" }
+Jul 12 07:47:48 saturn postfix/smtpd[8738]: NOQUEUE: reject: RCPT from 1-2-3-4-example.com[1.2.3.4]: 554 5.7.1 <smtp@example.com>: Relay access denied; from=<john@example.com> to=<smtp@example.org> proto=SMTP helo=<198.51.100.17>
+# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.65" }
+Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.65]: 554 5.7.1 <bad.domain>: Helo command rejected: match bad.domain; from=<foo@good.domain> to=<foo@porcupine.org> proto=SMTP helo=<bad.domain>
+# failJSON: { "time": "2005-07-18T23:12:56", "match": true , "host": "192.51.100.43" }
+Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 <foo@bad.domain>: Sender address rejected: match bad.domain; from=<foo@bad.domain> to=<foo@porcupine.org> proto=SMTP helo=<192.51.100.43>
+# failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" }
+Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab
--- a/testcases/files/logs/proftpd
+++ b/testcases/files/logs/proftpd
@ -1,9 +1,16 @@
+# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" }
 Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers
+# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" }
 Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 
+# failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" }
 Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password.
+# failJSON: { "time": "2005-06-09T11:15:43", "match": true , "host": "101.71.143.238" }
 Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21
+# failJSON: { "time": "2005-06-13T22:07:23", "match": true , "host": "59.167.242.100" }
 Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted.
+# failJSON: { "time": "2005-06-14T00:09:59", "match": true , "host": "59.167.242.100" }
 Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21
-
-
-
+# failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" }
+May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded 
+# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" }
+Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 
--- a/testcases/files/logs/pure-ftpd
+++ b/testcases/files/logs/pure-ftpd
@ -1,2 +1,4 @@
+# failJSON: { "time": "2005-01-31T16:54:07", "match": true , "host": "24.79.92.194" }
 Jan 31 16:54:07 desktop pure-ftpd: (?@24.79.92.194) [WARNING] Authentication failed for user [Administrator]
+# failJSON: { "time": "2004-11-05T18:54:02", "match": true , "host": "server202181210195.ixlink.net" }
 Nov 5 18:54:02 pure-ftpd: (?@server202181210195.ixlink.net) [WARNING] Authentication failed for user [Administrator] 
--- a/testcases/files/logs/qmail
+++ b/testcases/files/logs/qmail
@ -0,0 +1,4 @@
+# failJSON: { "time": "2004-09-06T07:33:33", "match": true , "host": "198.51.100.77" }
+Sep  6 07:33:33 sd6 qmail: 1157520813.485077 rblsmtpd: 198.51.100.77 pid 19597 sbl-xbl.spamhaus.org: 451 http://www.spamhaus.org/query/bl?ip=198.51.100.77
+# failJSON: { "time": "2004-09-06T07:18:29", "match": true , "host": "198.51.100.54" }
+Sep  6 07:18:29 sd6 qmail: 1157519909.633171 qmail-smtpd: 421 badiprbl: ip 198.51.100.54 rbl: example.com
--- a/testcases/files/logs/recidive
+++ b/testcases/files/logs/recidive
@ -0,0 +1,6 @@
+# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" }
+2006-02-13 15:52:30,388 fail2ban.actions: WARNING [sendmail] Ban 1.2.3.4
+# failJSON: { "match": false }
+2006-02-13 16:07:31,183 fail2ban.actions: WARNING [sendmail] Unban 1.2.3.4
+# failJSON: { "match": false }
+2006-02-13 15:52:30,388 fail2ban.actions: WARNING [recidive] Ban 1.2.3.4
--- a/testcases/files/logs/roundcube-auth
+++ b/testcases/files/logs/roundcube-auth
@ -1,2 +1,6 @@
+# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" }
 [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10
+# failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" }
 May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47
+# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" }
+Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login)
--- a/testcases/files/logs/sasl
+++ b/testcases/files/logs/sasl
@ -1,5 +1,7 @@
 #1 Example from postfix from dbts #507990
+# failJSON: { "time": "2004-12-02T22:24:22", "match": true , "host": "114.44.142.233" }
 Dec  2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.net[114.44.142.233]: SASL CRAM-MD5 authentication failed: PDc3OTEwNTkyNTEyMzA2NDIuMTIyODI1MzA2MUBoZWw+
 #2 Example from postfix from dbts #573314
+# failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" }
 Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure

--- a/testcases/files/logs/sieve
+++ b/testcases/files/logs/sieve
@ -0,0 +1,6 @@
+# failJSON: { "time": "2004-12-01T20:36:56", "match": true , "host": "1.2.3.4" }
+Dec 1 20:36:56 mail sieve[23713]: badlogin: example.com[1.2.3.4] PLAIN authentication failure
+# failJSON: { "time": "2005-07-18T17:21:58", "match": true , "host": "1.2.3.4" }
+Jul 18 17:21:58 ophelia cyrus/timsieved[12305]: badlogin: example.com[1.2.3.4] PLAIN authentication failure
+# failJSON: { "time": "2004-09-25T22:07:38", "match": true , "host": "1.2.3.4" }
+Sep 25 22:07:38 web9 timsieved[21040]: badlogin: web4[1.2.3.4] OTP authentication failure
--- a/testcases/files/logs/sogo-auth
+++ b/testcases/files/logs/sogo-auth
@ -1,17 +1,31 @@
 # yoh: Kept original apache log lines as well, just in case they might come useful
 #      for (testing) multiline regular expressions which would further constraint
 #      SOGo log lines
+# failJSON: { "match": false }
 Mar 24 08:58:32 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87c3008> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=hack0r,ou=users,dc=mail,dc=example,dc=org"; }
+# failJSON: { "time": "2005-03-24T08:58:32", "match": true , "host": "173.194.44.31" }
 Mar 24 08:58:32 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'hack0r' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+# failJSON: { "match": false }
 173.194.44.31 - - [24/Mar/2013:08:58:32 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/38 0.311 - - 2M
+# failJSON: { "match": false }
 Mar 24 08:58:40 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87bb8d8> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=kiddy,ou=users,dc=mail,dc=example,dc=org"; }
+# failJSON: { "time": "2005-03-24T08:58:40", "match": true , "host": "173.194.44.31" }
 Mar 24 08:58:40 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'kiddy' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+# failJSON: { "match": false }
 173.194.44.31 - - [24/Mar/2013:08:58:40 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/37 0.007 - - 32K
+# failJSON: { "match": false }
 Mar 24 08:58:50 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87c27f8> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=plsBanMe,ou=users,dc=mail,dc=example,dc=org"; }
+# failJSON: { "time": "2005-03-24T08:58:50", "match": true , "host": "173.194.44.31" }
 Mar 24 08:58:50 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'plsBanMe' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+# failJSON: { "match": false }
 173.194.44.31 - - [24/Mar/2013:08:58:50 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/40 0.008 - - 0
+# failJSON: { "match": false }
 Mar 24 08:58:59 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87be830> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=root,ou=users,dc=mail,dc=example,dc=org"; }
+# failJSON: { "time": "2005-03-24T08:58:59", "match": true , "host": "173.194.44.31" }
 Mar 24 08:58:59 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'root' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+# failJSON: { "match": false }
 173.194.44.31 - - [24/Mar/2013:08:58:59 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/36 0.007 - - 0
+# failJSON: { "match": false }
 Mar 24 08:59:04 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87bc088> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=admin,ou=users,dc=mail,dc=example,dc=org"; }
+# failJSON: { "time": "2005-03-24T08:59:04", "match": true , "host": "173.194.44.31" }
 Mar 24 08:59:04 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'admin' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
--- a/testcases/files/logs/sshd
+++ b/testcases/files/logs/sshd
@ -1,48 +1,87 @@
 #1
+# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.030.0.6" }
 Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6
+# failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" }
 May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com

 #2
+# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" }
 Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2
+# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" }
 Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345

 #3
+# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
 Jan  5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4
+# failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" }
 Jan  5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4

 #4
+# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "211.114.51.213" }
 Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 211.114.51.213

 #5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch
 # yoh: added ':' after [sshd] since the case without is not really common any more
+# failJSON: { "time": "2005-03-03T00:17:22", "match": true , "host": "211.188.220.49" }
 Mar  3 00:17:22 [sshd]: User root from 211.188.220.49 not allowed because not listed in AllowUsers
+# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "example.com" }
 Feb 25 14:34:11 belka sshd[31607]: User root from example.com not allowed because not listed in AllowUsers

 #6 ew filter introduced thanks to report Guido Bozzetto <reportbug@G-B.it>
+# failJSON: { "time": "2004-11-11T23:33:27", "match": true , "host": "218.249.210.161" }
 Nov 11 23:33:27 Server sshd[5174]: refused connect from _U2FsdGVkX19P3BCJmFBHhjLza8BcMH06WCUVwttMHpE=_@::ffff:218.249.210.161 (::ffff:218.249.210.161)

 #7 added exclamation mark to BREAK-IN
 #  Now should be a negative since we decided not to catch those
+# failJSON: { "match": false }
 Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT
+# failJSON: { "match": false }
 Oct 15 19:51:35 server sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT!

 #8 DenyUsers https://github.com/fail2ban/fail2ban/issues/47
+# failJSON: { "time": "2005-04-16T22:01:15", "match": true , "host": "46.45.128.3" }
 Apr 16 22:01:15 al-ribat sshd[5154]: User root from 46.45.128.3 not allowed because listed in DenyUsers

 #9 OpenSolaris patch - pull https://github.com/fail2ban/fail2ban/pull/182
+# failJSON: { "time": "2005-03-29T05:59:23", "match": true , "host": "205.186.180.55" }
 Mar 29 05:59:23 dusky sshd[20878]: [ID 800047 auth.info] Failed keyboard-interactive for <invalid username> from 205.186.180.55 port 42742 ssh2
+# failJSON: { "time": "2005-03-29T05:20:09", "match": true , "host": "205.186.180.30" }
 Mar 29 05:20:09 dusky sshd[19558]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.30 port 54520 ssh2

 #10 OSX syslog error
+# failJSON: { "time": "2005-04-29T17:16:20", "match": true , "host": "example.com" }
 Apr 29 17:16:20 Jamess-iMac.local sshd[62312]: error: PAM: authentication error for james from example.com via 192.168.1.201
+# failJSON: { "time": "2005-04-29T20:11:08", "match": true , "host": "205.186.180.35" }
 Apr 29 20:11:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for <invalid username> from 205.186.180.35 port 42742 ssh2
+# failJSON: { "time": "2005-04-29T20:12:08", "match": true , "host": "205.186.180.22" }
 Apr 29 20:12:08 Jamess-iMac.local sshd[63814]: [ID 800047 auth.info] Failed keyboard-interactive for james from 205.186.180.22 port 54520 ssh2
+# failJSON: { "time": "2005-04-29T20:13:08", "match": true , "host": "205.186.180.42" }
 Apr 29 20:13:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for james from 205.186.180.42 port 54520 ssh2
+# failJSON: { "time": "2005-04-29T20:14:08", "match": true , "host": "205.186.180.44" }
 Apr 29 20:14:08 Jamess-iMac.local sshd[63814]: Failed keyboard-interactive for <invalid username> from 205.186.180.44 port 42742 ssh2
+# failJSON: { "time": "2005-04-30T01:42:12", "match": true , "host": "205.186.180.77" }
 Apr 30 01:42:12 Jamess-iMac.local sshd[2554]: Failed keyboard-interactive/pam for invalid user jamedds from 205.186.180.77 port 33723 ssh2
+# failJSON: { "time": "2005-04-29T12:53:38", "match": true , "host": "205.186.180.88" }
 Apr 29 12:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication failure for james from 205.186.180.88 via 192.168.1.201
+# failJSON: { "time": "2005-04-29T13:53:38", "match": true , "host": "205.186.180.99" }
 Apr 29 13:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication failure for james from 205.186.180.99 via 192.168.1.201
+# failJSON: { "time": "2005-04-29T15:53:38", "match": true , "host": "205.186.180.100" }
 Apr 29 15:53:38 Jamess-iMac.local sshd[47831]: error: PAM: Authentication error for james from 205.186.180.100 via 192.168.1.201
+# failJSON: { "time": "2005-04-29T16:53:38", "match": true , "host": "205.186.180.101" }
 Apr 29 16:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.101 via 192.168.1.201
+# failJSON: { "time": "2005-04-29T17:53:38", "match": true , "host": "205.186.180.102" }
 Apr 29 17:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.102
+# failJSON: { "time": "2005-04-29T18:53:38", "match": true , "host": "205.186.180.103" }
 Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.103
+
+#11 https://github.com/fail2ban/fail2ban/issues/267  There might be no colon after [daemon]
+# failJSON: { "time": "2005-06-25T23:53:34", "match": true , "host": "1.2.3.4" }
+Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers
+# failJSON: { "time": "2004-12-12T20:04:39", "match": true , "host": "10.215.4.227" }
+Dec 12 20:04:39 aragorn sshd[1328]: error: PAM: User not known to the underlying authentication module for illegal user kernelitshell from 10.215.4.227 
+# failJSON: { "time": "2005-03-26T04:56:27", "match": true , "host": "example.com" }
+Mar 26 04:56:27 angel sshd[9739]: User allena from example.com not allowed because not in any group
+# failJSON: { "time": "2005-02-07T16:01:07", "match": true , "host": "192.51.100.54" }
+Feb 7 16:01:07 linux-m899 sshd[5106]: User root from 192.51.100.54 not allowed because a group is listed in DenyGroups
+# failJSON: { "time": "2005-01-05T11:15:05", "match": true , "host": "10.0.0.40" }
+Jan 5 11:15:05 NAS sshd[1966]: User root from 10.0.0.40 not allowed because none of user's groups are listed in AllowGroups
--- a/testcases/files/logs/sshd-ddos
+++ b/testcases/files/logs/sshd-ddos
@ -1,2 +1,3 @@
 # http://forums.powervps.com/showthread.php?t=1667
+# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" }
 Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114
--- a/testcases/files/logs/suhosin
+++ b/testcases/files/logs/suhosin
@ -0,0 +1,4 @@
+# failJSON: { "time": "2005-03-11T22:52:12", "match": true , "host": "198.51.100.167" }
+Mar 11 22:52:12   lighttpd[53690]: (mod_fastcgi.c.2676) FastCGI-stderr: ALERT - configured request variable name length limit exceeded - dropped variable 'upqchi07vFfAFuBjnIKGIwiLrHo3Vt68T3yqvhQu2TqetQ78roy7Q6bpTfDUtYFR593/MA' (attacker '198.51.100.167', file '/usr/local/captiveportal/index.php')
+# failJSON: { "time": "2005-02-26T22:52:29", "match": true , "host": "198.51.100.77" }
+Feb 26 22:52:29 host suhosin[9636]: ALERT - script tried to increase memory_limit to 268435456 bytes which is above the allowed value (attacker '198.51.100.77', file '/var/www/wordpress/wp-admin/includes/image.php', line 161)
--- a/testcases/files/logs/vsftpd
+++ b/testcases/files/logs/vsftpd
@ -1,7 +1,10 @@
 #1 PAM based
+# failJSON: { "time": "2004-10-11T01:06:47", "match": true , "host": "209.67.1.67" }
 Oct 11 01:06:47 ServerJV vsftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=209.67.1.67
+# failJSON: { "time": "2005-02-06T12:02:29", "match": true , "host": "64.168.103.1" }
 Feb 6 12:02:29 server vsftpd(pam_unix)[15522]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=64.168.103.1 user=user1

 #2 Internal
+# failJSON: { "time": "2005-01-19T12:20:33", "match": true , "host": "64.106.46.98" }
 Fri Jan 19 12:20:33 2007 [pid 27202] [anonymous] FAIL LOGIN: Client "64.106.46.98"

--- a/testcases/files/logs/webmin-auth
+++ b/testcases/files/logs/webmin-auth
@ -1,7 +1,9 @@
 #Webmin authentication failures from /var/log/auth.log

 #1 User exists, bad password
+# failJSON: { "time": "2004-12-13T08:15:18", "match": true , "host": "89.2.49.230" }
 Dec 13 08:15:18 sb1 webmin[25875]: Invalid login as root from 89.2.49.230 

 #2 User does not exists
+# failJSON: { "time": "2004-12-12T23:14:19", "match": true , "host": "188.40.105.142" }
 Dec 12 23:14:19 sb1 webmin[22134]: Non-existent login as robert from 188.40.105.142 
--- a/testcases/files/logs/wu-ftpd
+++ b/testcases/files/logs/wu-ftpd
@ -1,2 +0,0 @@
-# This login line is from syslog
-Oct  6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173]
--- a/testcases/files/logs/wuftpd
+++ b/testcases/files/logs/wuftpd
@ -0,0 +1,5 @@
+# This login line is from syslog
+# failJSON: { "time": "2004-10-06T09:59:26", "match": true , "host": "202.108.145.173" }
+Oct  6 09:59:26 myserver wu-ftpd[18760]: failed login from hj-145-173-a8.bta.net.cn [202.108.145.173]
+# failJSON: { "time": "2004-10-11T16:45:07", "match": true , "host": "198.51.100.71" }
+Oct 11 16:45:07 ubuntu wu-ftpd[2360]: failed login from example.com [198.51.100.71]
--- a/testcases/files/logs/xinetd-fail
+++ b/testcases/files/logs/xinetd-fail
@ -0,0 +1,4 @@
+# failJSON: { "time": "2005-05-15T17:38:49", "match": true , "host": "198.51.100.169" }
+May 15 17:38:49 boo xinetd[16256]: FAIL: telnet address from=198.51.100.169
+# failJSON: { "time": "2005-08-03T14:38:49", "match": true , "host": "198.51.100.223" }
+Aug  3 14:38:49 backup xinetd[31234]: FAIL: amanda libwrap from=198.51.100.223
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@ -39,6 +39,8 @@ from server.failmanager import FailManagerEmpty
 # Useful helpers
 #

+from utils import mtimesleep
+
 # yoh: per Steven Hiscocks's insight while troubleshooting
 # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836
 # adding a sufficiently large buffer might help to guarantee that
@ -68,18 +70,6 @@ def _killfile(f, name):
 		_killfile(None, name + '.bak')


-def _sleep_4_poll():
-	"""PollFilter relies on file timestamps - so we might need to
-	sleep to guarantee that they differ
-	"""
-	if sys.version_info[:2] <= (2,4):
-		# on old Python st_mtime is int, so we should give
-		# at least 1 sec so polling filter could detect
-		# the change
-		time.sleep(1.)
-	else:
-		time.sleep(0.1)
-
 def _assert_equal_entries(utest, found, output, count=None):
 	"""Little helper to unify comparisons with the target entries

@ -119,10 +109,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line

 	Returns open fout
 	"""
-	if sys.version_info[:2] <= (2,4): # pragma: no cover
-		# on old Python st_mtime is int, so we should give at least 1 sec so
-		# polling filter could detect the change
-		time.sleep(1)
+	# on older Pythons and on some FSs st_mtime is int, so we should
+	# give it some time so polling filter could detect the change
+	mtimesleep()
 	if isinstance(fin, str): # pragma: no branch - only used with str in test cases
 		fin = open(fin, 'r')
 	# Skip
@ -237,14 +226,14 @@ class LogFileMonitor(unittest.TestCase):
 		# but not any longer
 		self.assertTrue(self.notModified())
 		self.assertTrue(self.notModified())
-		_sleep_4_poll()				# to guarantee freshier mtime
+		mtimesleep()				# to guarantee freshier mtime
 		for i in range(4):			  # few changes
 			# unless we write into it
 			self.file.write("line%d\n" % i)
 			self.file.flush()
 			self.assertTrue(self.isModified())
 			self.assertTrue(self.notModified())
-			_sleep_4_poll()				# to guarantee freshier mtime
+			mtimesleep()				# to guarantee freshier mtime
 		os.rename(self.name, self.name + '.old')
 		# we are not signaling as modified whenever
 		# it gets away
@ -252,7 +241,7 @@ class LogFileMonitor(unittest.TestCase):
 		f = open(self.name, 'a')
 		self.assertTrue(self.isModified())
 		self.assertTrue(self.notModified())
-		_sleep_4_poll()
+		mtimesleep()
 		f.write("line%d\n" % i)
 		f.flush()
 		self.assertTrue(self.isModified())
@ -398,7 +387,7 @@ def get_monitor_failures_testcase(Filter_):
 			# actions might be happening too fast in the tests,
 			# sleep a bit to guarantee reliable time stamps
 			if isinstance(self.filter, FilterPoll):
-				_sleep_4_poll()
+				mtimesleep()

 		def isEmpty(self, delay=0.4):
 			# shorter wait time for not modified status
@ -458,7 +447,9 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  n=14, mode='w')
 			# Poll might need more time
-			self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2))
+			self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2),
+							"Queue must be empty but it is not: %s."
+							% (', '.join([str(x) for x in self.jail.queue])))
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 2)

--- a/testcases/samplestestcase.py
+++ b/testcases/samplestestcase.py
@ -0,0 +1,134 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Fail2Ban developers
+
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import unittest, sys, os, fileinput, re, datetime, inspect
+
+if sys.version_info >= (2, 6):
+	import json
+else:
+	import simplejson as json
+	next = lambda x: x.next()
+
+from server.filter import Filter
+from client.filterreader import FilterReader
+
+TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
+CONFIG_DIR = "config"
+
+class FilterSamplesRegex(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		self.filter = Filter(None)
+		self.filter.setActive(True)
+
+	def tearDown(self):
+		"""Call after every test case."""
+
+	def testFiltersPresent(self):
+		"""Check to ensure some tests exist"""
+		self.assertTrue(
+			len([test for test in inspect.getmembers(self)
+				if test[0].startswith('testSampleRegexs')])
+			>= 10,
+			"Expected more FilterSampleRegexs tests")
+
+def testSampleRegexsFactory(name):
+	def testFilter(self):
+
+		# Check filter exists
+		filterConf = FilterReader(name, "jail", basedir=CONFIG_DIR)
+		filterConf.read()
+		filterConf.getOptions({})
+
+		for opt in filterConf.convert():
+			if opt[2] == "addfailregex":
+				self.filter.addFailRegex(opt[3])
+			elif opt[2] == "addignoreregex":
+				self.filter.addIgnoreRegex(opt[3])
+
+		if not self.filter.getFailRegex():
+			# No fail regexs set: likely just common file for includes.
+			return
+
+		self.assertTrue(
+			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
+			"No sample log file available for '%s' filter" % name)
+
+		logFile = fileinput.FileInput(
+			os.path.join(TEST_FILES_DIR, "logs", name))
+
+		regexsUsed = set()
+		for line in logFile:
+			jsonREMatch = re.match("^# ?failJSON:(.+)$", line)
+			if jsonREMatch:
+				try:
+					faildata = json.loads(jsonREMatch.group(1))
+				except ValueError, e:
+					raise ValueError("%s: %s:%i" %
+						(e, logFile.filename(), logFile.filelineno()))
+				line = next(logFile)
+			elif line.startswith("#") or not line.strip():
+				continue
+			else:
+				faildata = {}
+
+			ret = self.filter.processLine(
+				line, returnRawHost=True, checkAllRegex=True)
+			if not ret:
+				# Check line is flagged as none match
+				self.assertFalse(faildata.get('match', True),
+					 "Line not matched when should have: %s:%i %r" %
+					(logFile.filename(), logFile.filelineno(), line))
+			elif ret:
+				# Check line is flagged to match
+				self.assertTrue(faildata.get('match', False),
+					"Line matched when shouldn't have: %s:%i %r" %
+					(logFile.filename(), logFile.filelineno(), line))
+				self.assertEqual(len(ret), 1, "Multiple regexs matched")
+				# Verify timestamp and host as expected
+				failregex, host, time = ret[0]
+				self.assertEqual(host, faildata.get("host", None))
+				self.assertEqual(
+					datetime.datetime.fromtimestamp(time),
+					datetime.datetime.strptime(
+						faildata.get("time", None), "%Y-%m-%dT%H:%M:%S"))
+
+				regexsUsed.add(failregex)
+
+		# TODO: Remove exception handling once all regexs have samples
+		for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()):
+			self.assertTrue(
+				failRegexIndex in regexsUsed,
+				"Regex for filter '%s' has no samples: %i: %r" %
+					(name, failRegexIndex, failRegex))
+
+	return testFilter
+
+for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")):
+	filterName = filter_.rpartition(".")[0]
+	setattr(
+		FilterSamplesRegex,
+		"testSampleRegexs%s" % filterName.upper(),
+		testSampleRegexsFactory(filterName))
--- a/testcases/servertestcase.py
+++ b/testcases/servertestcase.py
@ -334,9 +334,9 @@ class Transmitter(TransmitterBase):
 				"failed attempt from <HOST> again",
 			],
 			[
-				"user john at (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)",
-				"failed attempt from (?:::f{4,6}:)?(?P<host>[\w\-.^_]+) again",
+				"user john at (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
+				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
+				"failed attempt from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w) again",
 			],
 			self.jailName
 		)
@ -359,7 +359,7 @@ class Transmitter(TransmitterBase):
 			],
 			[
 				"user john",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)",
+				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
 				"Dont match me!",
 			],
 			self.jailName
--- a/testcases/utils.py
+++ b/testcases/utils.py
@ -22,7 +22,7 @@ __author__ = "Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko"
 __license__ = "GPL"

-import logging, os, re, traceback
+import logging, os, re, tempfile, sys, time, traceback
 from os.path import basename, dirname

 #
@ -100,3 +100,8 @@ class FormatterWithTraceBack(logging.Formatter):
 	def format(self, record):
 		record.tbc = record.tb = self._tb()
 		return logging.Formatter.format(self, record)
+
+def mtimesleep():
+	# no sleep now should be necessary since polling tracks now not only
+	# mtime but also ino and size
+	pass