Merge branch 'ipv6-support-0.10' into 0.10

ChangeLog

@@ -92,6 +92,14 @@ ver. 0.9.4 (2016/03/08) - for-you-ladies
    * sshd filter got new failregex to match "maximum authentication
      attempts exceeded" (introduced in openssh 6.8)
    * Added filter for Mac OS screen sharing (VNC) daemon
+   * IPv6 support:
+     - IP addresses are now handled as objects rather than strings capable for 
+       handling both address types IPv4 and IPv6
+     - iptables related actions have been amended to support IPv6 specific actions
+       additionally
+     - hostsdeny and route actions have been tested to be aware of v4 and v6 already
+     - pf action for *BSD systems has been improved and supports now also v4 and v6
+     - Name resolution is now working for either address type
 
 - Enhancements:
    * Do not rotate empty log files

MANIFEST

@@ -164,6 +164,7 @@ fail2ban/client/jailreader.py
 fail2ban/client/jailsreader.py
 fail2ban/exceptions.py
 fail2ban/helpers.py
+fail2ban/ipdns.py
 fail2ban/__init__.py
 fail2ban/protocol.py
 fail2ban/server/action.py
@@ -200,6 +201,7 @@ fail2ban/tests/actionstestcase.py
 fail2ban/tests/actiontestcase.py
 fail2ban/tests/banmanagertestcase.py
 fail2ban/tests/clientreadertestcase.py
+fail2ban/tests/clientbeautifiertestcase.py
 fail2ban/tests/config/action.d/brokenaction.conf
 fail2ban/tests/config/fail2ban.conf
 fail2ban/tests/config/filter.d/simple.conf

THANKS

@@ -12,6 +12,7 @@ Adrien Clerc
 ache
 ag4ve (Shawn)
 Alasdair D. Campbell
+Alexander Koeppe (IPv6 support)
 Alexandre Perrin (kAworu)
 Amir Caspi
 Amy

config/action.d/iptables-common.conf

@@ -6,6 +6,9 @@
 # used in all iptables based actions by default.
 #
 # The user can override the defaults in iptables-common.local
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -13,6 +16,7 @@ after = iptables-blocktype.local
         iptables-common.local
 # iptables-blocktype.local is obsolete
 
+
 [Init]
 
 # Option:  chain
@@ -62,3 +66,19 @@ lockingopt = -w
 # Notes.:  Actual command to be executed, including common to all calls options
 # Values:  STRING
 iptables = iptables <lockingopt>
+
+
+[Init?family=inet6]
+
+# Option:  blocktype (ipv6)
+# Note:    This is what the action does with rules. This can be any jump target
+#          as per the iptables man page (section 8). Common values are DROP
+#          REJECT, REJECT --reject-with icmp6-port-unreachable
+# Values:  STRING
+blocktype = REJECT --reject-with icmp6-port-unreachable
+
+# Option:  iptables (ipv6)
+# Notes.:  Actual command to be executed, including common to all calls options
+# Values:  STRING
+iptables = ip6tables <lockingopt>
+

config/action.d/iptables-ipset-proto6-allports.conf

@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules which probably won't be protocol version 6.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -m set --match-set f2b-<name> src -j <blockty
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6

config/action.d/iptables-ipset-proto6.conf

@@ -12,6 +12,9 @@
 #
 # If you are running on an older kernel you make need to patch in external
 # modules.
+#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable (see new section Init?family=inet6)
 
 [INCLUDES]
 
@@ -23,16 +26,16 @@ before = iptables-common.conf
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = ipset create f2b-<name> hash:ip timeout <bantime>
-              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
+actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
+              <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset flush f2b-<name>
-             ipset destroy f2b-<name>
+actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
+             ipset flush <ipmset>
+             ipset destroy <ipmset>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -40,7 +43,7 @@ actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
+actionban = ipset add <ipmset> <ip> timeout <bantime> -exist
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -48,7 +51,7 @@ actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = ipset del f2b-<name> <ip> -exist
+actionunban = ipset del <ipmset> <ip> -exist
 
 [Init]
 
@@ -57,3 +60,12 @@ actionunban = ipset del f2b-<name> <ip> -exist
 # Values:  [ NUM ]  Default: 600
 #
 bantime = 600
+
+ipmset = f2b-<name>
+familyopt =
+
+
+[Init?family=inet6]
+
+ipmset = f2b-<name>6
+familyopt = <sp>family inet6

config/action.d/iptables-xt_recent-echo.conf

@@ -2,7 +2,8 @@
 #
 # Author: Zbigniew Jędrzejewski-Szmek <zbyszek@in.waw.pl>
 #
-#
+# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de>
+#       made config file IPv6 capable
 
 [INCLUDES]
 
@@ -22,30 +23,30 @@ before = iptables-common.conf
 # iptables-persistent package).
 # 
 # Explanation of the rule below:
-#    Check if any packets coming from an IP on the f2b-<name>
+#    Check if any packets coming from an IP on the <iptname>
 #    list have been seen in the last 3600 seconds. If yes, update the
 #    timestamp for this IP and drop the packet. If not, let the packet
 #    through.
 #
-#    Fail2ban inserts blacklisted hosts into the f2b-<name> list
+#    Fail2ban inserts blacklisted hosts into the <iptname> list
 #    and removes them from the list after some time, according to its
 #    own rules. The 3600 second timeout is independent and acts as a
 #    safeguard in case the fail2ban process dies unexpectedly. The
 #    shorter of the two timeouts actually matters.
-actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = echo / > /proc/net/xt_recent/f2b-<name>
-             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name f2b-<name> -j <blocktype>;fi
+actionstop = echo / > /proc/net/xt_recent/<iptname>
+             if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = test -e /proc/net/xt_recent/f2b-<name>
+actioncheck = test -e /proc/net/xt_recent/<iptname>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -53,7 +54,7 @@ actioncheck = test -e /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
+actionban = echo +<ip> > /proc/net/xt_recent/<iptname>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -61,7 +62,12 @@ actionban = echo +<ip> > /proc/net/xt_recent/f2b-<name>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = echo -<ip> > /proc/net/xt_recent/f2b-<name>
+actionunban = echo -<ip> > /proc/net/xt_recent/<iptname>
 
 [Init]
 
+iptname              = f2b-<name>
+
+[Init?family=inet6]
+
+iptname = f2b-<name>6

config/action.d/pf.conf

@@ -3,6 +3,7 @@
 # OpenBSD pf ban/unban
 #
 # Author: Nick Hilliard <nick@foobar.org>
+# Modified by: Alexander Koeppe making PF work seamless and with IPv4 and IPv6
 #
 #
 
@@ -12,23 +13,27 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-# we don't enable PF automatically, as it will be enabled elsewhere
-actionstart = 
+# we don't enable PF automatically; to enable run pfctl -e 
+# or add `pf_enable="YES"` to /etc/rc.conf (tested on FreeBSD)
+actionstart = echo "table <<tablename>-<name>> persist counters" | pfctl -f- 
+              echo "block proto <protocol> from <<tablename>-<name>> to any port <port>" | pfctl -f-
 
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-# we don't disable PF automatically either
-actionstop = 
+# we only disable PF rules we've installed prior
+actionstop = pfctl -sr 2>/dev/null | grep -v <tablename>-<name> | pfctl -f-
+             pfctl -t <tablename>-<name> -T flush
+             pfctl -t <tablename>-<name> -T kill
 
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
 # Values:  CMD
 #
-actioncheck = 
+actioncheck = pfctl -sr | grep -q <tablename>-<name>
 
 
 # Option:  actionban
@@ -39,7 +44,7 @@ actioncheck =
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
+actionban = pfctl -t <tablename>-<name> -T add <ip>
 
 
 # Option:  actionunban
@@ -51,12 +56,24 @@ actionban = /sbin/pfctl -t <tablename> -T add <ip>/32
 # Values:  CMD
 #
 # note -r option used to remove matching rule
-actionunban = /sbin/pfctl -t <tablename> -T delete <ip>/32
+actionunban = pfctl -t <tablename>-<name> -T delete <ip>
 
 [Init]
 # Option:  tablename
 # Notes.:  The pf table name.
 # Values:  [ STRING ]
 #
-tablename = fail2ban
+tablename = f2b
+
+# Option:  protocol
+# Notes.:  internally used by config reader for interpolations.
+# Values:  [ tcp | udp | icmp | ipv6-icmp ] Default: tcp
+#
+protocol = tcp
+
+# Option:  port
+# Notes.:  the port to block, defaults to any
+# Values:  [ STRING ]
+#
+port = any
 

config/filter.d/ignorecommands/apache-fakegooglebot

@@ -14,7 +14,7 @@ def process_args(argv):
 
     ip = argv[1]
 
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
     if not DNSUtils.isValidIP(ip):
        sys.stderr.write("Argument must be a single valid IP. Got: %s\n"
                         % ip)
@@ -23,7 +23,7 @@ def process_args(argv):
 
 def is_googlebot(ip):
     import re
-    from fail2ban.server.filter import DNSUtils
+    from fail2ban.server.ipdns import DNSUtils
 
     host = DNSUtils.ipToName(ip)
     if not host or not re.match('.*\.google(bot)?\.com$', host):

config/jail.conf

@@ -47,7 +47,7 @@ before = paths-debian.conf
 # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
 # ban a host which matches an address in this list. Several addresses can be
 # defined using space (and/or comma) separator.
-ignoreip = 127.0.0.1/8
+ignoreip = 127.0.0.1/8 ::1
 
 # External command that will take an tagged arguments to ignore, e.g. <ip>,
 # and return true if the IP is to be ignored. False otherwise.

config/paths-common.conf

@@ -40,7 +40,8 @@ lighttpd_error_log = /var/log/lighttpd/error.log
 # http://www.hardened-php.net/suhosin/configuration.html#suhosin.log.syslog.facility
 # syslog_user is the default. Lighttpd also hooks errors into its log.
 
-suhosin_log = %(syslog_user)s %(lighttpd_error_log)s
+suhosin_log = %(syslog_user)s
+              %(lighttpd_error_log)s
 
 # defaults to ftp or local2 if ftp doesn't exist
 proftpd_log = %(syslog_ftp)s

fail2ban/client/actionreader.py

@@ -35,13 +35,13 @@ logSys = getLogger(__name__)
 
 class ActionReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "actionstart", None],
-		["string", "actionstop", None],
-		["string", "actioncheck", None],
-		["string", "actionban", None],
-		["string", "actionunban", None],
-	]
+	_configOpts = {
+		"actionstart": ["string", None],
+		"actionstop": ["string", None],
+		"actioncheck": ["string", None],
+		"actionban": ["string", None],
+		"actionunban": ["string", None],
+	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
 		self._name = initOpts.get("actname", file_)
@@ -65,20 +65,16 @@ class ActionReader(DefinitionInitConfigReader):
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
-		head.extend(["action", self._name])
-		for opt in self._opts:
-			if opt == "actionstart":
-				stream.append(head + ["actionstart", self._opts[opt]])
-			elif opt == "actionstop":
-				stream.append(head + ["actionstop", self._opts[opt]])
-			elif opt == "actioncheck":
-				stream.append(head + ["actioncheck", self._opts[opt]])
-			elif opt == "actionban":
-				stream.append(head + ["actionban", self._opts[opt]])
-			elif opt == "actionunban":
-				stream.append(head + ["actionunban", self._opts[opt]])
+		multi = []
+		for opt, optval in self._opts.iteritems():
+			if opt in self._configOpts:
+				multi.append([opt, optval])
 		if self._initOpts:
-			for p in self._initOpts:
-				stream.append(head + [p, self._initOpts[p]])
+			for opt, optval in self._initOpts.iteritems():
+				multi.append([opt, optval])
+		if len(multi) > 1:
+			stream.append(["multi-set", self._jailName, "action", self._name, multi])
+		elif len(multi):
+			stream.append(["set", self._jailName, "action", self._name] + multi[0])
 
 		return stream

fail2ban/client/beautifier.py

@@ -78,56 +78,56 @@ class Beautifier:
 						prefix1 = "   " if n == len(response) - 1 else "|  "
 						for m, res2 in enumerate(res1[1]):
 							prefix2 = prefix1 + ("`-" if m == len(res1[1]) - 1 else "|-")
-							val = " ".join(res2[1]) if isinstance(res2[1], list) else res2[1]
+							val = " ".join(map(str, res2[1])) if isinstance(res2[1], list) else res2[1]
 							msg.append("%s %s:\t%s" % (prefix2, res2[0], val))
 				else:
 					msg = ["Status"]
 					for n, res1 in enumerate(response):
 						prefix1 = "`-" if n == len(response) - 1 else "|-"
-						val = " ".join(res1[1]) if isinstance(res1[1], list) else res1[1]
+						val = " ".join(map(str, res1[1])) if isinstance(res1[1], list) else res1[1]
 						msg.append("%s %s:\t%s" % (prefix1, res1[0], val))
 				msg = "\n".join(msg)
 			elif inC[1] == "syslogsocket":
 				msg = "Current syslog socket is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1] == "logtarget":
 				msg = "Current logging target is:\n"
-				msg = msg + "`- " + response
+				msg += "`- " + response
 			elif inC[1:2] == ['loglevel']:
 				msg = "Current logging level is "
 				if response == 1:
-					msg = msg + "ERROR"
+					msg += "ERROR"
 				elif response == 2:
-					msg = msg + "WARN"
+					msg += "WARN"
 				elif response == 3:
-					msg = msg + "INFO"
+					msg += "INFO"
 				elif response == 4:
-					msg = msg + "DEBUG"
+					msg += "DEBUG"
 				else:
-					msg = msg + repr(response)
+					msg += repr(response)
 			elif inC[1] == "dbfile":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database file is:\n"
-					msg = msg + "`- " + response
+					msg += "`- " + response
 			elif inC[1] == "dbpurgeage":
 				if response is None:
 					msg = "Database currently disabled"
 				else:
 					msg = "Current database purge age is:\n"
-					msg = msg + "`- %iseconds" % response
+					msg += "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
 				else:
 					msg = "Current monitored log file(s):\n"
 					for path in response[:-1]:
-						msg = msg + "|- " + path + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + path + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] == "logencoding":
 				msg = "Current log encoding is set to:\n"
-				msg = msg + response
+				msg += response
 			elif inC[2] in ("journalmatch", "addjournalmatch", "deljournalmatch"):
 				if len(response) == 0:
 					msg = "No journal match filter set"
@@ -137,19 +137,19 @@ class Beautifier:
 			elif inC[2] == "datepattern":
 				msg = "Current date pattern set to: "
 				if response is None:
-					msg = msg + "Not set/required"
+					msg += "Not set/required"
 				elif response[0] is None:
-					msg = msg + "%s" % response[1]
+					msg += "%s" % response[1]
 				else:
-					msg = msg + "%s (%s)" % response
+					msg += "%s (%s)" % response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"
 				else:
 					msg = "These IP addresses/networks are ignored:\n"
 					for ip in response[:-1]:
-						msg = msg + "|- " + ip + "\n"
-					msg = msg + "`- " + response[len(response)-1]
+						msg += "|- " + ip + "\n"
+					msg += "`- " + response[-1]
 			elif inC[2] in ("failregex", "addfailregex", "delfailregex",
 							"ignoreregex", "addignoreregex", "delignoreregex"):
 				if len(response) == 0:
@@ -157,10 +157,10 @@ class Beautifier:
 				else:
 					msg = "The following regular expression are defined:\n"
 					c = 0
-					for ip in response[:-1]:
-						msg = msg + "|- [" + str(c) + "]: " + ip + "\n"
+					for l in response[:-1]:
+						msg += "|- [" + str(c) + "]: " + l + "\n"
 						c += 1
-					msg = msg + "`- [" + str(c) + "]: " + response[len(response)-1]
+					msg += "`- [" + str(c) + "]: " + response[-1]
 			elif inC[2] == "actions":
 				if len(response) == 0:
 					msg = "No actions for jail %s" % inC[1]
@@ -187,7 +187,7 @@ class Beautifier:
 			logSys.warning("Beautifier error. Please report the error")
 			logSys.error("Beautify " + repr(response) + " with "
 				+ repr(self.__inputCmd) + " failed")
-			msg = msg + repr(response)
+			msg += repr(response)
 		return msg
 
 	def beautifyError(self, response):

fail2ban/client/configparserinc.py

@@ -25,6 +25,7 @@ __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko'
 __license__ = 'GPL'
 
 import os
+import re
 import sys
 from ..helpers import getLogger
 
@@ -99,6 +100,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
+
 	if sys.version_info >= (3,2):
 		# overload constructor only for fancy new Python3's
 		def __init__(self, share_config=None, *args, **kwargs):
@@ -225,21 +228,31 @@ after = 1.conf
 					# merge defaults and all sections to self:
 					alld.update(cfg.get_defaults())
 					for n, s in cfg.get_sections().iteritems():
-						if isinstance(s, dict):
-							s2 = alls.get(n)
-							if isinstance(s2, dict):
-								# save previous known values, for possible using in local interpolations later:
-								sk = {}
-								for k, v in s2.iteritems():
-									if not k.startswith('known/'):
-										sk['known/'+k] = v
-								s2.update(sk)
-								# merge section
-								s2.update(s)
-							else:
-								alls[n] = s.copy()
+						curalls = alls
+						# conditional sections
+						cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+						if cond:
+							n, cond = cond.groups()
+							s = s.copy()
+							try: 
+								del(s['__name__'])
+							except KeyError:
+								pass
+							for k in s.keys():
+								v = s.pop(k)
+								s[k + cond] = v
+						s2 = alls.get(n)
+						if isinstance(s2, dict):
+							# save previous known values, for possible using in local interpolations later:
+							sk = {}
+							for k, v in s2.iteritems():
+								if not k.startswith('known/') and k != '__name__':
+									sk['known/'+k] = v
+							s2.update(sk)
+							# merge section
+							s2.update(s)
 						else:
-							alls[n] = s
+							alls[n] = s.copy()
 
 			return ret
 
@@ -254,9 +267,12 @@ after = 1.conf
 
 	def merge_section(self, section, options, pref='known/'):
 		alls = self.get_sections()
+		if pref == '':
+			alls[section].update(options)
+			return
 		sk = {}
 		for k, v in options.iteritems():
-			if pref == '' or not k.startswith(pref):
+			if not k.startswith(pref) and k != '__name__':
 				sk[pref+k] = v
 		alls[section].update(sk)
 

fail2ban/client/configreader.py

@@ -203,40 +203,47 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 	#
 	# Read the given option in the configuration file. Default values
 	# are used...
-	# Each optionValues entry is composed of an array with:
-	# 0 -> the type of the option
-	# 1 -> the name of the option
-	# 2 -> the default value for the option
+	# Each options entry is composed of an array with:
+	#  [[type, name, default], ...]
+	# Or it is a dict:
+	#  {name: [type, default], ...}
 	
 	def getOptions(self, sec, options, pOptions=None):
 		values = dict()
-		for option in options:
-			try:
-				if option[0] == "bool":
-					v = self.getboolean(sec, option[1])
-				elif option[0] == "int":
-					v = self.getint(sec, option[1])
+		for optname in options:
+			if isinstance(options, (list,tuple)):
+				if len(optname) > 2:
+					opttype, optname, optvalue = optname
 				else:
-					v = self.get(sec, option[1])
-				if not pOptions is None and option[1] in pOptions:
+					(opttype, optname), optvalue = optname, None
+			else:
+				opttype, optvalue = options[optname]
+			try:
+				if opttype == "bool":
+					v = self.getboolean(sec, optname)
+				elif opttype == "int":
+					v = self.getint(sec, optname)
+				else:
+					v = self.get(sec, optname)
+				if not pOptions is None and optname in pOptions:
 					continue
-				values[option[1]] = v
+				values[optname] = v
 			except NoSectionError, e:
 				# No "Definition" section or wrong basedir
 				logSys.error(e)
-				values[option[1]] = option[2]
+				values[optname] = optvalue
 				# TODO: validate error handling here.
 			except NoOptionError:
-				if not option[2] is None:
+				if not optvalue is None:
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
-								% (option[1], sec, option[2]))
-					values[option[1]] = option[2]
+								% (optname, sec, optvalue))
+					values[optname] = optvalue
 				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec)
+					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec)
 			except ValueError:
-				logSys.warning("Wrong value for '" + option[1] + "' in '" + sec +
-							"'. Using default one: '" + repr(option[2]) + "'")
-				values[option[1]] = option[2]
+				logSys.warning("Wrong value for '" + optname + "' in '" + sec +
+							"'. Using default one: '" + repr(optvalue) + "'")
+				values[optname] = optvalue
 		return values
 
 
@@ -286,7 +293,8 @@ class DefinitionInitConfigReader(ConfigReader):
 		if self.has_section("Init"):
 			for opt in self.options("Init"):
 				v = self.get("Init", opt)
-				self._initOpts['known/'+opt] = v
+				if not opt.startswith('known/') and opt != '__name__':
+					self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
 	

fail2ban/client/fail2banregex.py

@@ -291,7 +291,14 @@ class Fail2banRegex(object):
 				RegexStat(m[3])
 				for m in filter(
 					lambda x: x[0] == 'set' and x[2] == "add%sregex" % regextype,
-					readercommands)]
+					readercommands)
+			] + [
+				RegexStat(m)
+				for mm in filter(
+					lambda x: x[0] == 'multi-set' and x[2] == "add%sregex" % regextype,
+					readercommands)
+				for m in mm[3]
+			]
 			# Read out and set possible value of maxlines
 			for command in readercommands:
 				if command[2] == "maxlines":

fail2ban/client/filterreader.py

@@ -37,10 +37,10 @@ logSys = getLogger(__name__)
 
 class FilterReader(DefinitionInitConfigReader):
 
-	_configOpts = [
-		["string", "ignoreregex", None],
-		["string", "failregex", ""],
-	]
+	_configOpts = {
+		"ignoreregex": ["string", None],
+		"failregex": ["string", ""],
+	}
 
 	def setFile(self, fileName):
 		self.__file = fileName
@@ -64,16 +64,16 @@ class FilterReader(DefinitionInitConfigReader):
 		if not len(opts):
 			return stream
 		for opt, value in opts.iteritems():
-			if opt == "failregex":
+			if opt in ("failregex", "ignoreregex"):
+				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self._jailName, "addfailregex", regex])
-			elif opt == "ignoreregex":
-				for regex in value.split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self._jailName, "addignoreregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self._jailName, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self._jailName, "add" + opt, multi[0]])
 		if self._initOpts:
 			if 'maxlines' in self._initOpts:
 				# We warn when multiline regex is used without maxlines > 1

fail2ban/client/jailreader.py

@@ -190,11 +190,11 @@ class JailReader(ConfigReader):
 		 """
 
 		stream = []
-		for opt in self.__opts:
+		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
 					self.__opts.get('backend', None) != "systemd":
 				found_files = 0
-				for path in self.__opts[opt].split("\n"):
+				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
 					path, tail = path if len(path) > 1 else (path[0], "head")
 					pathList = JailReader._glob(path)
@@ -208,32 +208,32 @@ class JailReader(ConfigReader):
 					raise ValueError(
 						"Have not found any log file for %s jail" % self.__name)
 			elif opt == "logencoding":
-				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
+				stream.append(["set", self.__name, "logencoding", value])
 			elif opt == "backend":
-				backend = self.__opts[opt]
+				backend = value
 			elif opt == "maxretry":
-				stream.append(["set", self.__name, "maxretry", self.__opts[opt]])
+				stream.append(["set", self.__name, "maxretry", value])
 			elif opt == "ignoreip":
-				for ip in splitcommaspace(self.__opts[opt]):
+				for ip in splitcommaspace(value):
 					stream.append(["set", self.__name, "addignoreip", ip])
 			elif opt == "findtime":
-				stream.append(["set", self.__name, "findtime", self.__opts[opt]])
+				stream.append(["set", self.__name, "findtime", value])
 			elif opt == "bantime":
-				stream.append(["set", self.__name, "bantime", self.__opts[opt]])
+				stream.append(["set", self.__name, "bantime", value])
 			elif opt == "usedns":
-				stream.append(["set", self.__name, "usedns", self.__opts[opt]])
-			elif opt == "failregex":
-				for regex in self.__opts[opt].split('\n'):
+				stream.append(["set", self.__name, "usedns", value])
+			elif opt in ("failregex", "ignoreregex"):
+				multi = []
+				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
 					if regex != '':
-						stream.append(["set", self.__name, "addfailregex", regex])
+						multi.append(regex)
+				if len(multi) > 1:
+					stream.append(["multi-set", self.__name, "add" + opt, multi])
+				elif len(multi):
+					stream.append(["set", self.__name, "add" + opt, multi[0]])
 			elif opt == "ignorecommand":
-				stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]])
-			elif opt == "ignoreregex":
-				for regex in self.__opts[opt].split('\n'):
-					# Do not send a command if the rule is empty.
-					if regex != '':
-						stream.append(["set", self.__name, "addignoreregex", regex])
+				stream.append(["set", self.__name, "ignorecommand", value])
 		if self.__filter:
 			stream.extend(self.__filter.convert())
 		for action in self.__actions:

fail2ban/server/action.py

@@ -32,6 +32,8 @@ import time
 from abc import ABCMeta
 from collections import MutableMapping
 
+from .ipdns import asip
+from .mytime import MyTime
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -41,6 +43,11 @@ logSys = getLogger(__name__)
 # Create a lock for running system commands
 _cmd_lock = threading.Lock()
 
+# Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`:
+allowed_ipv6 = True
+
+# compiled RE for tag name (replacement name) 
+TAG_CRE = re.compile(r'<([^ <>]+)>')
 
 class CallingMap(MutableMapping):
 	"""A Mapping type which returns the result of callable values.
@@ -197,36 +204,40 @@ class CommandAction(ActionBase):
 
 	_escapedTags = set(('matches', 'ipmatches', 'ipjailmatches'))
 
+	timeout = 60
+	## Command executed in order to initialize the system.
+	actionstart = ''
+	## Command executed when an IP address gets banned.
+	actionban = ''
+	## Command executed when an IP address gets removed.
+	actionunban = ''
+	## Command executed in order to check requirements.
+	actioncheck = ''
+	## Command executed in order to stop the system.
+	actionstop = ''
+
 	def __init__(self, jail, name):
 		super(CommandAction, self).__init__(jail, name)
-		self.timeout = 60
-		## Command executed in order to initialize the system.
-		self.actionstart = ''
-		## Command executed when an IP address gets banned.
-		self.actionban = ''
-		## Command executed when an IP address gets removed.
-		self.actionunban = ''
-		## Command executed in order to check requirements.
-		self.actioncheck = ''
-		## Command executed in order to stop the system.
-		self.actionstop = ''
+		self.__properties = None
+		self.__substCache = {}
 		self._logSys.debug("Created %s" % self.__class__)
 
 	@classmethod
 	def __subclasshook__(cls, C):
 		return NotImplemented # Standard checks
 
-	@property
-	def timeout(self):
-		"""Time out period in seconds for execution of commands.
-		"""
-		return self._timeout
-
-	@timeout.setter
-	def timeout(self, timeout):
-		self._timeout = int(timeout)
-		self._logSys.debug("Set action %s timeout = %i" %
-			(self._name, self.timeout))
+	def __setattr__(self, name, value):
+		if not name.startswith('_') and not callable(value):
+			# special case for some pasrameters:
+			if name == 'timeout':
+				value = MyTime.str2seconds(value)
+			# parameters changed - clear properties and substitution cache:
+			self.__properties = None
+			self.__substCache.clear()
+			#self._logSys.debug("Set action %r %s = %r", self._name, name, value)
+			self._logSys.debug("  Set %s = %r", name, value)
+		# set:
+		self.__dict__[name] = value
 
 	@property
 	def _properties(self):
@@ -234,21 +245,20 @@ class CommandAction(ActionBase):
 
 		This is used to subsitute "tags" in the commands.
 		"""
-		return dict(
+		# if we have a properties - return it:
+		if self.__properties is not None:
+			return self.__properties
+		# otherwise retrieve:
+		self.__properties = dict(
 			(key, getattr(self, key))
 			for key in dir(self)
 			if not key.startswith("_") and not callable(getattr(self, key)))
+		#
+		return self.__properties
 
 	@property
-	def actionstart(self):
-		"""The command executed on start of the jail/action.
-		"""
-		return self._actionstart
-
-	@actionstart.setter
-	def actionstart(self, value):
-		self._actionstart = value
-		self._logSys.debug("Set actionstart = %s" % value)
+	def _substCache(self):
+		return self.__substCache
 
 	def start(self):
 		"""Executes the "actionstart" command.
@@ -256,26 +266,22 @@ class CommandAction(ActionBase):
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		if (self._properties and
-			not self.substituteRecursiveTags(self._properties)):
-			self._logSys.error(
-				"properties contain self referencing definitions "
-				"and cannot be resolved")
-			raise RuntimeError("Error starting action")
-		startCmd = self.replaceTag(self.actionstart, self._properties)
-		if not self.executeCmd(startCmd, self.timeout):
-			raise RuntimeError("Error starting action")
-
-	@property
-	def actionban(self):
-		"""The command used when a ban occurs.
-		"""
-		return self._actionban
-
-	@actionban.setter
-	def actionban(self, value):
-		self._actionban = value
-		self._logSys.debug("Set actionban = %s" % value)
+		# check valid tags in properties (raises ValueError if self recursion, etc.):
+		try:
+			# common (resp. ipv4):
+			startCmd = self.replaceTag('<actionstart>', self._properties, 
+				conditional='family=inet4', cache=self.__substCache)
+			res = self.executeCmd(startCmd, self.timeout)
+			# start ipv6 actions if available:
+			if allowed_ipv6:
+				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
+					conditional='family=inet6', cache=self.__substCache)
+				if startCmd6 != startCmd:
+					res &= self.executeCmd(startCmd6, self.timeout)
+			if not res:
+				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
+		except ValueError, e:
+			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -289,20 +295,9 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionban, aInfo):
+		if not self._processCmd('<actionban>', aInfo):
 			raise RuntimeError("Error banning %(ip)s" % aInfo)
 
-	@property
-	def actionunban(self):
-		"""The command used when an unban occurs.
-		"""
-		return self._actionunban
-
-	@actionunban.setter
-	def actionunban(self, value):
-		self._actionunban = value
-		self._logSys.debug("Set actionunban = %s" % value)
-
 	def unban(self, aInfo):
 		"""Executes the "actionunban" command.
 
@@ -315,47 +310,30 @@ class CommandAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
-		if not self._processCmd(self.actionunban, aInfo):
+		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
-	@property
-	def actioncheck(self):
-		"""The command used to check the environment.
-
-		This is used prior to a ban taking place to ensure the
-		environment is appropriate. If this check fails, `stop` and
-		`start` is executed prior to the check being called again.
-		"""
-		return self._actioncheck
-
-	@actioncheck.setter
-	def actioncheck(self, value):
-		self._actioncheck = value
-		self._logSys.debug("Set actioncheck = %s" % value)
-
-	@property
-	def actionstop(self):
-		"""The command executed when the jail/actions stops.
-		"""
-		return self._actionstop
-
-	@actionstop.setter
-	def actionstop(self, value):
-		self._actionstop = value
-		self._logSys.debug("Set actionstop = %s" % value)
-
 	def stop(self):
 		"""Executes the "actionstop" command.
 
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		stopCmd = self.replaceTag(self.actionstop, self._properties)
-		if not self.executeCmd(stopCmd, self.timeout):
+		# common (resp. ipv4):
+		stopCmd = self.replaceTag('<actionstop>', self._properties, 
+			conditional='family=inet4', cache=self.__substCache)
+		res = self.executeCmd(stopCmd, self.timeout)
+		# ipv6 actions if available:
+		if allowed_ipv6:
+			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
+				conditional='family=inet6', cache=self.__substCache)
+			if stopCmd6 != stopCmd:
+				res &= self.executeCmd(stopCmd6, self.timeout)
+		if not res:
 			raise RuntimeError("Error stopping action")
 
 	@classmethod
-	def substituteRecursiveTags(cls, tags):
+	def substituteRecursiveTags(cls, inptags, conditional=''):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -365,7 +343,7 @@ class CommandAction(ActionBase):
 
 		Parameters
 		----------
-		tags : dict
+		inptags : dict
 			Dictionary of tags(keys) and their values.
 
 		Returns
@@ -374,7 +352,9 @@ class CommandAction(ActionBase):
 			Dictionary of tags(keys) and their values, with tags
 			within the values recursively replaced.
 		"""
-		t = re.compile(r'<([^ <>]+)>')
+		# copy return tags dict to prevent modifying of inptags:
+		tags = inptags.copy()
+		t = TAG_CRE
 		# repeat substitution while embedded-recursive (repFlag is True)
 		while True:
 			repFlag = False
@@ -394,14 +374,21 @@ class CommandAction(ActionBase):
 					if found_tag == tag or found_tag in done:
 						# recursive definitions are bad
 						#logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) )
-						return False
-					if found_tag in cls._escapedTags or not found_tag in tags:
+						raise ValueError(
+							"properties contain self referencing definitions "
+							"and cannot be resolved, fail tag: %s value: %s" % (tag, value))
+					repl = None
+					if found_tag not in cls._escapedTags:
+						repl = tags.get(found_tag + '?' + conditional)
+						if repl is None:
+							repl = tags.get(found_tag)
+					if repl is None:
 						# Escaped or missing tags - just continue on searching after end of match
 						# Missing tags are ok - cInfo can contain aInfo elements like <HOST> and valid shell
 						# constructs like <STDIN>.
 						m = t.search(value, m.end())
 						continue
-					value = value.replace('<%s>' % found_tag , tags[found_tag])
+					value = value.replace('<%s>' % found_tag, repl)
 					#logSys.log(5, 'value now: %s' % value)
 					done.append(found_tag)
 					m = t.search(value, m.start())
@@ -443,7 +430,7 @@ class CommandAction(ActionBase):
 		return value
 
 	@classmethod
-	def replaceTag(cls, query, aInfo):
+	def replaceTag(cls, query, aInfo, conditional='', cache=None):
 		"""Replaces tags in `query` with property values.
 
 		Parameters
@@ -458,21 +445,35 @@ class CommandAction(ActionBase):
 		str
 			`query` string with tags replaced.
 		"""
+		# use cache if allowed:
+		if cache is not None:
+			ckey = (query, conditional)
+			string = cache.get(ckey)
+			if string is not None:
+				return string
+		# replace:
 		string = query
-		aInfo = cls.substituteRecursiveTags(aInfo)
+		aInfo = cls.substituteRecursiveTags(aInfo, conditional)
 		for tag in aInfo:
 			if "<%s>" % tag in query:
-				value = str(aInfo[tag])			  # assure string
+				value = aInfo.get(tag + '?' + conditional)
+				if value is None:
+					value = aInfo.get(tag)
+				value = str(value)			  # assure string
 				if tag in cls._escapedTags:
 					# That one needs to be escaped since its content is
 					# out of our control
 					value = cls.escapeTag(value)
 				string = string.replace('<' + tag + '>', value)
-		# New line
-		string = string.replace("<br>", '\n')
+		# New line, space
+		string = reduce(lambda s, kv: s.replace(*kv), (("<br>", '\n'), ("<sp>", " ")), string)
+		# cache if properties:
+		if cache is not None:
+			cache[ckey] = string
+		#
 		return string
 
-	def _processCmd(self, cmd, aInfo = None):
+	def _processCmd(self, cmd, aInfo=None, conditional=''):
 		"""Executes a command with preliminary checks and substitutions.
 
 		Before executing any commands, executes the "check" command first
@@ -496,7 +497,19 @@ class CommandAction(ActionBase):
 			self._logSys.debug("Nothing to do")
 			return True
 
-		checkCmd = self.replaceTag(self.actioncheck, self._properties)
+		# conditional corresponding family of the given ip:
+		if conditional == '':
+			conditional = 'family=inet4'
+			if allowed_ipv6:
+				try:
+					ip = aInfo["ip"]
+					if ip and asip(ip).isIPv6:
+						conditional = 'family=inet6'
+				except KeyError:
+					pass
+
+		checkCmd = self.replaceTag('<actioncheck>', self._properties, 
+			conditional=conditional, cache=self.__substCache)
 		if not self.executeCmd(checkCmd, self.timeout):
 			self._logSys.error(
 				"Invariant check failed. Trying to restore a sane environment")
@@ -506,15 +519,16 @@ class CommandAction(ActionBase):
 				self._logSys.critical("Unable to restore environment")
 				return False
 
+		# Replace static fields
+		realCmd = self.replaceTag(cmd, self._properties, 
+			conditional=conditional, cache=self.__substCache)
+
 		# Replace tags
-		if not aInfo is None:
-			realCmd = self.replaceTag(cmd, aInfo)
+		if aInfo is not None:
+			realCmd = self.replaceTag(realCmd, aInfo, conditional=conditional)
 		else:
 			realCmd = cmd
 
-		# Replace static fields
-		realCmd = self.replaceTag(realCmd, self._properties)
-
 		return self.executeCmd(realCmd, self.timeout)
 
 	@staticmethod

fail2ban/server/actions.py

@@ -42,6 +42,7 @@ from .banmanager import BanManager
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
+from .filter import IPAddr
 from .utils import Utils
 from ..helpers import getLogger
 
@@ -188,8 +189,8 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ip : str
-			The IP address to unban
+		ip : str or IPAddr
+			The IP address  to unban
 
 		Raises
 		------

fail2ban/server/banmanager.py

@@ -152,9 +152,10 @@ class BanManager:
 			for banData in self.__banList:
 				ip = banData.getIP()
 				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
-				# TODO: IPv6 compatibility
-				reversed_ip = ".".join(reversed(ip.split(".")))
-				question = "%s.origin.asn.cymru.com" % reversed_ip
+				question = ip.getPTR(
+					"origin.asn.cymru.com" if ip.isIPv4
+					else "origin6.asn.cymru.com"
+				)
 				try:
 					answers = dns.resolver.query(question, "TXT")
 					for rdata in answers:

fail2ban/server/database.py

@@ -411,18 +411,19 @@ class Fail2BanDb(object):
 		ticket : BanTicket
 			Ticket of the ban to be added.
 		"""
+		ip = str(ticket.getIP())
 		try:
-			del self._bansMergedCache[(ticket.getIP(), jail)]
+			del self._bansMergedCache[(ip, jail)]
 		except KeyError:
 			pass
 		try:
-			del self._bansMergedCache[(ticket.getIP(), None)]
+			del self._bansMergedCache[(ip, None)]
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ticket.getIP(), int(round(ticket.getTime())),
+			(jail.name, ip, int(round(ticket.getTime())),
 				ticket.getData()))
 
 	@commitandrollback
@@ -436,7 +437,7 @@ class Fail2BanDb(object):
 		ip : str
 			IP to be removed.
 		"""
-		queryArgs = (jail.name, ip);
+		queryArgs = (jail.name, str(ip));
 		cur.execute(
 			"DELETE FROM bans WHERE jail = ? AND ip = ?", 
 			queryArgs);

fail2ban/server/failmanager.py

@@ -54,6 +54,15 @@ class FailManager:
 		with self.__lock:
 			return self.__failTotal
 	
+	def getFailCount(self):
+		# may be slow on large list of failures, should be used for test purposes only...
+		with self.__lock:
+			return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()])
+
+	def getFailTotal(self):
+		with self.__lock:
+			return self.__failTotal
+
 	def setMaxRetry(self, value):
 		self.__maxRetry = value
 	

fail2ban/server/failregex.py

@@ -43,8 +43,8 @@ class Regex:
 	def __init__(self, regex):
 		self._matchCache = None
 		# Perform shortcuts expansions.
-		# Replace "<HOST>" with default regular expression for host.
-		regex = regex.replace("<HOST>", "(?:::f{4,6}:)?(?P<host>[\w\-.^_]*\w)")
+		# Resolve "<HOST>" tag using default regular expression for host:
+		regex = Regex._resolveHostTag(regex)
 		# Replace "<SKIPLINES>" with regular expression for multiple lines.
 		regexSplit = regex.split("<SKIPLINES>")
 		regex = regexSplit[0]
@@ -61,6 +61,20 @@ class Regex:
 
 	def __str__(self):
 		return "%s(%r)" % (self.__class__.__name__, self._regex)
+
+	@staticmethod
+	def _resolveHostTag(regex):
+		# Replace "<HOST>" with default regular expression for host:
+		# Other candidates (see gh-1374 for the discussion about):
+		#   differentiate: r"""(?:(?:::f{4,6}:)?(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?|(?P<HOST>[\w\-.^_]*\w))"""
+		#     expected many changes in filter, failregex, etc...
+		#   simple:        r"""(?:::f{4,6}:)?(?P<host>[\w\-.^_:]*\w)"""
+		#     not good enough, if not precise expressions around <HOST>, because for example will match '1.2.3.4:23930' as ip-address;
+		# Todo: move this functionality to filter reader, as default <HOST> replacement,
+		#       make it configurable (via jail/filter configs)
+		return regex.replace("<HOST>",
+		 	r"""(?:::f{4,6}:)?(?P<host>(?:\d{1,3}\.){3}\d{1,3}|\[?(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}\]?|(?<=:):)|[\w\-.^_]*\w)""")
+
 	##
 	# Gets the regular expression.
 	#

fail2ban/server/filter.py

@@ -31,6 +31,7 @@ import re
 import sys
 
 from .failmanager import FailManagerEmpty, FailManager
+from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
 from .jailthread import JailThread
 from .datedetector import DateDetector
@@ -313,6 +314,8 @@ class Filter(JailThread):
 	# to enable banip fail2ban-client BAN command
 
 	def addBannedIP(self, ip):
+		if not isinstance(ip, IPAddr):
+			ip = IPAddr(ip)
 		if self.inIgnoreIPList(ip):
 			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.' % ip)
 
@@ -336,12 +339,19 @@ class Filter(JailThread):
 	# when finding failures. CIDR mask and DNS are also accepted.
 	# @param ip IP address to ignore
 
-	def addIgnoreIP(self, ip):
-		logSys.debug("Add " + ip + " to ignore list")
+	def addIgnoreIP(self, ipstr):
+		# An empty string is always false
+		if ipstr == "":
+			return
+		# Create IP address object
+		ip = IPAddr(ipstr)
+
+		# log and append to ignore list
+		logSys.debug("Add %r to ignore list (%r)", ip, ipstr)
 		self.__ignoreIpList.append(ip)
 
 	def delIgnoreIP(self, ip):
-		logSys.debug("Remove " + ip + " from ignore list")
+		logSys.debug("Remove %r from ignore list", ip)
 		self.__ignoreIpList.remove(ip)
 
 	def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"):
@@ -356,35 +366,16 @@ class Filter(JailThread):
 	#
 	# Check if the given IP address matches an IP address/DNS or a CIDR
 	# mask in the ignore list.
-	# @param ip IP address
+	# @param ip IP address object
 	# @return True if IP address is in ignore list
 
 	def inIgnoreIPList(self, ip, log_ignore=False):
-		for i in self.__ignoreIpList:
-			# An empty string is always false
-			if i == "":
-				continue
-			s = i.split('/', 1)
-			# IP address without CIDR mask
-			if len(s) == 1:
-				s.insert(1, '32')
-			elif "." in s[1]: # 255.255.255.0 style mask
-				s[1] = len(re.search(
-					"(?<=b)1+", bin(DNSUtils.addr2bin(s[1]))).group())
-			s[1] = long(s[1])
-			try:
-				a = DNSUtils.addr2bin(s[0], cidr=s[1])
-				b = DNSUtils.addr2bin(ip, cidr=s[1])
-			except Exception:
-				# Check if IP in DNS
-				ips = DNSUtils.dnsToIp(i)
-				if ip in ips:
-					self.logIgnoreIp(ip, log_ignore, ignore_source="dns")
-					return True
-				else:
-					continue
-			if a == b:
-				self.logIgnoreIp(ip, log_ignore, ignore_source="ip")
+		if not isinstance(ip, IPAddr):
+			ip = IPAddr(ip)
+		for net in self.__ignoreIpList:
+			# check if the IP is covered by ignore IP
+			if ip.isInNet(net):
+				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
 				return True
 
 		if self.__ignoreCommand:
@@ -530,16 +521,16 @@ class Filter(JailThread):
 					try:
 						host = failRegex.getHost()
 						if returnRawHost:
-							failList.append([failRegexIndex, host, date,
+							failList.append([failRegexIndex, IPAddr(host), date,
 								 failRegex.getMatchedLines()])
 							if not checkAllRegex:
 								break
 						else:
-							ipMatch = DNSUtils.textToIp(host, self.__useDns)
-							if ipMatch:
-								for ip in ipMatch:
-									failList.append([failRegexIndex, ip, date,
-										 failRegex.getMatchedLines()])
+							ips = DNSUtils.textToIp(host, self.__useDns)
+							if ips:
+								for ip in ips:
+									failList.append([failRegexIndex, ip, 
+										 date, failRegex.getMatchedLines()])
 								if not checkAllRegex:
 									break
 					except RegexException, e: # pragma: no cover - unsure if reachable
@@ -982,117 +973,3 @@ class JournalFilter(Filter): # pragma: systemd no cover
 	def getJournalMatch(self, match): # pragma: no cover - Base class, not used
 		return []
 
-##
-# Utils class for DNS and IP handling.
-#
-# This class contains only static methods used to handle DNS and IP
-# addresses.
-
-import socket
-import struct
-from .utils import Utils
-
-
-class DNSUtils:
-
-	IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$")
-
-	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
-	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
-
-	@staticmethod
-	def dnsToIp(dns):
-		""" Convert a DNS into an IP address using the Python socket module.
-			Thanks to Kevin Drapel.
-		"""
-		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
-		v = DNSUtils.CACHE_nameToIp.get(dns)
-		if v is not None: 
-			return v
-		# retrieve ip (todo: use AF_INET6 for IPv6)
-		try:
-			v = set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)])
-		except socket.error, e:
-			# todo: make configurable the expired time of cache entry:
-			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
-			v = list()
-		DNSUtils.CACHE_nameToIp.set(dns, v)
-		return v
-
-	@staticmethod
-	def ipToName(ip):
-		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
-		v = DNSUtils.CACHE_ipToName.get(ip, ())
-		if v != ():
-			return v
-		# retrieve name
-		try:
-			v = socket.gethostbyaddr(ip)[0]
-		except socket.error, e:
-			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
-			v = None
-		DNSUtils.CACHE_ipToName.set(ip, v)
-		return v
-
-	@staticmethod
-	def searchIP(text):
-		""" Search if an IP address if directly available and return
-			it.
-		"""
-		match = DNSUtils.IP_CRE.match(text)
-		if match:
-			return match
-		else:
-			return None
-
-	@staticmethod
-	def isValidIP(string):
-		""" Return true if str is a valid IP
-		"""
-		s = string.split('/', 1)
-		try:
-			socket.inet_aton(s[0])
-			return True
-		except socket.error: # pragma: no cover
-			return False
-
-	@staticmethod
-	def textToIp(text, useDns):
-		""" Return the IP of DNS found in a given text.
-		"""
-		ipList = list()
-		# Search for plain IP
-		plainIP = DNSUtils.searchIP(text)
-		if not plainIP is None:
-			plainIPStr = plainIP.group(0)
-			if DNSUtils.isValidIP(plainIPStr):
-				ipList.append(plainIPStr)
-
-		# If we are allowed to resolve -- give it a try if nothing was found
-		if useDns in ("yes", "warn") and not ipList:
-			# Try to get IP from possible DNS
-			ip = DNSUtils.dnsToIp(text)
-			ipList.extend(ip)
-			if ip and useDns == "warn":
-				logSys.warning("Determined IP using DNS Lookup: %s = %s",
-					text, ipList)
-
-		return ipList
-
-	@staticmethod
-	def addr2bin(ipstring, cidr=None):
-		""" Convert a string IPv4 address into binary form.
-		If cidr is supplied, return the network address for the given block
-		"""
-		if cidr is None:
-			return struct.unpack("!L", socket.inet_aton(ipstring))[0]
-		else:
-			MASK = 0xFFFFFFFFL
-			return ~(MASK >> cidr) & MASK & DNSUtils.addr2bin(ipstring)
-
-	@staticmethod
-	def bin2addr(ipbin):
-		""" Convert a binary IPv4 address into string n.n.n.n form.
-		"""
-		return socket.inet_ntoa(struct.pack("!L", ipbin))

fail2ban/server/ipdns.py

@@ -0,0 +1,423 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Fail2Ban Developers, Alexander Koeppe, Serg G. Brester, Yaroslav Halchenko"
+__copyright__ = "Copyright (c) 2004-2016 Fail2ban Developers"
+__license__ = "GPL"
+
+import socket
+import struct
+import re
+
+from .utils import Utils
+from ..helpers import getLogger
+
+# Gets the instance of the logger.
+logSys = getLogger(__name__)
+
+
+##
+# Helper functions
+#
+#
+def asip(ip):
+	"""A little helper to guarantee ip being an IPAddr instance"""
+	if isinstance(ip, IPAddr):
+		return ip
+	return IPAddr(ip)
+
+
+##
+# Utils class for DNS handling.
+#
+# This class contains only static methods used to handle DNS 
+#
+class DNSUtils:
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	@staticmethod
+	def dnsToIp(dns):
+		""" Convert a DNS into an IP address using the Python socket module.
+			Thanks to Kevin Drapel.
+		"""
+		# cache, also prevent long wait during retrieving of ip for wrong dns or lazy dns-system:
+		ips = DNSUtils.CACHE_nameToIp.get(dns)
+		if ips is not None: 
+			return ips
+		# retrieve ips
+		try:
+			ips = list()
+			for result in socket.getaddrinfo(dns, None, 0, 0, socket.IPPROTO_TCP):
+				ip = IPAddr(result[4][0])
+				if ip.isValid:
+					ips.append(ip)
+		except socket.error, e:
+			# todo: make configurable the expired time of cache entry:
+			logSys.warning("Unable to find a corresponding IP address for %s: %s", dns, e)
+			ips = list()
+		DNSUtils.CACHE_nameToIp.set(dns, ips)
+		return ips
+
+	@staticmethod
+	def ipToName(ip):
+		# cache, also prevent long wait during retrieving of name for wrong addresses, lazy dns:
+		v = DNSUtils.CACHE_ipToName.get(ip, ())
+		if v != ():
+			return v
+		# retrieve name
+		try:
+			if not isinstance(ip, IPAddr):
+				v = socket.gethostbyaddr(ip)[0]
+			else:
+				v = socket.gethostbyaddr(ip.ntoa)[0]
+		except socket.error, e:
+			logSys.debug("Unable to find a name for the IP %s: %s", ip, e)
+			v = None
+		DNSUtils.CACHE_ipToName.set(ip, v)
+		return v
+
+	@staticmethod
+	def textToIp(text, useDns):
+		""" Return the IP of DNS found in a given text.
+		"""
+		ipList = list()
+		# Search for plain IP
+		plainIP = IPAddr.searchIP(text)
+		if plainIP is not None:
+			ip = IPAddr(plainIP)
+			if ip.isValid:
+				ipList.append(ip)
+
+		# If we are allowed to resolve -- give it a try if nothing was found
+		if useDns in ("yes", "warn") and not ipList:
+			# Try to get IP from possible DNS
+			ip = DNSUtils.dnsToIp(text)
+			ipList.extend(ip)
+			if ip and useDns == "warn":
+				logSys.warning("Determined IP using DNS Lookup: %s = %s",
+					text, ipList)
+
+		return ipList
+
+
+##
+# Class for IP address handling.
+#
+# This class contains methods for handling IPv4 and IPv6 addresses.
+#
+class IPAddr(object):
+	"""Encapsulate functionality for IPv4 and IPv6 addresses
+	"""
+	IP_4_6_CRE = re.compile(
+	  r"""^(?:(?P<IPv4>(?:\d{1,3}\.){3}\d{1,3})|\[?(?P<IPv6>(?:[0-9a-fA-F]{1,4}::?|::){1,7}(?:[0-9a-fA-F]{1,4}|(?<=:):))\]?)$""")
+	# An IPv4 compatible IPv6 to be reused (see below)
+	IP6_4COMPAT = None
+
+	# object attributes
+	__slots__ = '_family','_addr','_plen','_maskplen','_raw'
+
+	# todo: make configurable the expired time and max count of cache entries:
+	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+
+	def __new__(cls, ipstr, cidr=-1):
+		# check already cached as IPAddr
+		args = (ipstr, cidr)
+		ip = IPAddr.CACHE_OBJ.get(args)
+		if ip is not None:
+			return ip
+		# wrap mask to cidr (correct plen):
+		if cidr == -1:
+			ipstr, cidr = IPAddr.__wrap_ipstr(ipstr)
+			args = (ipstr, cidr)
+			# check cache again:
+			if cidr != -1:
+				ip = IPAddr.CACHE_OBJ.get(args)
+				if ip is not None:
+					return ip
+		ip = super(IPAddr, cls).__new__(cls)
+		ip.__init(ipstr, cidr)
+		IPAddr.CACHE_OBJ.set(args, ip)
+		return ip
+
+	@staticmethod
+	def __wrap_ipstr(ipstr):
+		# because of standard spelling of IPv6 (with port) enclosed in brackets ([ipv6]:port),
+		# remove they now (be sure the <HOST> inside failregex uses this for IPv6 (has \[?...\]?)
+		if len(ipstr) > 2 and ipstr[0] == '[' and ipstr[-1] == ']':
+			ipstr = ipstr[1:-1]
+		# test mask:
+		if "/" not in ipstr:
+			return ipstr, -1
+		s = ipstr.split('/', 1)
+		# IP address without CIDR mask
+		if len(s) > 2:
+			raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,))
+		if "." in s[1]: # 255.255.255.0 style mask
+			s[1] = IPAddr.masktoplen(s[1])
+		s[1] = long(s[1])
+		return s
+		
+	def __init(self, ipstr, cidr=-1):
+		""" initialize IP object by converting IP address string
+			to binary to integer
+		"""
+		self._family = socket.AF_UNSPEC
+		self._addr = 0
+		self._plen = 0
+		self._maskplen = None
+		self._raw = ""
+
+		for family in [socket.AF_INET, socket.AF_INET6]:
+			try:
+				binary = socket.inet_pton(family, ipstr)
+				self._family = family
+				break
+			except socket.error:
+				continue
+
+		if self._family == socket.AF_INET:
+			# convert host to network byte order
+			self._addr, = struct.unpack("!L", binary)
+			self._plen = 32
+
+			# mask out host portion if prefix length is supplied
+			if cidr is not None and cidr >= 0:
+				mask = ~(0xFFFFFFFFL >> cidr)
+				self._addr &= mask
+				self._plen = cidr
+
+		elif self._family == socket.AF_INET6:
+			# convert host to network byte order
+			hi, lo = struct.unpack("!QQ", binary)
+			self._addr = (hi << 64) | lo
+			self._plen = 128
+
+			# mask out host portion if prefix length is supplied
+			if cidr is not None and cidr >= 0:
+				mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr)
+				self._addr &= mask
+				self._plen = cidr
+
+			# if IPv6 address is a IPv4-compatible, make instance a IPv4
+			elif self.isInNet(IPAddr.IP6_4COMPAT):
+				self._addr = lo & 0xFFFFFFFFL
+				self._family = socket.AF_INET
+				self._plen = 32
+		else:
+			# string couldn't be converted neither to a IPv4 nor
+			# to a IPv6 address - retain raw input for later use
+			# (e.g. DNS resolution)
+			self._raw = ipstr
+
+	def __repr__(self):
+		return self.ntoa
+
+	def __str__(self):
+		return self.ntoa
+	
+	@property
+	def addr(self):
+		return self._addr
+
+	@property
+	def family(self):
+		return self._family
+
+	@property
+	def plen(self):
+		return self._plen
+
+	@property
+	def raw(self):
+		"""The raw address
+
+		Should only be set to a non-empty string if prior address
+		conversion wasn't possible
+		"""
+		return self._raw
+
+	@property
+	def isValid(self):
+		"""Either the object corresponds to a valid IP address
+		"""
+		return self._family != socket.AF_UNSPEC
+
+	def __eq__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		if self._family != other._family: return False
+		if self._family == socket.AF_UNSPEC:
+			return self._raw == other._raw
+		return (
+			(self._addr == other._addr) and
+			(self._plen == other._plen)
+		)
+
+	def __ne__(self, other):
+		return not (self == other)
+
+	def __lt__(self, other):
+		if not isinstance(other, IPAddr):
+			if other is None: return False
+			other = IPAddr(other)
+		return self._family < other._family or self._addr < other._addr
+
+	def __add__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (self, other)
+
+	def __radd__(self, other):
+		if not isinstance(other, IPAddr):
+			other = IPAddr(other)
+		return "%s%s" % (other, self)
+
+	def __hash__(self):
+		# should be the same as by string (because of possible compare with string):
+		return hash(self.ntoa)
+		#return hash(self._addr)^hash((self._plen<<16)|self._family)
+
+	@property
+	def hexdump(self):
+		"""Hex representation of the IP address (for debug purposes)
+		"""
+		if self._family == socket.AF_INET:
+			return "%08x" % self._addr
+		elif self._family == socket.AF_INET6:
+			return "%032x" % self._addr
+		else:
+			return ""
+
+	# TODO: could be lazily evaluated
+	@property
+	def ntoa(self):
+		""" represent IP object as text like the deprecated
+			C pendant inet.ntoa but address family independent
+		"""
+		add = ''
+		if self.isIPv4:
+			# convert network to host byte order
+			binary = struct.pack("!L", self._addr)
+			if self._plen and self._plen < 32:
+				add = "/%d" % self._plen
+		elif self.isIPv6:
+			# convert network to host byte order
+			hi = self._addr >> 64
+			lo = self._addr & 0xFFFFFFFFFFFFFFFFL
+			binary = struct.pack("!QQ", hi, lo)
+			if self._plen and self._plen < 128:
+				add = "/%d" % self._plen
+		else:
+			return self._raw
+		
+		return socket.inet_ntop(self._family, binary) + add
+
+	def getPTR(self, suffix=""):
+		""" return the DNS PTR string of the provided IP address object
+
+			If "suffix" is provided it will be appended as the second and top
+			level reverse domain.
+			If omitted it is implicitly set to the second and top level reverse
+			domain of the according IP address family
+		"""
+		if self.isIPv4:
+			exploded_ip = self.ntoa.split(".")
+			if not suffix:
+				suffix = "in-addr.arpa."
+		elif self.isIPv6:
+			exploded_ip = self.hexdump()
+			if not suffix:
+				suffix = "ip6.arpa."
+		else:
+			return ""
+
+		return "%s.%s" % (".".join(reversed(exploded_ip)), suffix)
+
+	@property
+	def isIPv4(self):
+		"""Either the IP object is of address family AF_INET
+		"""
+		return self.family == socket.AF_INET
+
+	@property
+	def isIPv6(self):
+		"""Either the IP object is of address family AF_INET6
+		"""
+		return self.family == socket.AF_INET6
+
+	def isInNet(self, net):
+		"""Return either the IP object is in the provided network
+		"""
+		# if it isn't a valid IP address, try DNS resolution
+		if not net.isValid and net.raw != "":
+			# Check if IP in DNS
+			return self in DNSUtils.dnsToIp(net.raw)
+
+		if self.family != net.family:
+			return False
+		if self.isIPv4:
+			mask = ~(0xFFFFFFFFL >> net.plen)
+		elif self.isIPv6:
+			mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen)
+		else:
+			return False
+		
+		return (self.addr & mask) == net.addr
+
+	@property
+	def maskplen(self):
+		mplen = 0
+		if self._maskplen is not None:
+			return self._maskplen
+		maddr = self._addr
+		while maddr:
+			if not (maddr & 0x80000000):
+				raise ValueError("invalid mask %r, no plen representation" % (str(self),))
+			maddr = (maddr << 1) & 0xFFFFFFFFL
+			mplen += 1
+		self._maskplen = mplen
+		return mplen
+		
+	@staticmethod
+	def masktoplen(mask):
+		"""Convert mask string to prefix length
+
+		To be used only for IPv4 masks
+		"""
+		return IPAddr(mask).maskplen
+
+	@staticmethod
+	def searchIP(text):
+		"""Search if text is an IP address, and return it if so, else None
+		"""
+		match = IPAddr.IP_4_6_CRE.match(text)
+		if not match:
+			return None
+		ipstr = match.group('IPv4')
+		if ipstr != '':
+			return ipstr
+		return match.group('IPv6')
+
+
+# An IPv4 compatible IPv6 to be reused
+IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96)

fail2ban/server/server.py

@@ -262,8 +262,13 @@ class Server:
 	def getIgnoreCommand(self, name):
 		return self.__jails[name].filter.getIgnoreCommand()
 
-	def addFailRegex(self, name, value):
-		self.__jails[name].filter.addFailRegex(value)
+	def addFailRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addFailRegex(value)
+		else:
+			flt.addFailRegex(value)
 	
 	def delFailRegex(self, name, index):
 		self.__jails[name].filter.delFailRegex(index)
@@ -271,8 +276,13 @@ class Server:
 	def getFailRegex(self, name):
 		return self.__jails[name].filter.getFailRegex()
 	
-	def addIgnoreRegex(self, name, value):
-		self.__jails[name].filter.addIgnoreRegex(value)
+	def addIgnoreRegex(self, name, value, multiple=False):
+		flt = self.__jails[name].filter
+		if multiple:
+			for value in value:
+				flt.addIgnoreRegex(value)
+		else:
+			flt.addIgnoreRegex(value)
 	
 	def delIgnoreRegex(self, name, index):
 		self.__jails[name].filter.delIgnoreRegex(index)

fail2ban/server/ticket.py

@@ -27,6 +27,7 @@ __license__ = "GPL"
 import sys
 
 from ..helpers import getLogger
+from .ipdns import IPAddr
 from .mytime import MyTime
 
 # Gets the instance of the logger.
@@ -72,9 +73,9 @@ class Ticket:
 			return False
 
 	def setIP(self, value):
+		# guarantee using IPAddr instead of unicode, str for the IP
 		if isinstance(value, basestring):
-			# guarantee using regular str instead of unicode for the IP
-			value = str(value)
+			value = IPAddr(value)
 		self.__ip = value
 	
 	def getIP(self):

fail2ban/server/transmitter.py

@@ -99,6 +99,8 @@ class Transmitter:
 			return None
 		elif command[0] == "flushlogs":
 			return self.__server.flushLogs()
+		elif command[0] == "multi-set":
+			return self.__commandSet(command[1:], True)
 		elif command[0] == "set":
 			return self.__commandSet(command[1:])
 		elif command[0] == "get":
@@ -109,7 +111,7 @@ class Transmitter:
 			return version.version
 		raise Exception("Invalid command")
 	
-	def __commandSet(self, command):
+	def __commandSet(self, command, multiple=False):
 		name = command[0]
 		# Logging
 		if name == "loglevel":
@@ -196,7 +198,9 @@ class Transmitter:
 			return self.__server.getJournalMatch(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
-			self.__server.addFailRegex(name, value)
+			self.__server.addFailRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getFailRegex(name)
 		elif command[1] == "delfailregex":
 			value = int(command[2])
@@ -204,7 +208,9 @@ class Transmitter:
 			return self.__server.getFailRegex(name)
 		elif command[1] == "addignoreregex":
 			value = command[2]
-			self.__server.addIgnoreRegex(name, value)
+			self.__server.addIgnoreRegex(name, value, multiple=multiple)
+			if multiple:
+				return True
 			return self.__server.getIgnoreRegex(name)
 		elif command[1] == "delignoreregex":
 			value = int(command[2])
@@ -254,15 +260,26 @@ class Transmitter:
 			return None
 		elif command[1] == "action":
 			actionname = command[2]
-			actionkey = command[3]
 			action = self.__server.getAction(name, actionname)
-			if callable(getattr(action, actionkey, None)):
-				actionvalue = json.loads(command[4]) if len(command)>4 else {}
-				return getattr(action, actionkey)(**actionvalue)
+			if multiple:
+				for cmd in command[3]:
+					actionkey = cmd[0]
+					if callable(getattr(action, actionkey, None)):
+						actionvalue = json.loads(cmd[1]) if len(cmd)>1 else {}
+						getattr(action, actionkey)(**actionvalue)
+					else:
+						actionvalue = cmd[1]
+						setattr(action, actionkey, actionvalue)
+				return True
 			else:
-				actionvalue = command[4]
-				setattr(action, actionkey, actionvalue)
-				return getattr(action, actionkey)
+				actionkey = command[3]
+				if callable(getattr(action, actionkey, None)):
+					actionvalue = json.loads(command[4]) if len(command)>4 else {}
+					return getattr(action, actionkey)(**actionvalue)
+				else:
+					actionvalue = command[4]
+					setattr(action, actionkey, actionvalue)
+					return getattr(action, actionkey)
 		raise Exception("Invalid command (no set action or not yet implemented)")
 	
 	def __commandGet(self, command):

fail2ban/tests/actiontestcase.py

@@ -54,12 +54,17 @@ class CommandActionTest(LogCaptureTestCase):
 			'xyz': "890 <ABC>",
 		}
 		# Recursion is bad
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# Unresolveable substition
-		self.assertFalse(CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
-		self.assertFalse(CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''}))
+		self.assertRaises(ValueError,
+			lambda: CommandAction.substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''}))
 		# missing tags are ok
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
 		self.assertEqual(CommandAction.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
@@ -127,12 +132,55 @@ class CommandActionTest(LogCaptureTestCase):
 				CallingMap(matches=lambda: str(10))),
 			"09 10 11")
 
+	def testReplaceNoTag(self):
 		# As tag not present, therefore callable should not be called
 		# Will raise ValueError if it is
 		self.assertEqual(
 			self.__action.replaceTag("abc",
 				CallingMap(matches=lambda: int("a"))), "abc")
 
+	def testReplaceTagConditionalCached(self):
+		setattr(self.__action, 'abc', "123")
+		setattr(self.__action, 'abc?family=inet4', "345")
+		setattr(self.__action, 'abc?family=inet6', "567")
+		setattr(self.__action, 'xyz', "890-<abc>")
+		setattr(self.__action, 'banaction', "Text <xyz> text <abc>")
+		# test replacement in sub tags and direct, conditional, cached:
+		cache = self.__action._substCache
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 890-123 text 123 '123'")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 890-345 text 345 '345'")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 890-567 text 567 '567'")
+		self.assertEqual(len(cache) if cache is not None else -1, 3)
+		# set one parameter - internal properties and cache should be reseted:
+		setattr(self.__action, 'xyz', "000-<abc>")
+		self.assertEqual(len(cache) if cache is not None else -1, 0)
+		# test againg, should have 000 instead of 890:
+		for i in range(2):
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="", cache=cache),
+				"Text 000-123 text 123 '123'")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="family=inet4", cache=cache),
+				"Text 000-345 text 345 '345'")
+			self.assertEqual(
+				self.__action.replaceTag("<banaction> '<abc>'", self.__action._properties, 
+					conditional="family=inet6", cache=cache),
+				"Text 000-567 text 567 '567'")
+		self.assertEqual(len(cache), 3)
+
+
 	def testExecuteActionBan(self):
 		self.__action.actionstart = "touch /tmp/fail2ban.test"
 		self.assertEqual(self.__action.actionstart, "touch /tmp/fail2ban.test")

fail2ban/tests/clientbeautifiertestcase.py

@@ -0,0 +1,261 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Alexander Koeppe"
+__copyright__ = "Copyright (c) 2016 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
+__license__ = "GPL"
+
+import unittest
+
+from ..client.beautifier import Beautifier
+from ..version import version
+from ..server.ipdns import IPAddr
+from ..exceptions import UnknownJailException, DuplicateJailException
+
+class BeautifierTest(unittest.TestCase):
+
+	def setUp(self):
+		""" Call before every test case """
+		self.b = Beautifier()
+
+	def tearDown(self):
+		""" Call after every test case """
+
+	def testGetInputCmd(self):
+		cmd = ["test"]
+		self.b.setInputCmd(cmd)
+		self.assertEqual(self.b.getInputCmd(), cmd)
+
+	def testPing(self):
+		self.b.setInputCmd(["ping"])
+		self.assertEqual(self.b.beautify("pong"), "Server replied: pong")
+	
+	def testVersion(self):
+		self.b.setInputCmd(["version"])
+		self.assertEqual(self.b.beautify(version), version)
+
+	def testAddJail(self):
+		self.b.setInputCmd(["add"])
+		self.assertEqual(self.b.beautify("ssh"), "Added jail ssh")
+
+	def testStartJail(self):
+		self.b.setInputCmd(["start"])
+		self.assertEqual(self.b.beautify(None), "Jail started")
+
+	def testStopJail(self):
+		self.b.setInputCmd(["stop", "ssh"])
+		self.assertEqual(self.b.beautify(None), "Jail stopped")
+
+	def testShutdown(self):
+		self.b.setInputCmd(["stop"])
+		self.assertEqual(self.b.beautify(None), "Shutdown successful")
+
+	def testStatus(self):
+		self.b.setInputCmd(["status"])
+		response = (("Number of jails", 0), ("Jail list", ["ssh", "exim4"]))
+		output = "Status\n|- Number of jails:\t0\n`- Jail list:\tssh exim4"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["status", "ssh"])
+		response = (
+					("Filter", [
+							("Currently failed", 0),
+							("Total failed", 0),
+							("File list", "/var/log/auth.log")
+						]
+					),
+					("Actions", [
+							("Currently banned", 3),
+							("Total banned", 3),
+							("Banned IP list", [
+									IPAddr("192.168.0.1"),
+									IPAddr("::ffff:10.2.2.1"),
+									IPAddr("2001:db8::1")
+								]
+							)
+						]
+					)
+				)
+		output = "Status for the jail: ssh\n"
+		output += "|- Filter\n"
+		output += "|  |- Currently failed:	0\n"
+		output += "|  |- Total failed:	0\n"
+		output += "|  `- File list:	/var/log/auth.log\n"
+		output += "`- Actions\n"
+		output += "   |- Currently banned:	3\n"
+		output += "   |- Total banned:	3\n"
+		output += "   `- Banned IP list:	192.168.0.1 10.2.2.1 2001:db8::1"
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testFlushLogs(self):
+		self.b.setInputCmd(["flushlogs"])
+		self.assertEqual(self.b.beautify("rolled over"), "logs: rolled over")
+	
+	def testSyslogSocket(self):
+		self.b.setInputCmd(["get", "syslogsocket"])
+		output = "Current syslog socket is:\n`- auto"
+		self.assertEqual(self.b.beautify("auto"), output)
+
+	def testLogTarget(self):
+		self.b.setInputCmd(["get", "logtarget"])
+		output = "Current logging target is:\n`- /var/log/fail2ban.log"
+		self.assertEqual(self.b.beautify("/var/log/fail2ban.log"), output)
+
+	def testLogLevel(self):
+		self.b.setInputCmd(["get", "loglevel"])
+		output = "Current logging level is 'INFO'"
+		self.assertEqual(self.b.beautify("INFO"), output)
+
+	def testDbFile(self):
+		self.b.setInputCmd(["get", "dbfile"])
+		response = "/var/lib/fail2ban/fail2ban.sqlite3"
+		output = "Current database file is:\n`- " + response
+		self.assertEqual(self.b.beautify(response), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testDbPurgeAge(self):
+		self.b.setInputCmd(["get", "dbpurgeage"])
+		output = "Current database purge age is:\n`- 86400seconds"
+		self.assertEqual(self.b.beautify(86400), output)
+		self.assertEqual(self.b.beautify(None), "Database currently disabled")
+
+	def testLogPath(self):
+		self.b.setInputCmd(["get", "sshd", "logpath"])
+		response = []
+		output = "No file is currently monitored"
+		self.assertEqual(self.b.beautify(response), output)
+		response = ["/var/log/auth.log"]
+		output = "Current monitored log file(s):\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "addlogpath", "/var/log/messages"])
+		response = ["/var/log/messages", "/var/log/auth.log"]
+		outputadd = "Current monitored log file(s):\n"
+		outputadd += "|- /var/log/messages\n`- /var/log/auth.log"
+		self.assertEqual(self.b.beautify(response), outputadd)
+
+		self.b.setInputCmd(["set", "sshd", "dellogpath", "/var/log/messages"])
+		response = ["/var/log/auth.log"]
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testLogEncoding(self):
+		self.b.setInputCmd(["get", "sshd", "logencoding"])
+		output = "Current log encoding is set to:\nUTF-8"
+		self.assertEqual(self.b.beautify("UTF-8"), output)
+
+	def testJournalMatch(self):
+		self.b.setInputCmd(["get", "sshd", "journalmatch"])
+		self.assertEqual(self.b.beautify([]), "No journal match filter set")
+
+		self.b.setInputCmd(["set", "sshd", "addjournalmatch"])
+		response = [["_SYSTEMD_UNIT", "sshd.service"]]
+		output = "Current match filter:\n"
+		output += "_SYSTEMD_UNIT sshd.service"
+		self.assertEqual(self.b.beautify(response), output)
+		
+		response.append(["_COMM", "sshd"])
+		output += " + _COMM sshd"
+		self.assertEqual(self.b.beautify(response), output)
+
+		self.b.setInputCmd(["set", "sshd", "deljournalmatch"])
+		response.remove(response[1])
+		self.assertEqual(self.b.beautify(response), output.split(" + ")[0])
+
+	def testDatePattern(self):
+		self.b.setInputCmd(["get", "sshd", "datepattern"])
+		output = "Current date pattern set to: "
+		response = (None, "Default Detectors")
+		self.assertEqual(self.b.beautify(None), 
+				output + "Not set/required")
+		self.assertEqual(self.b.beautify(response), 
+				output + "Default Detectors")
+		self.assertEqual(self.b.beautify(("test", "test")), 
+				output + "test (test)")
+
+	def testIgnoreIP(self):
+		self.b.setInputCmd(["get", "sshd", "ignoreip"])
+		output = "No IP address/network is ignored"
+		self.assertEqual(self.b.beautify([]), output)
+
+		self.b.setInputCmd(["set", "sshd", "addignoreip"])
+		response = [
+			IPAddr("127.0.0.0", 8), 
+			IPAddr("::1"), 
+			IPAddr("2001:db8::", 32), 
+			IPAddr("::ffff:10.0.2.1")
+		]
+		output = "These IP addresses/networks are ignored:\n"
+		output += "|- 127.0.0.0/8\n"
+		output += "|- ::1\n"
+		output += "|- 2001:db8::/32\n"
+		output += "`- 10.0.2.1"
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testFailRegex(self):
+		self.b.setInputCmd(["get", "sshd", "failregex"])
+		output = "No regular expression is defined"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The following regular expression are defined:\n"
+		output += "|- [0]: ^$\n`- [1]: .*"
+		self.assertEqual(self.b.beautify(["^$", ".*"]), output)
+
+	def testActions(self):
+		self.b.setInputCmd(["get", "sshd", "actions"])
+		output = "No actions for jail sshd"
+		self.assertEqual(self.b.beautify([]), output)
+		
+		output = "The jail sshd has the following actions:\n"
+		output += "iptables-multiport"
+		self.assertEqual(self.b.beautify(["iptables-multiport"]), output)
+
+	def testActionProperties(self):
+		self.b.setInputCmd(["get", "sshd", "actionproperties", "iptables"])
+		output = "No properties for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following properties:"
+		output += "\nactionban, actionunban"
+		response = ("actionban", "actionunban")
+		self.assertEqual(self.b.beautify(response), output)
+
+	def testActionMethods(self):
+		self.b.setInputCmd(["get", "sshd", "actionmethods", "iptables"])
+		output = "No methods for jail sshd action iptables"
+		self.assertEqual(self.b.beautify([]), output)
+
+		output = "The jail sshd action iptables has the following methods:\n"
+		output += "ban, unban"
+		self.assertEqual(self.b.beautify(["ban", "unban"]), output)
+	
+#	def testException(self):
+#		self.b.setInputCmd(["get", "sshd", "logpath"])
+#		self.assertRaises(self.b.beautify(1), TypeError)
+
+	def testBeautifyError(self):
+		response = UnknownJailException("sshd")
+		output = "Sorry but the jail 'sshd' does not exist"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		response = DuplicateJailException("sshd")
+		output = "The jail 'sshd' already exists"
+		self.assertEqual(self.b.beautifyError(response), output)
+
+		output = "Sorry but the command is invalid"
+		self.assertEqual(self.b.beautifyError(IndexError()), output)

fail2ban/tests/clientreadertestcase.py

@@ -275,7 +275,15 @@ class JailReaderTest(LogCaptureTestCase):
 		# convert and get stream
 		stream = jail.convert()
 		# get action and retrieve agent from it, compare with agent saved in version:
-		act = [o for o in stream if len(o) > 4 and (o[4] == 'agent' or o[4].endswith('badips.py'))]
+		act = []
+		for cmd in stream:
+			if len(cmd) <= 4:
+				continue
+			# differentiate between set and multi-set (wrop it here to single set):
+			if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')):
+				act.append(cmd)
+			elif cmd[0] == 'multi-set':
+				act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent'])
 		useragent = 'Fail2Ban/%s' % version
 		self.assertEqual(len(act), 4)
 		self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent])
@@ -311,23 +319,21 @@ class FilterReaderTest(unittest.TestCase):
 		self.__share_cfg = {}
 
 	def testConvert(self):
-		output = [['set', 'testcase01', 'addfailregex',
+		output = [['multi-set', 'testcase01', 'addfailregex', [
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
-			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"error: PAM: )?Authentication failure for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the underlying authentication mo"
-			"dule for .* from <HOST>\\s*$"],
-			['set', 'testcase01', 'addfailregex',
+			"dule for .* from <HOST>\\s*$",
 			"^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )"
 			"?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|"
 			"[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:"
 			"error: PAM: )?User not known to the\\nunderlying authentication."
-			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"],
+			"+$<SKIPLINES>^.+ module for .* from <HOST>\\s*$"]],
 			['set', 'testcase01', 'addignoreregex', 
 			"^.+ john from host 192.168.1.1\\s*$"],
 			['set', 'testcase01', 'addjournalmatch',
@@ -495,9 +501,11 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertEqual(sorted(comm_commands),
 			sorted([['add', 'emptyaction', 'auto'],
 			 ['add', 'test-known-interp', 'auto'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) <HOST>'],
-			 ['set', 'test-known-interp', 'addfailregex', 'failure test 3 (jail.local) <HOST>'],
+			 ['multi-set', 'test-known-interp', 'addfailregex', [
+			   'failure test 1 (filter.d/test.conf) <HOST>',
+			   'failure test 2 (filter.d/test.local) <HOST>',
+			   'failure test 3 (jail.local) <HOST>'
+			 ]],
 			 ['start', 'test-known-interp'],
 			 ['add', 'missinglogfiles', 'auto'],
 			 ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
@@ -660,12 +668,16 @@ class JailsReaderTest(LogCaptureTestCase):
 						self.assertTrue('blocktype' in action._initOpts)
 						# Verify that we have a call to set it up
 						blocktype_present = False
-						target_command = ['set', jail_name, 'action', action_name, 'blocktype']
+						target_command = [jail_name, 'action', action_name]
 						for command in commands:
-							if (len(command) > 5 and
-								command[:5] == target_command):
-								blocktype_present = True
-								continue
+							if (len(command) > 4 and command[0] == 'multi-set' and
+								command[1:4] == target_command):
+									blocktype_present = ('blocktype' in [cmd[0] for cmd in command[4]])
+							elif (len(command) > 5 and command[0] == 'set' and
+								command[1:4] == target_command and command[4] == 'blocktype'): # pragma: no cover - because of multi-set
+									blocktype_present = True
+							if blocktype_present:
+								break
 						self.assertTrue(
 							blocktype_present,
 							msg="Found no %s command among %s"

fail2ban/tests/failmanagertestcase.py

@@ -28,6 +28,7 @@ import unittest
 
 from ..server import failmanager
 from ..server.failmanager import FailManager, FailManagerEmpty
+from ..server.ipdns import IPAddr
 from ..server.ticket import FailTicket
 
 
@@ -140,7 +141,7 @@ class AddFailure(unittest.TestCase):
 		#ticket = FailTicket('193.168.0.128', None)
 		ticket = self.__failManager.toBan()
 		self.assertEqual(ticket.getIP(), "193.168.0.128")
-		self.assertTrue(isinstance(ticket.getIP(), str))
+		self.assertTrue(isinstance(ticket.getIP(), (str, IPAddr)))
 
 		# finish with rudimentary tests of the ticket
 		# verify consistent str

fail2ban/tests/files/logs/apache-auth

@@ -8,6 +8,8 @@
 
 # failJSON: { "time": "2013-07-11T01:21:43", "match": true , "host": "194.228.20.113" } 
 [Thu Jul 11 01:21:43 2013] [error] [client 194.228.20.113] user dsfasdf not found: /
+# failJSON: { "time": "2013-07-11T01:21:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Thu Jul 11 01:21:44 2013] [error] [client 2606:2800:220:1:248:1893:25c8:1946] user test-ipv6 not found: /
 
 # The failures below use the configuration described in fail2ban/tests/files/config/apache-auth
 #
@@ -56,6 +58,8 @@
 
 # failJSON: { "time": "2013-07-20T22:11:43", "match": true , "host": "127.0.0.1" } 
 [Sat Jul 20 22:11:43.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client 127.0.0.1:51548] AH01637: Authorization of user username to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
+# failJSON: { "time": "2013-07-20T22:11:44", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } 
+[Sat Jul 20 22:11:44.147674 2013] [authz_owner:error] [pid 17540:tid 140122922129152] [client [2606:2800:220:1:248:1893:25c8:1946]:51548] AH01637: Authorization of user test-ipv6 to access /basic/authz_owner/cant_get_me.html failed, reason: file owner dan does not match
 
 # wget --http-user=username --http-password=password http://localhost/basic/authz_owner/cant_get_me.html -O /dev/null
 # failJSON: { "time": "2013-07-20T21:42:44", "match": true , "host": "127.0.0.1" } 

fail2ban/tests/files/logs/sshd

@@ -3,6 +3,8 @@
 Jun 21 16:47:48 digital-mlhhyiqscv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6
 # failJSON: { "time": "2005-05-29T20:56:52", "match": true , "host": "example.com" }
 May 29 20:56:52 imago sshd[28732]: error: PAM: Authentication failure for stefanor from example.com
+# failJSON: { "time": "2005-05-29T20:56:56", "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" }
+May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-ipv6 from 2606:2800:220:1:248:1893:25c8:1946
 
 #2
 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" }

fail2ban/tests/filtertestcase.py

@@ -38,8 +38,9 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, DNSUtils
+from ..server.filter import Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
+from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
@@ -154,19 +155,41 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None):
 
 	Test filter to contain target ticket
 	"""
+	# one or multiple tickets:
+	if not isinstance(output[0], (tuple,list)):
+		tickcount = 1
+		failcount = (count if count else output[1])
+	else:
+		tickcount = len(output)
+		failcount = (count if count else sum((o[1] for o in output)))
+
+	found = []
 	if isinstance(filter_, DummyJail):
 		# get fail ticket from jail
-		found = _ticket_tuple(filter_.getFailTicket())
+		found.append(_ticket_tuple(filter_.getFailTicket()))
 	else:
 		# when we are testing without jails
 		# wait for failures (up to max time)
 		Utils.wait_for(
-			lambda: filter_.failManager.getFailTotal() >= (count if count else output[1]),
+			lambda: filter_.failManager.getFailCount() >= (tickcount, failcount),
 			_maxWaitTime(10))
-		# get fail ticket from filter
-		found = _ticket_tuple(filter_.failManager.toBan())
+		# get fail ticket(s) from filter
+		while tickcount:
+			try:
+				found.append(_ticket_tuple(filter_.failManager.toBan()))
+			except FailManagerEmpty:
+				break
+			tickcount -= 1
 
-	_assert_equal_entries(utest, found, output, count)
+	if not isinstance(output[0], (tuple,list)):
+		utest.assertEqual(len(found), 1)
+		_assert_equal_entries(utest, found[0], output, count)
+	else:
+		# sort by string representation of ip (multiple failures with different ips):
+		found = sorted(found, key=lambda x: str(x))
+		output = sorted(output, key=lambda x: str(x))
+		for f, o in zip(found, output):
+			_assert_equal_entries(utest, f, o)
 
 
 def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""):
@@ -315,6 +338,10 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.1.255'))
 		self.assertFalse(self.filter.inIgnoreIPList('192.168.0.255'))
 
+	def testWrongIPMask(self):
+		self.filter.addIgnoreIP('192.168.1.0/255.255.0.0')
+		self.assertRaises(ValueError, self.filter.addIgnoreIP, '192.168.1.0/255.255.0.128')
+
 	def testIgnoreInProcessLine(self):
 		setUpMyTime()
 		self.filter.addIgnoreIP('192.168.1.0/25')
@@ -345,16 +372,21 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertNotLogged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED"))
 
 
-class IgnoreIPDNS(IgnoreIP):
+class IgnoreIPDNS(LogCaptureTestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
 		unittest.F2B.SkipIfNoNetwork()
-		IgnoreIP.setUp(self)
+		LogCaptureTestCase.setUp(self)
+		self.jail = DummyJail()
+		self.filter = FileFilter(self.jail)
 
 	def testIgnoreIPDNSOK(self):
 		self.filter.addIgnoreIP("www.epfl.ch")
 		self.assertTrue(self.filter.inIgnoreIPList("128.178.50.12"))
+		self.filter.addIgnoreIP("example.com")
+		self.assertTrue(self.filter.inIgnoreIPList("93.184.216.34"))
+		self.assertTrue(self.filter.inIgnoreIPList("2606:2800:220:1:248:1893:25c8:1946"))
 
 	def testIgnoreIPDNSNOK(self):
 		# Test DNS
@@ -1109,18 +1141,18 @@ class GetFailures(LogCaptureTestCase):
 		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailures04(self):
-		output = [('212.41.96.186', 4, 1124013600.0),
-				  ('212.41.96.185', 4, 1124017198.0)]
+		# because of not exact time in testcase04.log (no year), we should always use our test time:
+		self.assertEqual(MyTime.time(), 1124013600)
+		# should find exact 4 failures for *.186 and 2 failures for *.185
+		output = (('212.41.96.186', 4, 1124013600.0),
+				  ('212.41.96.185', 2, 1124013598.0))
 
+		self.filter.setMaxRetry(2)
 		self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0)
 		self.filter.addFailRegex("Invalid user .* <HOST>")
 		self.filter.getFailures(GetFailures.FILENAME_04)
 
-		try:
-			for i, out in enumerate(output):
-				_assert_correct_last_attempt(self, self.filter, out)
-		except FailManagerEmpty:
-			pass
+		_assert_correct_last_attempt(self, self.filter, output)
 
 	def testGetFailuresWrongChar(self):
 		# write wrong utf-8 char:
@@ -1160,20 +1192,31 @@ class GetFailures(LogCaptureTestCase):
 	def testGetFailuresUseDNS(self):
 		unittest.F2B.SkipIfNoNetwork()
 		# We should still catch failures with usedns = no ;-)
-		output_yes = ('93.184.216.34', 2, 1124013539.0,
-					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
-					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_yes = (
+			('93.184.216.34', 2, 1124013539.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2',
+			   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			),
+			('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0,
+			  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2']
+			),
+		)
 
-		output_no = ('93.184.216.34', 1, 1124013539.0,
-					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'])
+		output_no = (
+			('93.184.216.34', 1, 1124013539.0,
+			  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2']
+			)
+		)
 
 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,
 		#				  FileFilter, None, useDns='wrong_value_for_useDns')
 
-		for useDns, output in (('yes',  output_yes),
-							   ('no',   output_no),
-							   ('warn', output_yes)):
+		for useDns, output in (
+			('yes',  output_yes),
+			('no',   output_no),
+			('warn', output_yes)
+		):
 			jail = DummyJail()
 			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.active = True
@@ -1314,9 +1357,9 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		res = DNSUtils.textToIp('www.example.com', 'no')
 		self.assertEqual(res, [])
 		res = DNSUtils.textToIp('www.example.com', 'warn')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 		res = DNSUtils.textToIp('www.example.com', 'yes')
-		self.assertEqual(res, ['93.184.216.34'])
+		self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 
 	def testTextToIp(self):
 		# Test hostnames
@@ -1328,7 +1371,7 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		for s in hostnames:
 			res = DNSUtils.textToIp(s, 'yes')
 			if s == 'www.example.com':
-				self.assertEqual(res, ['93.184.216.34'])
+				self.assertEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'])
 			else:
 				self.assertEqual(res, [])
 
@@ -1341,20 +1384,95 @@ class DNSUtilsNetworkTests(unittest.TestCase):
 		self.assertEqual(res, None)
 
 	def testAddr2bin(self):
-		res = DNSUtils.addr2bin('10.0.0.0')
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=None)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.0', cidr=32L)
-		self.assertEqual(res, 167772160L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=32L)
-		self.assertEqual(res, 167772161L)
-		res = DNSUtils.addr2bin('10.0.0.1', cidr=31L)
-		self.assertEqual(res, 167772160L)
+		res = IPAddr('10.0.0.0')
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=None)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.0', cidr=32L)
+		self.assertEqual(res.addr, 167772160L)
+		res = IPAddr('10.0.0.1', cidr=32L)
+		self.assertEqual(res.addr, 167772161L)
+		res = IPAddr('10.0.0.1', cidr=31L)
+		self.assertEqual(res.addr, 167772160L)
 
-	def testBin2addr(self):
-		res = DNSUtils.bin2addr(167772160L)
-		self.assertEqual(res, '10.0.0.0')
+	def testIPAddr_Equal6(self):
+		self.assertEqual(
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:0:0')
+		)
+
+	def testIPAddr_Compare(self):
+		ip4 = [
+			IPAddr('93.184.0.1'),
+			IPAddr('93.184.216.1'),
+			IPAddr('93.184.216.34')
+		]
+		ip6 = [
+			IPAddr('2606:2800:220:1:248:1893::'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:0'),
+			IPAddr('2606:2800:220:1:248:1893:25c8:1946')
+		]
+		# ip4
+		self.assertNotEqual(ip4[0], None)
+		self.assertTrue(ip4[0] is not None)
+		self.assertFalse(ip4[0] is None)
+		self.assertTrue(ip4[0] < ip4[1])
+		self.assertTrue(ip4[1] < ip4[2])
+		self.assertEqual(sorted(reversed(ip4)), ip4)
+		# ip6
+		self.assertNotEqual(ip6[0], None)
+		self.assertTrue(ip6[0] is not None)
+		self.assertFalse(ip6[0] is None)
+		self.assertTrue(ip6[0] < ip6[1])
+		self.assertTrue(ip6[1] < ip6[2])
+		self.assertEqual(sorted(reversed(ip6)), ip6)
+		# ip4 vs ip6
+		self.assertNotEqual(ip4[0], ip6[0])
+		self.assertTrue(ip4[0] < ip6[0])
+		self.assertTrue(ip4[2] < ip6[2])
+		self.assertEqual(sorted(reversed(ip4+ip6)), ip4+ip6)
+		# hashing (with string as key):
+		d={
+			'93.184.216.34': 'ip4-test', 
+			'2606:2800:220:1:248:1893:25c8:1946': 'ip6-test'
+		}
+		d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()])
+		self.assertTrue(isinstance(d.keys()[0], basestring))
+		self.assertTrue(isinstance(d2.keys()[0], IPAddr))
+		self.assertEqual(d.get(ip4[2], ''), 'ip4-test')
+		self.assertEqual(d.get(ip6[2], ''), 'ip6-test')
+		self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test')
+		self.assertEqual(d2.get(str(ip6[2]), ''), 'ip6-test')
+		# compare with string direct:
+		self.assertEqual(d, d2)
+
+	def testIPAddr_CIDR(self):
+		self.assertEqual(str(IPAddr('93.184.0.1', 24)), '93.184.0.0/24')
+		self.assertEqual(str(IPAddr('192.168.1.0/255.255.255.128')), '192.168.1.0/25')
+		self.assertEqual(IPAddr('93.184.0.1', 24).ntoa, '93.184.0.0/24')
+		self.assertEqual(IPAddr('192.168.1.0/255.255.255.128').ntoa, '192.168.1.0/25')
+
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8::', 120)), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8::', 120).ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(str(IPAddr('2606:2800:220:1:248:1893:25c8:0/120')), '2606:2800:220:1:248:1893:25c8:0/120')
+		self.assertEqual(IPAddr('2606:2800:220:1:248:1893:25c8:0/120').ntoa, '2606:2800:220:1:248:1893:25c8:0/120')
+
+	def testIPAddr_CIDR_Repr(self):
+		self.assertEqual(["127.0.0.0/8", "::/32", "2001:db8::/32"],
+			[IPAddr("127.0.0.0", 8), IPAddr("::1", 32), IPAddr("2001:db8::", 32)]
+		)
+
+	def testIPAddr_CompareDNS(self):
+		ips = IPAddr('example.com')
+		self.assertTrue(IPAddr("93.184.216.34").isInNet(ips))
+		self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips))
+
+	def testIPAddr_Cached(self):
+		ips = [DNSUtils.dnsToIp('example.com'), DNSUtils.dnsToIp('example.com')]
+		for ip1, ip2 in zip(ips, ips):
+			self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('93.184.216.34'); ip2 = IPAddr('93.184.216.34'); self.assertEqual(id(ip1), id(ip2))
+		ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2))
 
 
 class JailTests(unittest.TestCase):

fail2ban/tests/samplestestcase.py

@@ -72,14 +72,21 @@ def testSampleRegexsFactory(name):
 		filterConf.getOptions({})
 
 		for opt in filterConf.convert():
-			if opt[2] == "addfailregex":
-				self.filter.addFailRegex(opt[3])
-			elif opt[2] == "maxlines":
-				self.filter.setMaxLines(opt[3])
-			elif opt[2] == "addignoreregex":
-				self.filter.addIgnoreRegex(opt[3])
-			elif opt[2] == "datepattern":
-				self.filter.setDatePattern(opt[3])
+			if opt[0] == 'multi-set':
+				optval = opt[3]
+			elif opt[0] == 'set':
+				optval = [opt[3]]
+			else:
+				continue
+			for optval in optval:
+				if opt[2] == "addfailregex":
+					self.filter.addFailRegex(optval)
+				elif opt[2] == "addignoreregex":
+					self.filter.addIgnoreRegex(optval)
+				elif opt[2] == "maxlines":
+					self.filter.setMaxLines(optval)
+				elif opt[2] == "datepattern":
+					self.filter.setDatePattern(optval)
 
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),

fail2ban/tests/servertestcase.py

@@ -33,7 +33,9 @@ import sys
 import platform
 
 from ..server.failregex import Regex, FailRegex, RegexException
+from ..server import actions as _actions
 from ..server.server import Server
+from ..server.ipdns import IPAddr
 from ..server.jail import Jail
 from ..server.jailthread import JailThread
 from ..server.utils import Utils
@@ -49,6 +51,8 @@ except ImportError: # pragma: no cover
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 FAST_BACKEND = "polling"
 
+logSys = getLogger("fail2ban")
+
 
 class TestServer(Server):
 	def setLogLevel(self, *args, **kwargs):
@@ -125,14 +129,14 @@ class TransmitterBase(unittest.TestCase):
 			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdAdd, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[:n+1]))))
 		for n, value in enumerate(values):
 			ret = self.transm.proceed(["set", jail, cmdDel, value])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 			ret = self.transm.proceed(["get", jail, cmd])
-			self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:])))
+			self.assertEqual((ret[0], sorted(map(str, ret[1]))), (0, sorted(map(str, values[n+1:]))))
 
 	def jailAddDelRegexTest(self, cmd, inValues, outValues, jail):
 		cmdAdd = "add" + cmd
@@ -454,9 +458,9 @@ class Transmitter(TransmitterBase):
 				"failed attempt from <HOST> again",
 			],
 			[
-				"user john at (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
-				"failed attempt from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w) again",
+				"user john at %s" % (Regex._resolveHostTag('<HOST>')),
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
+				"failed attempt from %s again" % (Regex._resolveHostTag('<HOST>')),
 			],
 			self.jailName
 		)
@@ -479,7 +483,7 @@ class Transmitter(TransmitterBase):
 			],
 			[
 				"user john",
-				"Admin user login from (?:::f{4,6}:)?(?P<host>[\w\-.^_]*\\w)",
+				"Admin user login from %s" % (Regex._resolveHostTag('<HOST>')),
 				"Dont match me!",
 			],
 			self.jailName
@@ -963,3 +967,423 @@ class LoggingTests(LogCaptureTestCase):
 			sys.__excepthook__ = prev_exchook
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
+
+
+from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK
+
+class ServerConfigReaderTests(LogCaptureTestCase):
+
+	def __init__(self, *args, **kwargs):
+		super(ServerConfigReaderTests, self).__init__(*args, **kwargs)
+		self.__share_cfg = {}
+
+	def setUp(self):
+		"""Call before every test case."""
+		super(ServerConfigReaderTests, self).setUp()
+		self._execCmdLst = []
+	
+	def tearDown(self):
+		"""Call after every test case."""
+		super(ServerConfigReaderTests, self).tearDown()
+
+	def _executeCmd(self, realCmd, timeout=60):
+		for l in realCmd.split('\n'):
+			if not l.startswith('#'):
+				logSys.debug('exec-cmd: `%s`', l)
+			else:
+				logSys.debug(l)
+		return True
+
+	def test_IPAddr(self):
+		self.assertTrue(IPAddr('192.0.2.1').isIPv4)
+		self.assertTrue(IPAddr('2001:DB8::').isIPv6)
+
+	if STOCK:
+
+		def testCheckStockJailActions(self):
+			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
+			self.assertTrue(jails.read())		  # opens fine
+			self.assertTrue(jails.getOptions())	  # reads fine
+			stream = jails.convert(allow_no_files=True)
+
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			# for cmd in stream:
+			# 	print(cmd)
+
+			# filter all start commands (we want not start all jails):
+			for cmd in stream:
+				if cmd[0] != 'start':
+					# change to the fast init backend:
+					if cmd[0] == 'add':
+						cmd[2] = 'polling'
+					# change log path to test log of jail (to prevent "Permission denied" on /var/logs/ for test-user):
+					elif len(cmd) > 3 and cmd[0] == 'set' and cmd[2] == 'addlogpath':
+						cmd[3] = os.path.join(TEST_FILES_DIR, 'logs', cmd[1])
+					# add dummy regex to prevent too long compile of all regexp (we don't use it in this test at all):
+					# [todo sebres] remove `not hasattr(unittest, 'F2B') or `, after merge with "f2b-perfom-prepare-716" ...
+					elif (not hasattr(unittest, 'F2B') or unittest.F2B.fast) and (
+						len(cmd) > 3 and cmd[0] in ('set', 'multi-set') and cmd[2] == 'addfailregex'
+					):
+						cmd[0] = "set"
+						cmd[3] = "DUMMY-REGEX <HOST>"
+					# command to server, use cmdHandler direct instead of `transm.proceed(cmd)`:
+					try:
+						cmdHandler(cmd)
+					except Exception, e: # pragma: no cover
+						self.fail("Command %r has failed. Received %r" % (cmd, e))
+
+			# jails = server._Server__jails
+			# for j in jails:
+			# 	print(j, jails[j])
+
+		def getDefaultJailStream(self, jail, act):
+			act = act.replace('%(__name__)s', jail)
+			actName, actOpt = JailReader.extractOptions(act)
+			stream = [
+				['add', jail, 'polling'],
+				# ['set', jail, 'addfailregex', 'DUMMY-REGEX <HOST>'],
+			]
+			action = ActionReader(
+				actName, jail, actOpt,
+				share_config=self.__share_cfg, basedir=CONFIG_DIR)
+			self.assertTrue(action.read())
+			action.getOptions({})
+			stream.extend(action.convert())
+			return stream
+
+		def testCheckStockCommandActions(self):
+			# test cases to check valid ipv4/ipv6 action definition, tuple with (('jail', 'action[params]', 'tests', ...)
+			# where tests is a dictionary contains:
+			#   'ip4' - should not be found (logged) on ban/unban of IPv6 (negative test),
+			#   'ip6' - should not be found (logged) on ban/unban of IPv4 (negative test),
+			#   'start', 'stop' - should be found (logged) on action start/stop,
+			#   etc.
+			testJailsActions = (
+				# iptables-multiport --
+				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="600", port="http,https", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-mp`",
+						"`iptables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -N f2b-j-w-iptables-mp`",
+						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`iptables -w -F f2b-j-w-iptables-mp`",
+						"`iptables -w -X f2b-j-w-iptables-mp`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
+						"`ip6tables -w -F f2b-j-w-iptables-mp`",
+						"`ip6tables -w -X f2b-j-w-iptables-mp`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-mp 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-mp -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-mp 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-mp -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-allports --
+				('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="600", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-ap`",
+						"`iptables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -N f2b-j-w-iptables-ap`",
+						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`iptables -w -F f2b-j-w-iptables-ap`",
+						"`iptables -w -X f2b-j-w-iptables-ap`",
+						"`ip6tables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
+						"`ip6tables -w -F f2b-j-w-iptables-ap`",
+						"`ip6tables -w -X f2b-j-w-iptables-ap`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-ap 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-ap -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-ap 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-ap -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-ipset-proto6 --
+				('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',),
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`",
+						"`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset`",
+						"`ipset destroy f2b-j-w-iptables-ipset`",
+						"`ip6tables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset6`",
+						"`ipset destroy f2b-j-w-iptables-ipset6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`",
+					),					
+				}),
+				# iptables-ipset-proto6-allports --
+				('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',),
+					'start': (
+						"`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`",
+						"`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
+						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap`",
+						"`ip6tables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap6`",
+						"`ipset destroy f2b-j-w-iptables-ipset-ap6`",
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`",
+					),
+					'ip4-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`",
+					),
+					'ip6-ban': (
+						r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`",
+					),
+					'ip6-unban': (
+						r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`",
+					),					
+				}),
+				# iptables --
+				('j-w-iptables', 'iptables[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables`",
+						"`iptables -w -A f2b-j-w-iptables -j RETURN`",
+						"`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -N f2b-j-w-iptables`",
+						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
+						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`iptables -w -F f2b-j-w-iptables`",
+						"`iptables -w -X f2b-j-w-iptables`",
+						"`ip6tables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
+						"`ip6tables -w -F f2b-j-w-iptables`",
+						"`ip6tables -w -X f2b-j-w-iptables`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-new --
+				('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="600", port="http", protocol="tcp", chain="INPUT"]', {
+					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
+					'start': (
+						"`iptables -w -N f2b-j-w-iptables-new`",
+						"`iptables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -N f2b-j-w-iptables-new`",
+						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
+						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+					),
+					'stop': (
+						"`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`iptables -w -F f2b-j-w-iptables-new`",
+						"`iptables -w -X f2b-j-w-iptables-new`",
+						"`ip6tables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
+						"`ip6tables -w -F f2b-j-w-iptables-new`",
+						"`ip6tables -w -X f2b-j-w-iptables-new`",
+					),
+					'ip4-check': (
+						r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip6-check': (
+						r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""",
+					),
+					'ip4-ban': (
+						r"`iptables -w -I f2b-j-w-iptables-new 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip4-unban': (
+						r"`iptables -w -D f2b-j-w-iptables-new -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`",
+					),
+					'ip6-ban': (
+						r"`ip6tables -w -I f2b-j-w-iptables-new 1 -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),
+					'ip6-unban': (
+						r"`ip6tables -w -D f2b-j-w-iptables-new -s 2001:db8:: -j REJECT --reject-with icmp6-port-unreachable`",
+					),					
+				}),
+				# iptables-xt_recent-echo --
+				('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="600", chain="INPUT"]', {
+					'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'),
+					'start': (
+						"`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'stop': (
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+						"`if [ `id -u` -eq 0 ];then iptables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`",
+						"`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+						"`if [ `id -u` -eq 0 ];then ip6tables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`",
+					),
+					'ip4-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-check': (
+						r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip4-ban': (
+						r"`echo +192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip4-unban': (
+						r"`echo -192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`",
+					),
+					'ip6-ban': (
+						r"`echo +2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),
+					'ip6-unban': (
+						r"`echo -2001:db8:: > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`",
+					),					
+				}),
+				# pf --
+				('j-w-pf', 'pf[name=%(__name__)s]', {
+					'ip4': (), 'ip6': (),
+					'start': (
+						'`echo "table <f2b-j-w-pf> persist counters" | pfctl -f-`',
+ 						'`echo "block proto tcp from <f2b-j-w-pf> to any port any" | pfctl -f-`',
+					),
+					'stop': (
+						'`pfctl -sr 2>/dev/null | grep -v f2b-j-w-pf | pfctl -f-`',
+						'`pfctl -t f2b-j-w-pf -T flush`',
+						'`pfctl -t f2b-j-w-pf -T kill`',
+					),
+					'ip4-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip6-check': ("`pfctl -sr | grep -q f2b-j-w-pf`",),
+					'ip4-ban':   ("`pfctl -t f2b-j-w-pf -T add 192.0.2.1`",),
+					'ip4-unban': ("`pfctl -t f2b-j-w-pf -T delete 192.0.2.1`",),
+					'ip6-ban':   ("`pfctl -t f2b-j-w-pf -T add 2001:db8::`",),
+					'ip6-unban': ("`pfctl -t f2b-j-w-pf -T delete 2001:db8::`",),
+				}),
+			)
+			server = TestServer()
+			transm = server._Server__transm
+			cmdHandler = transm._Transmitter__commandHandler
+
+			for jail, act, tests in testJailsActions:
+				stream = self.getDefaultJailStream(jail, act)
+
+				# for cmd in stream:
+				# 	print(cmd)
+
+				# filter all start commands (we want not start all jails):
+				for cmd in stream:
+					# command to server:
+					ret, res = transm.proceed(cmd)
+					self.assertEqual(ret, 0)
+
+			jails = server._Server__jails
+
+			for jail, act, tests in testJailsActions:
+				# print(jail, jails[jail])
+				for a in jails[jail].actions:
+					action = jails[jail].actions[a]
+					logSys.debug('# ' + ('=' * 50))
+					logSys.debug('# == %-44s ==', jail + ' - ' + action._name)
+					logSys.debug('# ' + ('=' * 50))
+					self.assertTrue(isinstance(action, _actions.CommandAction))
+					# wrap default command processor:
+					action.executeCmd = self._executeCmd
+					# test start :
+					logSys.debug('# === start ==='); self.pruneLog()
+					action.start()
+					self.assertLogged(*tests['start'], all=True)
+					# test ban ip4 :
+					logSys.debug('# === ban-ipv4 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('192.0.2.1')})
+					self.assertLogged(*tests['ip4-check']+tests['ip4-ban'], all=True)
+					self.assertNotLogged(*tests['ip6'], all=True)
+					# test unban ip4 :
+					logSys.debug('# === unban ipv4 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('192.0.2.1')})
+					self.assertLogged(*tests['ip4-check']+tests['ip4-unban'], all=True)
+					self.assertNotLogged(*tests['ip6'], all=True)
+					# test ban ip6 :
+					logSys.debug('# === ban ipv6 ==='); self.pruneLog()
+					action.ban({'ip': IPAddr('2001:DB8::')})
+					self.assertLogged(*tests['ip6-check']+tests['ip6-ban'], all=True)
+					self.assertNotLogged(*tests['ip4'], all=True)
+					# test unban ip6 :
+					logSys.debug('# === unban ipv6 ==='); self.pruneLog()
+					action.unban({'ip': IPAddr('2001:DB8::')})
+					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
+					self.assertNotLogged(*tests['ip4'], all=True)
+					# test stop :
+					logSys.debug('# === stop ==='); self.pruneLog()
+					action.stop()
+					self.assertLogged(*tests['stop'], all=True)
+

fail2ban/tests/utils.py

@@ -32,7 +32,7 @@ import unittest
 from StringIO import StringIO
 
 from ..helpers import getLogger
-from ..server.filter import DNSUtils
+from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils
 # for action_d.test_smtp :
@@ -101,6 +101,16 @@ def initTests(opts):
 		c.set('192.0.2.%s' % i, None)
 		c.set('198.51.100.%s' % i, None)
 		c.set('203.0.113.%s' % i, None)
+	if unittest.F2B.no_network: # pragma: no cover
+		# precache all wrong dns to ip's used in test cases:
+		c = DNSUtils.CACHE_nameToIp
+		for i in (
+			('999.999.999.999', []),
+			('abcdef.abcdef', []),
+			('192.168.0.', []),
+			('failed.dns.ch', []),
+		):
+			c.set(*i)
 
 
 def mtimesleep():
@@ -133,6 +143,7 @@ def gatherTests(regexps=None, opts=None):
 	# Import all the test cases here instead of a module level to
 	# avoid circular imports
 	from . import banmanagertestcase
+	from . import clientbeautifiertestcase
 	from . import clientreadertestcase
 	from . import tickettestcase
 	from . import failmanagertestcase
@@ -173,6 +184,7 @@ def gatherTests(regexps=None, opts=None):
 	tests.addTest(unittest.makeSuite(servertestcase.JailTests))
 	tests.addTest(unittest.makeSuite(servertestcase.RegexTests))
 	tests.addTest(unittest.makeSuite(servertestcase.LoggingTests))
+	tests.addTest(unittest.makeSuite(servertestcase.ServerConfigReaderTests))
 	tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest))
 	tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions))
 	# Ticket, BanTicket, FailTicket
@@ -187,6 +199,10 @@ def gatherTests(regexps=None, opts=None):
 		tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo))
 	except ImportError: # pragma: no cover
 		pass
+	
+	# ClientBeautifier
+	tests.addTest(unittest.makeSuite(clientbeautifiertestcase.BeautifierTest))
+
 	# ClientReaders
 	tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))
@@ -319,7 +335,7 @@ class LogCaptureTestCase(unittest.TestCase):
 	def _is_logged(self, s):
 		return s in self._log.getvalue()
 
-	def assertLogged(self, *s):
+	def assertLogged(self, *s, **kwargs):
 		"""Assert that one of the strings was logged
 
 		Preferable to assertTrue(self._is_logged(..)))
@@ -329,14 +345,23 @@ class LogCaptureTestCase(unittest.TestCase):
 		----------
 		s : string or list/set/tuple of strings
 		  Test should succeed if string (or any of the listed) is present in the log
+		all : boolean (default False) if True should fail if any of s not logged
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ in logged:
-				return
-		raise AssertionError("None among %r was found in the log: %r" % (s, logged))
+		if not kwargs.get('all', False):
+			# at least one entry should be found:
+			for s_ in s:
+				if s_ in logged:
+					return
+			if True: # pragma: no cover
+				self.fail("None among %r was found in the log: ===\n%s===" % (s, logged))
+		else:
+			# each entry should be found:
+			for s_ in s:
+				if s_ not in logged: # pragma: no cover
+					self.fail("%r was not found in the log: ===\n%s===" % (s_, logged))
 
-	def assertNotLogged(self, *s):
+	def assertNotLogged(self, *s, **kwargs):
 		"""Assert that strings were not logged
 
 		Parameters
@@ -344,13 +369,22 @@ class LogCaptureTestCase(unittest.TestCase):
 		s : string or list/set/tuple of strings
 		  Test should succeed if the string (or at least one of the listed) is not
 		  present in the log
+		all : boolean (default False) if True should fail if any of s logged
 		"""
 		logged = self._log.getvalue()
-		for s_ in s:
-			if s_ not in logged:
-				return
-		raise AssertionError("All of the %r were found present in the log: %r" % (s, logged))
+		if not kwargs.get('all', False):
+			for s_ in s:
+				if s_ not in logged:
+					return
+			if True: # pragma: no cover
+				self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged))
+		else:
+			for s_ in s:
+				if s_ in logged: # pragma: no cover
+					self.fail("%r was found in the log: ===\n%s===" % (s_, logged))
 
+	def pruneLog(self):
+		self._log.truncate(0)
 
 	def getLog(self):
 		return self._log.getvalue()