Merge pull request #1743 from sebres/0.10-flush-bulk-unban

0.10 - flush resp. bulk unban

config/action.d/dummy.conf

@@ -10,14 +10,23 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = touch /var/run/fail2ban/fail2ban.dummy
+actionstart = if [ ! -z '<target>' ]; then touch <target>; fi;
-              printf %%b "<init>\n" >> /var/run/fail2ban/fail2ban.dummy
+              printf %%b "<init>\n" <to_target>
+              echo "%(debug)s started"
+
+# Option:  actionflush
+# Notes.:  command executed once to flush (clear) all IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = printf %%b "-*\n" <to_target>
+              echo "%(debug)s clear all"
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
-actionstop = rm -f /var/run/fail2ban/fail2ban.dummy
+actionstop = if [ ! -z '<target>' ]; then rm -f <target>; fi;
+             echo "%(debug)s stopped"
 
 # Option:  actioncheck
 # Notes.:  command executed once before each actionban command
@@ -31,7 +40,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "+<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
+actionban = printf %%b "+<ip>\n" <to_target>
+            echo "%(debug)s banned <ip> (family: <family>)"
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -39,9 +49,15 @@ actionban = printf %%b "+<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = printf %%b "-<ip>\n" >> /var/run/fail2ban/fail2ban.dummy
+actionunban = printf %%b "-<ip>\n" <to_target>
+              echo "%(debug)s unbanned <ip> (family: <family>)"
+
+
+debug = [<name>] <actname> <target> --
 
 [Init]
 
 init = 123
 
+target = /var/run/fail2ban/fail2ban.dummy
+to_target = >> <target>

config/action.d/iptables-allports.conf

@@ -26,7 +26,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck

config/action.d/iptables-common.conf

@@ -16,6 +16,14 @@ after = iptables-blocktype.local
         iptables-common.local
 # iptables-blocktype.local is obsolete
 
+[Definition]
+
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = <iptables> -F f2b-<name>
+
 
 [Init]
 

config/action.d/iptables-ipset-proto4.conf

@@ -30,12 +30,19 @@ before = iptables-common.conf
 actionstart = ipset --create f2b-<name> iphash
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
 
+
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset --flush f2b-<name>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>
-             ipset --flush f2b-<name>
+             <actionflush>
              ipset --destroy f2b-<name>
 
 # Option:  actionban

config/action.d/iptables-ipset-proto6-allports.conf

@@ -29,12 +29,18 @@ before = iptables-common.conf
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset flush <ipmset>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>
-             ipset flush <ipmset>
+             <actionflush>
              ipset destroy <ipmset>
 
 # Option:  actionban

config/action.d/iptables-ipset-proto6.conf

@@ -29,12 +29,18 @@ before = iptables-common.conf
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = ipset flush <ipmset>
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>
-             ipset flush <ipmset>
+             <actionflush>
              ipset destroy <ipmset>
 
 # Option:  actionban

config/action.d/iptables-multiport-log.conf

@@ -26,13 +26,19 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I f2b-<name>-log -j LOG --log-prefix "$(expr f2b-<name> : '\(.\{1,23\}\)'):DROP " --log-level warning -m limit --limit 6/m --limit-burst 2
               <iptables> -A f2b-<name>-log -j <blocktype>
 
+# Option:  actionflush
+# Notes.:  command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action)
+# Values:  CMD
+#
+actionflush = <iptables> -F f2b-<name>
+              <iptables> -F f2b-<name>-log
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
-             <iptables> -F f2b-<name>-log
              <iptables> -X f2b-<name>
              <iptables> -X f2b-<name>-log
 

config/action.d/iptables-multiport.conf

@@ -23,7 +23,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck

config/action.d/iptables-new.conf

@@ -25,7 +25,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck

config/action.d/iptables-xt_recent-echo.conf

@@ -35,6 +35,12 @@ before = iptables-common.conf
 #    shorter of the two timeouts actually matters.
 actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi
 
+# Option:  actionflush
+#
+# [TODO] Flushing is currently not implemented for xt_recent
+#
+actionflush = 
+
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
 # Values:  CMD

config/action.d/iptables.conf

@@ -23,7 +23,7 @@ actionstart = <iptables> -N f2b-<name>
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> --dport <port> -j f2b-<name>
-             <iptables> -F f2b-<name>
+             <actionflush>
              <iptables> -X f2b-<name>
 
 # Option:  actioncheck

fail2ban/client/actionreader.py

@@ -40,6 +40,7 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionstart": ["string", None],
 		"actionstart_on_demand": ["string", None],
 		"actionstop": ["string", None],
+		"actionflush": ["string", None],
 		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],

fail2ban/server/action.py

@@ -281,6 +281,8 @@ class CommandAction(ActionBase):
 			self.actioncheck = ''
 			## Command executed in order to restore sane environment in error case.
 			self.actionrepair = ''
+			## Command executed in order to flush all bans at once (e. g. by stop/shutdown the system).
+			self.actionflush = ''
 			## Command executed in order to stop the system.
 			self.actionstop = ''
 			## Command executed in case of reloading action.
@@ -447,6 +449,25 @@ class CommandAction(ActionBase):
 		if not self._processCmd('<actionunban>', aInfo):
 			raise RuntimeError("Error unbanning %(ip)s" % aInfo)
 
+	def flush(self):
+		"""Executes the "actionflush" command.
+		
+		Command executed in order to flush all bans at once (e. g. by stop/shutdown 
+		the system), instead of unbunning of each single ticket.
+
+		Replaces the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		family = []
+		# cumulate started families, if started on demand (conditional):
+		if self._startOnDemand:
+			for f in CommandAction.COND_FAMILIES:
+				if self.__started.get(f) == 1: # only real started:
+					family.append(f)
+			# if no started (on demand) actions:
+			if not family: return True
+		return self._executeOperation('<actionflush>', 'flushing', family=family)
+
 	def stop(self):
 		"""Executes the "actionstop" command.
 

fail2ban/server/actions.py

@@ -447,25 +447,37 @@ class Actions(JailThread, Mapping):
 		If actions specified, don't flush list - just execute unban for 
 		given actions (reload, obsolete resp. removed actions).
 		"""
+		log = True
 		if actions is None:
 			logSys.debug("Flush ban list")
 			lst = self.__banManager.flushBanList()
 		else:
+			log = False # don't log "[jail] Unban ..." if removing actions only.
 			lst = iter(self.__banManager)
 		cnt = 0
+		# first we'll execute flush for actions supporting this operation:
+		unbactions = {}
+		for name, action in (actions if actions is not None else self._actions).iteritems():
+			if hasattr(action, 'flush') and action.actionflush:
+				logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name)
+				action.flush()
+			else:
+				unbactions[name] = action
+		actions = unbactions
+		# unban each ticket with non-flasheable actions:
 		for ticket in lst:
 			# delete ip from database also:
 			if db and self._jail.database is not None:
 				ip = str(ticket.getIP())
 				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
-			self.__unBan(ticket, actions=actions)
+			self.__unBan(ticket, actions=actions, log=log)
 			cnt += 1
 		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
 			cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __unBan(self, ticket, actions=None):
+	def __unBan(self, ticket, actions=None, log=True):
 		"""Unbans host corresponding to the ticket.
 
 		Executes the actions in order to unban the host given in the
@@ -482,7 +494,7 @@ class Actions(JailThread, Mapping):
 			unbactions = actions
 		ip = ticket.getIP()
 		aInfo = self.__getActionInfo(ticket)
-		if actions is None:
+		if log:
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
 			try:

fail2ban/tests/fail2banclienttestcase.py

@@ -762,6 +762,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"norestored = %(_exec_once)s",
 				"restore = ",
 				"info = ",
+				"_use_flush_ = echo [<name>] <actname>: -- flushing IPs",
 				"actionstart =  echo '[%(name)s] %(actname)s: ** start'", start,
 				"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
 				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s%(info)s'", ban,
@@ -788,7 +789,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 					if 1 in actions else "",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>',"
+										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
@@ -802,7 +804,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \
 					if 2 in actions else "",
-				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']"
+										" actionflush=<_use_flush_>]" \
 					if 3 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
@@ -874,6 +877,12 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Creating new jail 'test-jail2'",
 			"Jail 'test-jail2' started", all=True)
+		# test action3 removed, test flushing successful (and no single unban occurred):
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action3: -- flushing IPs'",
+			"stdout: '[test-jail1] test-action3: __ stop'", all=True)
+		self.assertNotLogged(
+			"stdout: '[test-jail1] test-action3: -- unban 192.0.2.1'")
 		
 		# update action1, delete action2 (should be stopped via configuration)...
 		self.pruneLog("[test-phase 2a]")
@@ -969,12 +978,18 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'",
 			all=True)
 
-		# don't need actions anymore:
+		# ban manually to test later flush by unban all:
-		_write_action_cfg(actname="test-action2", allow=False)
+		self.pruneLog("[test-phase 2d]")
-		_write_jail_cfg(actions=[])
+		self.execSuccess(startparams,
+			"set", "test-jail2", "banip", "192.0.2.21")
+		self.execSuccess(startparams,
+			"set", "test-jail2", "banip", "192.0.2.22")
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22 ", all=True, wait=MID_WAITTIME)
 
 		# restart jail with unban all:
-		self.pruneLog("[test-phase 2d]")
+		self.pruneLog("[test-phase 2e]")
 		self.execSuccess(startparams,
 			"restart", "--unban", "test-jail2")
 		self.assertLogged(
@@ -986,12 +1001,26 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Unban 192.0.2.4",
 			"[test-jail2] Unban 192.0.2.8", all=True
 		)
+		# test unban (action2):
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action2: -- unban 192.0.2.21",
+			"stdout: '[test-jail2] test-action2: -- unban 192.0.2.22'", all=True)
+		# test flush (action3, and no single unban via action3 occurred):
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action3: -- flushing IPs'")
+		self.assertNotLogged(
+			"stdout: '[test-jail2] test-action3: -- unban 192.0.2.21'",
+			"stdout: '[test-jail2] test-action3: -- unban 192.0.2.22'", all=True)
 		# no more ban (unbanned all):
 		self.assertNotLogged(
 			"[test-jail2] Ban 192.0.2.4",
 			"[test-jail2] Ban 192.0.2.8", all=True
 		)
 
+		# don't need actions anymore:
+		_write_action_cfg(actname="test-action2", allow=False)
+		_write_jail_cfg(actions=[])
+
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
 		self.execSuccess(startparams, "reload", "test-jail1")

fail2ban/tests/servertestcase.py

@@ -1182,6 +1182,33 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 			#   'start', 'stop' - should be found (logged) on action start/stop,
 			#   etc.
 			testJailsActions = (
+				# dummy --
+				('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', {
+					'ip4': ('family: inet4',), 'ip6': ('family: inet6',),
+					'start': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`',
+					), 
+					'flush': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- clear all"`',
+					),
+					'stop': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- stopped"`',
+					),
+					'ip4-check': (),
+					'ip6-check': (),
+					'ip4-ban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 192.0.2.1 (family: inet4)"`',
+					),
+					'ip4-unban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 192.0.2.1 (family: inet4)"`',
+					),
+					'ip6-ban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- banned 2001:db8:: (family: inet6)"`',
+					),
+					'ip6-unban': (
+						'`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- unbanned 2001:db8:: (family: inet6)"`',
+					),					
+				}),
 				# iptables-multiport --
 				('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="INPUT"]', {
 					'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'),
@@ -1195,6 +1222,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-mp`",
+						"`ip6tables -w -F f2b-j-w-iptables-mp`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`",
 						"`iptables -w -F f2b-j-w-iptables-mp`",
@@ -1235,6 +1266,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-ap`",
+						"`ip6tables -w -F f2b-j-w-iptables-ap`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`",
 						"`iptables -w -F f2b-j-w-iptables-ap`",
@@ -1273,6 +1308,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
+					'flush': (
+						"`ipset flush f2b-j-w-iptables-ipset`",
+						"`ipset flush f2b-j-w-iptables-ipset6`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-iptables-ipset`",
@@ -1307,6 +1346,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`",
 						"`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`",
 					),
+					'flush': (
+						"`ipset flush f2b-j-w-iptables-ipset-ap`",
+						"`ipset flush f2b-j-w-iptables-ipset-ap6`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`",
 						"`ipset flush f2b-j-w-iptables-ipset-ap`",
@@ -1343,6 +1386,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables -j RETURN`",
 						"`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables`",
+						"`ip6tables -w -F f2b-j-w-iptables`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`",
 						"`iptables -w -F f2b-j-w-iptables`",
@@ -1383,6 +1430,10 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 						"`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`",
 						"`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
 					),
+					'flush': (
+						"`iptables -w -F f2b-j-w-iptables-new`",
+						"`ip6tables -w -F f2b-j-w-iptables-new`",
+					),
 					'stop': (
 						"`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`",
 						"`iptables -w -F f2b-j-w-iptables-new`",
@@ -1684,6 +1735,11 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 					action.unban(ainfo['ip6'])
 					self.assertLogged(*tests['ip6-check']+tests['ip6-unban'], all=True)
 					self.assertNotLogged(*tests['ip4'], all=True)
+					# test flush for actions should supported this:
+					if tests.get('flush'):
+						self.pruneLog('# === flush ===')
+						action.flush()
+						self.assertLogged(*tests['flush'], all=True)
 					# test stop :
 					self.pruneLog('# === stop ===')
 					action.stop()