temp commit: reload now supported actions and action reloading (parameters, unban obsolete removed actions, etc.)

fail2ban/client/actionreader.py

@@ -38,6 +38,7 @@ class ActionReader(DefinitionInitConfigReader):
 	_configOpts = {
 		"actionstart": ["string", None],
 		"actionstop": ["string", None],
+		"actionreload": ["string", None],
 		"actioncheck": ["string", None],
 		"actionrepair": ["string", None],
 		"actionban": ["string", None],

fail2ban/server/action.py

@@ -201,6 +201,7 @@ class CommandAction(ActionBase):
 	----------
 	actionban
 	actioncheck
+	actionreload
 	actionrepair
 	actionstart
 	actionstop
@@ -210,24 +211,35 @@ class CommandAction(ActionBase):
 
 	_escapedTags = set(('matches', 'ipmatches', 'ipjailmatches'))
 
-	timeout = 60
-	## Command executed in order to initialize the system.
-	actionstart = ''
-	## Command executed when an IP address gets banned.
-	actionban = ''
-	## Command executed when an IP address gets removed.
-	actionunban = ''
-	## Command executed in order to check requirements.
-	actioncheck = ''
-	## Command executed in order to restore sane environment in error case.
-	actionrepair = ''
-	## Command executed in order to stop the system.
-	actionstop = ''
+	def clearAllParams(self):
+		""" Clear all lists/dicts parameters (used by reloading)
+		"""
+		self.__init = 1
+		try:
+			self.timeout = 60
+			## Command executed in order to initialize the system.
+			self.actionstart = ''
+			## Command executed when an IP address gets banned.
+			self.actionban = ''
+			## Command executed when an IP address gets removed.
+			self.actionunban = ''
+			## Command executed in order to check requirements.
+			self.actioncheck = ''
+			## Command executed in order to restore sane environment in error case.
+			self.actionrepair = ''
+			## Command executed in order to stop the system.
+			self.actionstop = ''
+			## Command executed in case of reloading action.
+			self.actionreload = ''
+		finally:
+			self.__init = 0
 
 	def __init__(self, jail, name):
 		super(CommandAction, self).__init__(jail, name)
+		self.__init = 1
 		self.__properties = None
 		self.__substCache = {}
+		self.clearAllParams()
 		self._logSys.debug("Created %s" % self.__class__)
 
 	@classmethod
@@ -235,7 +247,7 @@ class CommandAction(ActionBase):
 		return NotImplemented # Standard checks
 
 	def __setattr__(self, name, value):
-		if not name.startswith('_') and not callable(value):
+		if not name.startswith('_') and not self.__init and not callable(value):
 			# special case for some pasrameters:
 			if name in ('timeout', 'bantime'):
 				value = str(MyTime.str2seconds(value))
@@ -268,8 +280,8 @@ class CommandAction(ActionBase):
 	def _substCache(self):
 		return self.__substCache
 
-	def start(self):
-		"""Executes the "actionstart" command.
+	def _executeOperation(self, tag, operation):
+		"""Executes the operation commands (like "actionstart", "actionstop", etc).
 
 		Replace the tags in the action command with actions properties
 		and executes the resulting command.
@@ -278,20 +290,28 @@ class CommandAction(ActionBase):
 		res = True
 		try:
 			# common (resp. ipv4):
-			startCmd = self.replaceTag('<actionstart>', self._properties, 
+			startCmd = self.replaceTag(tag, self._properties, 
 				conditional='family=inet4', cache=self.__substCache)
 			if startCmd:
 				res &= self.executeCmd(startCmd, self.timeout)
 			# start ipv6 actions if available:
 			if allowed_ipv6:
-				startCmd6 = self.replaceTag('<actionstart>', self._properties, 
+				startCmd6 = self.replaceTag(tag, self._properties, 
 					conditional='family=inet6', cache=self.__substCache)
 				if startCmd6 and startCmd6 != startCmd:
 					res &= self.executeCmd(startCmd6, self.timeout)
 			if not res:
-				raise RuntimeError("Error starting action %s/%s" % (self._jail, self._name,))
+				raise RuntimeError("Error %s action %s/%s" % (operation, self._jail, self._name,))
 		except ValueError as e:
-			raise RuntimeError("Error starting action %s/%s: %r" % (self._jail, self._name, e))
+			raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e))
+
+	def start(self):
+		"""Executes the "actionstart" command.
+
+		Replace the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		return self._executeOperation('<actionstart>', 'starting')
 
 	def ban(self, aInfo):
 		"""Executes the "actionban" command.
@@ -329,20 +349,20 @@ class CommandAction(ActionBase):
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
-		res = True
-		# common (resp. ipv4):
-		stopCmd = self.replaceTag('<actionstop>', self._properties, 
-			conditional='family=inet4', cache=self.__substCache)
-		if stopCmd:
-			res &= self.executeCmd(stopCmd, self.timeout)
-		# ipv6 actions if available:
-		if allowed_ipv6:
-			stopCmd6 = self.replaceTag('<actionstop>', self._properties, 
-				conditional='family=inet6', cache=self.__substCache)
-			if stopCmd6 and stopCmd6 != stopCmd:
-				res &= self.executeCmd(stopCmd6, self.timeout)
-		if not res:
-			raise RuntimeError("Error stopping action")
+		return self._executeOperation('<actionstop>', 'stopping')
+
+	def reload(self, **kwargs):
+		"""Executes the "actionreload" command.
+
+		Parameters
+		----------
+		kwargs : dict
+		  Currently unused, because CommandAction do not support initOpts
+
+		Replaces the tags in the action command with actions properties
+		and executes the resulting command.
+		"""
+		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
 	def substituteRecursiveTags(cls, inptags, conditional=''):

fail2ban/server/actions.py

@@ -36,7 +36,7 @@ from collections import Mapping
 try:
 	from collections import OrderedDict
 except ImportError:
-	OrderedDict = None
+	OrderedDict = dict
 
 from .banmanager import BanManager
 from .jailthread import JailThread
@@ -81,14 +81,11 @@ class Actions(JailThread, Mapping):
 		JailThread.__init__(self)
 		## The jail which contains this action.
 		self._jail = jail
-		if OrderedDict is not None:
-			self._actions = OrderedDict()
-		else:
-			self._actions = dict()
+		self._actions = OrderedDict()
 		## The ban manager.
 		self.__banManager = BanManager()
 
-	def add(self, name, pythonModule=None, initOpts=None):
+	def add(self, name, pythonModule=None, initOpts=None, reload=False):
 		"""Adds a new action.
 
 		Add a new action if not already present, defaulting to standard
@@ -116,7 +113,17 @@ class Actions(JailThread, Mapping):
 		"""
 		# Check is action name already exists
 		if name in self._actions:
-			raise ValueError("Action %s already exists" % name)
+			if not reload:
+				raise ValueError("Action %s already exists" % name)
+			# don't create new action if reload supported:
+			action = self._actions[name]
+			if hasattr(action, 'reload'):
+				# don't execute reload right now, reload after all parameters are actualized
+				if hasattr(action, 'clearAllParams'):
+					action.clearAllParams()
+					self._reload_actions[name] = initOpts
+				return
+		## Create new action:
 		if pythonModule is None:
 			action = CommandAction(self._jail, name)
 		else:
@@ -138,6 +145,27 @@ class Actions(JailThread, Mapping):
 			action = customActionModule.Action(self._jail, name, **initOpts)
 		self._actions[name] = action
 
+	def reload(self, begin=True):
+		""" Begin or end of reloading resp. refreshing of all parameters
+		"""
+		if begin:
+			self._reload_actions = dict()
+		else:
+			if hasattr(self, '_reload_actions'):
+				# reload actions after all parameters set via stream:
+				for name, initOpts in self._reload_actions.iteritems():
+					if name in self._actions:
+						self._actions[name].reload(**initOpts if initOpts else {})
+				# remove obsolete actions (untouched by reload process):
+				delacts = OrderedDict((name, action) for name, action in self._actions.iteritems()
+					if name not in self._reload_actions)
+				if len(delacts):
+					# unban all tickets using remove action only:
+					self.__flushBan(db=False, actions=delacts)
+					# stop and remove it:
+					self.stopActions(actions=delacts)
+				delattr(self, '_reload_actions')
+
 	def __getitem__(self, name):
 		try:
 			return self._actions[name]
@@ -215,6 +243,24 @@ class Actions(JailThread, Mapping):
 		return 1
 
 
+	def stopActions(self, actions=None):
+		"""Stops the actions in reverse sequence (optionally filtered)
+		"""
+		if actions is None:
+			actions = self._actions
+		revactions = actions.items()
+		revactions.reverse()
+		for name, action in revactions:
+			try:
+				action.stop()
+			except Exception as e:
+				logSys.error("Failed to stop jail '%s' action '%s': %s",
+					self._jail.name, name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+			del self._actions[name]
+			logSys.debug("%s: action %s terminated", self._jail.name, name)
+
+
 	def run(self):
 		"""Main loop for Threading.
 
@@ -239,18 +285,9 @@ class Actions(JailThread, Mapping):
 				continue
 			if not Utils.wait_for(self.__checkBan, self.sleeptime):
 				self.__checkUnBan()
+		
 		self.__flushBan()
-
-		actions = self._actions.items()
-		actions.reverse()
-		for name, action in actions:
-			try:
-				action.stop()
-			except Exception as e:
-				logSys.error("Failed to stop jail '%s' action '%s': %s",
-					self._jail.name, name, e,
-					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
-		logSys.debug(self._jail.name + ": action terminated")
+		self.stopActions()
 		return True
 
 	def __getBansMerged(self, mi, overalljails=False):
@@ -355,26 +392,33 @@ class Actions(JailThread, Mapping):
 				cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __flushBan(self, db=False):
+	def __flushBan(self, db=False, actions=None):
 		"""Flush the ban list.
 
 		Unban all IP address which are still in the banning list.
+
+		If actions specified, don't flush list - just execute unban for 
+		given actions (reload, obsolete resp. removed actions).
 		"""
-		logSys.debug("Flush ban list")
-		lst = self.__banManager.flushBanList()
+		if actions is None:
+			logSys.debug("Flush ban list")
+			lst = self.__banManager.flushBanList()
+		else:
+			lst = iter(self.__banManager)
+		cnt = 0
 		for ticket in lst:
 			# delete ip from database also:
 			if db and self._jail.database is not None:
 				ip = str(ticket.getIP())
 				self._jail.database.delBan(self._jail, ip)
 			# unban ip:
-			self.__unBan(ticket)
-		cnt = len(lst)
+			self.__unBan(ticket, actions=actions)
+			cnt += 1
 		logSys.debug("Unbanned %s, %s ticket(s) in %r", 
 			cnt, self.__banManager.size(), self._jail.name)
 		return cnt
 
-	def __unBan(self, ticket):
+	def __unBan(self, ticket, actions=None):
 		"""Unbans host corresponding to the ticket.
 
 		Executes the actions in order to unban the host given in the
@@ -385,13 +429,15 @@ class Actions(JailThread, Mapping):
 		ticket : FailTicket
 			Ticket of failures of which to unban
 		"""
+		if actions is None:
+			actions = self._actions
 		aInfo = dict()
 		aInfo["ip"] = ticket.getIP()
 		aInfo["failures"] = ticket.getAttempt()
 		aInfo["time"] = ticket.getTime()
 		aInfo["matches"] = "".join(ticket.getMatches())
 		logSys.notice("[%s] Unban %s" % (self._jail.name, aInfo["ip"]))
-		for name, action in self._actions.iteritems():
+		for name, action in actions.iteritems():
 			try:
 				action.unban(aInfo.copy())
 			except Exception as e:

fail2ban/server/banmanager.py

@@ -106,6 +106,15 @@ class BanManager:
 		with self.__lock:
 			return self.__banList.keys()
 
+	##
+	# Returns a iterator to ban list (used in reload, so idle).
+	#
+	# @return ban list iterator
+	
+	def __iter__(self):
+		with self.__lock:
+			return self.__banList.itervalues()
+
 	##
 	# Returns normalized value
 	#

fail2ban/server/database.py

@@ -604,16 +604,14 @@ class Fail2BanDb(object):
 		results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
 
 		if results:
-			matches = []
-			failures = 0
 			for banip, timeofban, data in results:
-				#TODO: Implement data parts once arbitrary match keys completed
-				ticket = FailTicket(banip, timeofban, matches)
-				ticket.setAttempt(failures)
 				matches = []
 				failures = 0
-				matches.extend(data['matches'])
-				failures += data['failures']
+				if isinstance(data['matches'], list):
+					matches.extend(data['matches'])
+				if data['failures']:
+					failures += data['failures']
+				ticket = FailTicket(banip, timeofban, matches)
 				ticket.setAttempt(failures)
 				tickets.append(ticket)
 

fail2ban/server/filter.py

@@ -117,14 +117,11 @@ class Filter(JailThread):
 				self._reload_logs = dict((k, 1) for k in self.getLogPaths())
 		else:
 			if hasattr(self, '_reload_logs'):
-				dellogs = dict()
 				# if it was not reloaded - remove obsolete log file:
 				for path in self._reload_logs:
 					self.delLogPath(path)
 				delattr(self, '_reload_logs')
 
-
-
 	##
 	# Add a regular expression which matches the failure.
 	#

fail2ban/server/server.py

@@ -267,6 +267,7 @@ class Server:
 						jail.idle = True
 						self.__reload_state[jn] = jail
 						jail.filter.reload(begin=True)
+						jail.actions.reload(begin=True)
 				pass
 		else:
 			# end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started:
@@ -280,6 +281,7 @@ class Server:
 					else:
 						# commit (reload was finished):
 						jail.filter.reload(begin=False)
+						jail.actions.reload(begin=False)
 				for jn in deljails:
 					self.delJail(jn)
 			self.__reload_state = {}
@@ -416,7 +418,9 @@ class Server:
 	
 	# Action
 	def addAction(self, name, value, *args):
-		self.__jails[name].actions.add(value, *args)
+		## create (or reload) jail action:
+		self.__jails[name].actions.add(value, *args, 
+			reload=name in self.__reload_state)
 	
 	def getActions(self, name):
 		return self.__jails[name].actions

fail2ban/tests/fail2banclienttestcase.py

@@ -677,6 +677,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		test2log = pjoin(tmp, "test2.log")
 		test3log = pjoin(tmp, "test3.log")
 
+		os.mkdir(pjoin(cfg, "action.d"))
+		def _write_action_cfg(actname="test-action1"):
+			fn = pjoin(cfg, "action.d", "%s.conf" % actname)
+			_write_file(fn, "w",
+				"[Definition]",
+				"actionstart = echo '[<name>] %s: ** start'" % actname,
+				"actionstop =  echo '[<name>] %s: -- unban <ip>'" % actname,
+				"actionreload = echo '[<name>] %s: ** reload'" % actname,
+				"actionban =   echo '[<name>] %s: ++ ban <ip>'" % actname,
+				"actionunban = echo '[<name>] %s: // stop'" % actname,
+			)
+			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
+				_out_file(fn)
+
 		def _write_jail_cfg(enabled=[1, 2]):
 			_write_file(pjoin(cfg, "jail.conf"), "w",
 				"[INCLUDES]", "",
@@ -686,7 +700,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"findtime = 10m",
 				"failregex = ^\s*failure (401|403) from <HOST>",
 				"",
-				"[test-jail1]", "backend = polling", "filter =", "action =",
+				"[test-jail1]", "backend = polling", "filter =", 
+				"action = test-action1[name='%(__name__)s']",
+				"         test-action2[name='%(__name__)s']",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
@@ -701,6 +717,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 				_out_file(pjoin(cfg, "jail.conf"))
 
+		# create default test actions:
+		_write_action_cfg(actname="test-action1")
+		_write_action_cfg(actname="test-action2")
+
 		_write_jail_cfg(enabled=[1])
 		_write_file(test1log, "w", *((str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 1",) * 3))
 		_write_file(test2log, "w")
@@ -737,6 +757,11 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"Added logfile: %r" % test2log, 
 			"Added logfile: %r" % test3log, all=True)
+		# test actions reloaded:
+		self.assertLogged(
+			"echo '[test-jail1] test-action1: ** reload' -- returned successfully", 
+			"echo '[test-jail1] test-action2: ** reload' -- returned successfully", all=True)
+
 		# test 1 new jail:
 		self.assertLogged(
 			"Creating new jail 'test-jail2'",