background servicing (temporally executed from failmanager): prevents memory leak on some platforms/python versions, using forced GC in periodic intervals (latency and threshold);

Side effect: GC is disabled now inside fail2ban-server (to avoid multiple garbage collect)

fail2ban/helpers.py

@@ -20,11 +20,16 @@
 __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko"
 __license__ = "GPL"
 
-import sys
+import gc
-import os
-import traceback
-import re
 import logging
+import os
+import re
+import sys
+import traceback
+
+from threading import Lock
+
+from .server.mytime import MyTime
 
 
 def formatExceptionInfo():
@@ -137,3 +142,48 @@ def splitcommaspace(s):
 	if not s:
 		return []
 	return filter(bool, re.split('[ ,]', s))
+
+
+class BgService(object):
+	"""Background servicing
+
+	Prevents memory leak on some platforms/python versions, 
+	using forced GC in periodical intervals.
+	"""
+
+	_mutex = Lock()
+	_instance = None
+	def __new__(cls):
+		if not cls._instance:
+			cls._instance = \
+				super(BgService, cls).__new__(cls)
+		return cls._instance
+
+	def __init__(self):
+		self.__serviceTime = -0x7fffffff
+		self.__periodTime = 30
+		self.__threshold = 100;
+		self.__count = self.__threshold;
+		if hasattr(gc, 'set_threshold'):
+			gc.set_threshold(0)
+		gc.disable()
+
+	def service(self, force=False, wait=False):
+		self.__count -= 1
+		# avoid locking if next service time don't reached
+		if not force and (self.__count > 0 or MyTime.time() < self.__serviceTime):
+			return False
+		# return immediately if mutex already locked (other thread in servicing):
+		if not BgService._mutex.acquire(wait):
+			return False
+		try:
+			# check again in lock:
+			if MyTime.time() < self.__serviceTime:
+				return False
+			gc.collect()
+			self.__serviceTime = MyTime.time() + self.__periodTime
+			self.__count = self.__threshold
+			return True
+		finally:
+			BgService._mutex.release()
+		return False

fail2ban/server/failmanager.py

@@ -28,7 +28,7 @@ from threading import Lock
 import logging
 
 from .ticket import FailTicket
-from ..helpers import getLogger
+from ..helpers import getLogger, BgService
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -43,6 +43,7 @@ class FailManager:
 		self.__maxRetry = 3
 		self.__maxTime = 600
 		self.__failTotal = 0
+		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
 		with self.__lock:
@@ -102,6 +103,8 @@ class FailManager:
 											  for k,v in  self.__failList.iteritems()])
 				logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
 							 % (self.__failTotal, len(self.__failList), failures_summary))
+
+		self.__bgSvc.service()
 		return attempts
 	
 	def size(self):
@@ -126,6 +129,7 @@ class FailManager:
 				# create new dictionary without items to be deleted:
 				self.__failList = dict((ip,item) for ip,item in self.__failList.iteritems() \
 					if item.getLastTime() + self.__maxTime > time)
+		self.__bgSvc.service()
 	
 	def delFailure(self, ip):
 		with self.__lock:
@@ -141,7 +145,8 @@ class FailManager:
 				if data.getRetry() >= self.__maxRetry:
 					del self.__failList[ip]
 					return data
-			raise FailManagerEmpty
+		self.__bgSvc.service()
+		raise FailManagerEmpty
 
 
 class FailManagerEmpty(Exception):

fail2ban/tests/failmanagertestcase.py

@@ -122,6 +122,29 @@ class AddFailure(unittest.TestCase):
 		self.assertNotEqual(ticket.getIP(), "100.100.10.10")
 		self.assertRaises(FailManagerEmpty, self.__failManager.toBan)
 
+	def testBgService(self):
+		bgSvc = self.__failManager._FailManager__bgSvc
+		failManager2nd = FailManager()
+		# test singleton (same object):
+		bgSvc2 = failManager2nd._FailManager__bgSvc
+		self.assertTrue(id(bgSvc) == id(bgSvc2))
+		bgSvc2 = None
+		# test service :
+		self.assertTrue(bgSvc.service(True, True))
+		self.assertFalse(bgSvc.service())
+		# bypass threshold and time:
+		for i in range(1, bgSvc._BgService__threshold):
+			self.assertFalse(bgSvc.service())
+		# bypass time check:
+		bgSvc._BgService__serviceTime = -0x7fffffff
+		self.assertTrue(bgSvc.service())
+		# bypass threshold and time:
+		bgSvc._BgService__serviceTime = -0x7fffffff
+		for i in range(1, bgSvc._BgService__threshold):
+			self.assertFalse(bgSvc.service())
+		self.assertTrue(bgSvc.service(False, True))
+		self.assertFalse(bgSvc.service(False, True))
+
 
 class FailmanagerComplex(unittest.TestCase):