closes gh-2277: fixed and optimized cache facilities (operations on OrderedDict are not atomic); increased max-size of IPAddr cache; don't cache raw objects (it is fast enough).

fail2ban/server/ipdns.py

@@ -197,7 +197,7 @@ class IPAddr(object):
 	__slots__ = '_family','_addr','_plen','_maskplen','_raw'
 
 	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_OBJ = Utils.Cache(maxCount=10000, maxTime=5*60)
 
 	CIDR_RAW = -2
 	CIDR_UNSPEC = -1
@@ -205,6 +205,10 @@ class IPAddr(object):
 	FAM_IPv6 = CIDR_RAW - socket.AF_INET6
 
 	def __new__(cls, ipstr, cidr=CIDR_UNSPEC):
+		if cidr == IPAddr.CIDR_RAW: # don't cache raw
+			ip = super(IPAddr, cls).__new__(cls)
+			ip.__init(ipstr, cidr)
+			return ip
 		# check already cached as IPAddr
 		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
@@ -221,7 +225,8 @@ class IPAddr(object):
 					return ip
 		ip = super(IPAddr, cls).__new__(cls)
 		ip.__init(ipstr, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
+		if ip._family != IPAddr.CIDR_RAW:
+			IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
 	@staticmethod

fail2ban/server/utils.py

@@ -95,31 +95,35 @@ class Utils():
 			
 		def set(self, k, v):
 			t = time.time()
-			cache = self._cache  # for shorter local access
+			# avoid multiple modification of dict multi-threaded:
-			# clean cache if max count reached:
+			cache = self._cache
-			if len(cache) >= self.maxCount:
+			with self.__lock:
-				# avoid multiple modification of list multi-threaded:
+				# clean cache if max count reached:
-				with self.__lock:
+				if len(cache) >= self.maxCount:
-					if len(cache) >= self.maxCount:
+					if OrderedDict is not dict:
-						for (ck, cv) in cache.items():
+						# ordered (so remove some from ahead, FIFO)
+						while cache:
+							(ck, cv) = cache.popitem(last=False)
+							# if not yet expired (but has free slot for new entry):
+							if cv[1] > t and len(cache) < self.maxCount:
+								break
+					else: # pragma: 3.x no cover (dict is in 2.6 only)
+						remlst = []
+						for (ck, cv) in cache.iteritems():
 							# if expired:
 							if cv[1] <= t:
-								self.unset(ck)
+								remlst.append(ck)
-							elif OrderedDict is not dict:
+						for ck in remlst:
-								break
+							self._cache.pop(ck, None)
 						# if still max count - remove any one:
-						if len(cache) >= self.maxCount:
+						while cache and len(cache) >= self.maxCount:
-							if OrderedDict is not dict: # first (older):
+							cache.popitem()
-								cache.popitem(False)
+				# set now:
-							else: # pragma: 3.x no cover
+				cache[k] = (v, t + self.maxTime)
-								cache.popitem()
-			cache[k] = (v, t + self.maxTime)
 
 		def unset(self, k):
-			try:
+			with self.__lock:
-				del self._cache[k]
+				self._cache.pop(k, None)
-			except KeyError:
-				pass
 
 
 	@staticmethod