Merge branch '0.10' into 0.11

fail2ban/client/fail2banregex.py

@@ -694,6 +694,14 @@ def exec_command_line(*args):
 	stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt)))
 	logSys.addHandler(stdout)
 
-	fail2banRegex = Fail2banRegex(opts)
+	try:
+		fail2banRegex = Fail2banRegex(opts)
+	except Exception as e:
+		if opts.verbose or logSys.getEffectiveLevel()<=logging.DEBUG:
+			logSys.critical(e, exc_info=True)
+		else:
+			output( 'ERROR: %s' % e )
+		sys.exit(255)
+
 	if not fail2banRegex.start(args):
 		sys.exit(255)

fail2ban/server/datetemplate.py

@@ -301,14 +301,17 @@ class DatePatternRegex(DateTemplate):
 		if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern):
 			pattern = RE_EXLINE_BOUND_BEG.sub('', pattern)
 			wordBegin = 'start'
-		# wrap to regex:
-		fmt = self._patternRE.sub(r'%(\1)s', pattern)
-		self.name = fmt % self._patternName
-		regex = fmt % timeRE
-		# if expected add (?iu) for "ignore case" and "unicode":
-		if RE_ALPHA_PATTERN.search(pattern):
-			regex = r'(?iu)' + regex
-		super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
+		try:
+			# wrap to regex:
+			fmt = self._patternRE.sub(r'%(\1)s', pattern)
+			self.name = fmt % self._patternName
+			regex = fmt % timeRE
+			# if expected add (?iu) for "ignore case" and "unicode":
+			if RE_ALPHA_PATTERN.search(pattern):
+				regex = r'(?iu)' + regex
+			super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
+		except Exception as e:
+			raise TypeError("Failed to set datepattern '%s' (may be an invalid format or unescaped percent char): %s" % (pattern, e))
 
 	def getDate(self, line, dateMatch=None, default_tz=None):
 		"""Method to return the date for a log line.

fail2ban/server/ipdns.py

@@ -197,7 +197,7 @@ class IPAddr(object):
 	__slots__ = '_family','_addr','_plen','_maskplen','_raw'
 
 	# todo: make configurable the expired time and max count of cache entries:
-	CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60)
+	CACHE_OBJ = Utils.Cache(maxCount=10000, maxTime=5*60)
 
 	CIDR_RAW = -2
 	CIDR_UNSPEC = -1
@@ -205,6 +205,10 @@ class IPAddr(object):
 	FAM_IPv6 = CIDR_RAW - socket.AF_INET6
 
 	def __new__(cls, ipstr, cidr=CIDR_UNSPEC):
+		if cidr == IPAddr.CIDR_RAW: # don't cache raw
+			ip = super(IPAddr, cls).__new__(cls)
+			ip.__init(ipstr, cidr)
+			return ip
 		# check already cached as IPAddr
 		args = (ipstr, cidr)
 		ip = IPAddr.CACHE_OBJ.get(args)
@@ -221,7 +225,8 @@ class IPAddr(object):
 					return ip
 		ip = super(IPAddr, cls).__new__(cls)
 		ip.__init(ipstr, cidr)
-		IPAddr.CACHE_OBJ.set(args, ip)
+		if ip._family != IPAddr.CIDR_RAW:
+			IPAddr.CACHE_OBJ.set(args, ip)
 		return ip
 
 	@staticmethod

fail2ban/server/utils.py

@@ -95,31 +95,35 @@ class Utils():
 			
 		def set(self, k, v):
 			t = time.time()
-			cache = self._cache  # for shorter local access
-			# clean cache if max count reached:
-			if len(cache) >= self.maxCount:
-				# avoid multiple modification of list multi-threaded:
-				with self.__lock:
-					if len(cache) >= self.maxCount:
-						for (ck, cv) in cache.items():
+			# avoid multiple modification of dict multi-threaded:
+			cache = self._cache
+			with self.__lock:
+				# clean cache if max count reached:
+				if len(cache) >= self.maxCount:
+					if OrderedDict is not dict:
+						# ordered (so remove some from ahead, FIFO)
+						while cache:
+							(ck, cv) = cache.popitem(last=False)
+							# if not yet expired (but has free slot for new entry):
+							if cv[1] > t and len(cache) < self.maxCount:
+								break
+					else: # pragma: 3.x no cover (dict is in 2.6 only)
+						remlst = []
+						for (ck, cv) in cache.iteritems():
 							# if expired:
 							if cv[1] <= t:
-								self.unset(ck)
-							elif OrderedDict is not dict:
-								break
+								remlst.append(ck)
+						for ck in remlst:
+							self._cache.pop(ck, None)
 						# if still max count - remove any one:
-						if len(cache) >= self.maxCount:
-							if OrderedDict is not dict: # first (older):
-								cache.popitem(False)
-							else: # pragma: 3.x no cover
-								cache.popitem()
-			cache[k] = (v, t + self.maxTime)
+						while cache and len(cache) >= self.maxCount:
+							cache.popitem()
+				# set now:
+				cache[k] = (v, t + self.maxTime)
 
 		def unset(self, k):
-			try:
-				del self._cache[k]
-			except KeyError:
-				pass
+			with self.__lock:
+				self._cache.pop(k, None)
 
 
 	@staticmethod

fail2ban/tests/fail2banregextestcase.py

@@ -368,3 +368,16 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			r"Authentication failure"
 		), 0)
 		self.assertLogged('No failure-id group in ')
+
+	def testExecCmdLine_ErrorParam(self):
+		# single line error:
+		self.assertNotEqual(_test_exec_command_line(
+			'-l', 'notice', '-d', '%:%.%-', 'LOG', 'RE'
+		), 0)
+		self.assertLogged('ERROR: Failed to set datepattern')
+		# verbose (traceback/callstack):
+		self.pruneLog()
+		self.assertNotEqual(_test_exec_command_line(
+			'-v', '-d', '%:%.%-', 'LOG', 'RE'
+		), 0)
+		self.assertLogged('Failed to set datepattern')

fail2ban/tests/filtertestcase.py

@@ -1753,6 +1753,44 @@ class DNSUtilsTests(unittest.TestCase):
 		# here the whole cache should be empty:
 		self.assertEqual(len(c), 0)
 		
+	def testOverflowedIPCache(self):
+		# test overflow of IP-cache multi-threaded (2 "parasite" threads flooding cache):
+		from threading import Thread
+		from random import shuffle
+		# save original cache and use smaller cache during the test here:
+		_org_cache = IPAddr.CACHE_OBJ
+		cache = IPAddr.CACHE_OBJ = Utils.Cache(maxCount=5, maxTime=60)
+		result = list()
+		count = 1 if unittest.F2B.fast else 50
+		try:
+			# tester procedure of worker:
+			def _TestCacheStr2IP(forw=True, result=[], random=False):
+				try:
+					c = count
+					while c:
+						c -= 1
+						s = xrange(0, 256, 1) if forw else xrange(255, -1, -1)
+						if random: shuffle([i for i in s])
+						for i in s:
+							IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4)
+							IPAddr('2001:db8::'+str(i), IPAddr.FAM_IPv6)
+					result.append(None)
+				except Exception as e:
+					DefLogSys.debug(e, exc_info=True)
+					result.append(e)
+
+			# 2 workers flooding it forwards and backwards:
+			th1 = Thread(target=_TestCacheStr2IP, args=(True,  result)); th1.start()
+			th2 = Thread(target=_TestCacheStr2IP, args=(False, result)); th2.start()
+			# and here we flooding it with random IPs too:
+			_TestCacheStr2IP(True, result, True)
+		finally:
+			# wait for end of threads and restore cache:
+			th1.join()
+			th2.join()
+			IPAddr.CACHE_OBJ = _org_cache
+		self.assertEqual(result, [None]*3) # no errors
+		self.assertTrue(len(cache) <= cache.maxCount)
 
 
 class DNSUtilsNetworkTests(unittest.TestCase):