Merge '0.10.3.fix1' into 0.10

ChangeLog

@@ -44,6 +44,9 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition
 ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right
 -----------
 
+### ver. 0.10.3.1:
+* fixed JSON serialization for the set-object within dump into database (gh-2103).
+
 ### Fixes
 * `filter.d/asterisk.conf`: fixed failregex prefix by log over remote syslog server (gh-2060);
 * `filter.d/exim.conf`: failregex extended - SMTP call dropped: too many syntax or protocol errors (gh-2048);

fail2ban/server/database.py

@@ -39,9 +39,14 @@ from ..helpers import getLogger, PREFER_ENC
 logSys = getLogger(__name__)
 
 if sys.version_info >= (3,):
+	def _json_default(x):
+		if isinstance(x, set):
+			x = list(x)
+		return x
+
 	def _json_dumps_safe(x):
 		try:
-			x = json.dumps(x, ensure_ascii=False).encode(
+			x = json.dumps(x, ensure_ascii=False, default=_json_default).encode(
 				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
@@ -60,7 +65,7 @@ else:
 	def _normalize(x):
 		if isinstance(x, dict):
 			return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems())
-		elif isinstance(x, list):
+		elif isinstance(x, (list, set)):
 			return [_normalize(element) for element in x]
 		elif isinstance(x, unicode):
 			return x.encode(PREFER_ENC)
@@ -527,10 +532,13 @@ class Fail2BanDb(object):
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
+		data = ticket.getData()
+		matches = data.get('matches')
+		if matches and len(matches) > self.maxEntries:
+			data['matches'] = matches[-self.maxEntries:]
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ip, int(round(ticket.getTime())),
+			(jail.name, ip, int(round(ticket.getTime())), data))
-				ticket.getData()))
 
 	@commitandrollback
 	def delBan(self, cur, jail, *args):
@@ -659,11 +667,11 @@ class Fail2BanDb(object):
 						else:
 							matches = m[-maxadd:] + matches
 					failures += data.get('failures', 1)
-					tickdata.update(data.get('data', {}))
+					data['failures'] = failures
+					data['matches'] = matches
+					tickdata.update(data)
 					prev_timeofban = timeofban
-				ticket = FailTicket(banip, prev_timeofban, matches)
+				ticket = FailTicket(banip, prev_timeofban, data=tickdata)
-				ticket.setAttempt(failures)
-				ticket.setData(**tickdata)
 				tickets.append(ticket)
 
 			if cacheKey:

fail2ban/tests/action_d/test_badips.py

@@ -20,6 +20,8 @@
 import os
 import unittest
 import sys
+from socket import timeout
+from ssl import SSLError
 
 from ..actiontestcase import CallingMap
 from ..dummyjail import DummyJail
@@ -51,7 +53,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 						BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName)
 					BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action
 					self.jail.actions._load_python_module(pythonModuleName)
-				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 60)
+				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30)
 			if not BadIPsActionTest.available[0]:
 				raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1])
 
@@ -62,7 +64,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 				'score': 5,
 				'key': "fail2ban-test-suite",
 				#'bankey': "fail2ban-test-suite",
-				'timeout': (3 if unittest.F2B.fast else 30),
+				'timeout': (3 if unittest.F2B.fast else 60),
 				})
 			self.action = self.jail.actions["badips"]
 
@@ -108,11 +110,16 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			self.action.updateperiod = "900"
 
 		def testStartStop(self):
-			self.action.start()
+			try:
-			self.assertTrue(len(self.action._bannedips) > 10,
+				self.action.start()
-				"%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips))
+				self.assertTrue(len(self.action._bannedips) > 10,
-			self.action.stop()
+					"%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips))
-			self.assertTrue(len(self.action._bannedips) == 0)
+				self.action.stop()
+				self.assertTrue(len(self.action._bannedips) == 0)
+			except (SSLError, timeout) as e: # pragma: no cover - timeout only
+				if not isinstance(e, timeout) and 'timed out' not in str(e):
+					raise
+				raise unittest.SkipTest('Skip test because of %s' % e)
 
 		def testBanIP(self):
 			aInfo = CallingMap({

fail2ban/tests/databasetestcase.py

@@ -302,12 +302,18 @@ class DatabaseTest(LogCaptureTestCase):
 	def testGetBansMerged_MaxEntries(self):
 		self.testAddJail()
 		maxEntries = 2
-		failures = ["abc\n", "123\n", "ABC\n", "1234\n"]
+		failures = [
+			{"matches": ["abc\n"], "user": set(['test'])},
+			{"matches": ["123\n"], "user": set(['test'])},
+			{"matches": ["ABC\n"], "user": set(['test', 'root'])},
+			{"matches": ["1234\n"], "user": set(['test', 'root'])},
+		]
+		matches2find = [f["matches"][0] for f in failures]
 		# add failures sequential:
 		i = 80
 		for f in failures:
 			i -= 10
-			ticket = FailTicket("127.0.0.1", MyTime.time() - i, [f])
+			ticket = FailTicket("127.0.0.1", MyTime.time() - i, data=f)
 			ticket.setAttempt(1)
 			self.db.addBan(self.jail, ticket)
 		# should retrieve 2 matches only, but count of all attempts:
@@ -316,9 +322,10 @@ class DatabaseTest(LogCaptureTestCase):
 		self.assertEqual(ticket.getIP(), "127.0.0.1")
 		self.assertEqual(ticket.getAttempt(), len(failures))
 		self.assertEqual(len(ticket.getMatches()), maxEntries)
-		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
     # add more failures at once:
-		ticket = FailTicket("127.0.0.1", MyTime.time() - 10, failures)
+		ticket = FailTicket("127.0.0.1", MyTime.time() - 10, matches2find,
+			data={"user": set(['test', 'root'])})
 		ticket.setAttempt(len(failures))
 		self.db.addBan(self.jail, ticket)
 		# should retrieve 2 matches only, but count of all attempts:
@@ -326,7 +333,13 @@ class DatabaseTest(LogCaptureTestCase):
 		ticket = self.db.getBansMerged("127.0.0.1")
 		self.assertEqual(ticket.getAttempt(), 2 * len(failures))
 		self.assertEqual(len(ticket.getMatches()), maxEntries)
-		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
+		# also using getCurrentBans:
+		ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100)
+		self.assertTrue(ticket is not None)
+		self.assertEqual(ticket.getAttempt(), len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
 
 	def testGetBansMerged(self):
 		self.testAddJail()