BF: binding parameter error (unsupported type) by writing json with invalid encoded lines into sqlite database (gh-973);

especially python < 3.0; try to prevent occurring such errors in the future;

ChangeLog

@@ -38,6 +38,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
      system authentication issues
    * fail2ban-regex reads filter file(s) completely, incl. '.local' file etc. (gh-954)
    * firewallcmd-* actions: split output into separate lines for grepping (gh-908)
+   * binding parameter error (unsupported type) by writing json with invalid encoded
+     lines into sqlite database (gh-973), thanks to kot for issue reporting;
 
 - New Features:
    - New filters:

fail2ban/server/database.py

@@ -37,17 +37,35 @@ from ..helpers import getLogger
 logSys = getLogger(__name__)
 
 if sys.version_info >= (3,):
-	sqlite3.register_adapter(
-		dict,
-		lambda x: json.dumps(x, ensure_ascii=False).encode(
-			locale.getpreferredencoding(), 'replace'))
-	sqlite3.register_converter(
-		"JSON",
-		lambda x: json.loads(x.decode(
-			locale.getpreferredencoding(), 'replace')))
+	def _json_dumps_safe(x):
+		try:
+			x = json.dumps(x, ensure_ascii=False).encode(
+				locale.getpreferredencoding(), 'replace')
+		except Exception, e: # pragma: no cover
+			logSys.error('json dumps failed: %s', e)
+			x = '{}'
+		return x
 else:
-	sqlite3.register_adapter(dict, json.dumps)
-	sqlite3.register_converter("JSON", json.loads)
+	def _json_dumps_safe(x):
+		try:
+			x = json.dumps(x, ensure_ascii=False).decode(
+				locale.getpreferredencoding(), 'replace')
+		except Exception, e: # pragma: no cover
+			logSys.error('json dumps failed: %s', e)
+			x = '{}'
+		return x
+
+def _json_loads_safe(x):
+	try:
+		x = json.loads(x.decode(
+			locale.getpreferredencoding(), 'replace'))
+	except Exception, e: # pragma: no cover
+		logSys.error('json loads failed: %s', e)
+		x = {}
+	return x
+
+sqlite3.register_adapter(dict, _json_dumps_safe)
+sqlite3.register_converter("JSON", _json_loads_safe)
 
 def commitandrollback(f):
 	@wraps(f)

fail2ban/server/filter.py

@@ -802,8 +802,8 @@ class FileContainer:
 				" encoding) for this jail. Continuing"
 				" to process line ignoring invalid characters: %r" %
 				(self.getFileName(), self.getEncoding(), line))
-			if sys.version_info >= (3,): # In python3, must be decoded
-				line = line.decode(self.getEncoding(), 'ignore')
+			# decode with replacing error chars:
+			line = line.decode(self.getEncoding(), 'replace')
 		return line
 
 	def close(self):

fail2ban/tests/databasetestcase.py

@@ -177,6 +177,17 @@ class DatabaseTest(LogCaptureTestCase):
 		self.assertTrue(
 			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
 
+	def testAddBanInvalidEncoded(self):
+		if Fail2BanDb is None: # pragma: no cover
+			return
+		self.testAddJail()
+		ticket = FailTicket("127.0.0.1", 0, {'m': ['... user "\xd1\xe2\xe5\xf2\xe0"  ...'], 'a': 1})
+		self.db.addBan(self.jail, ticket)
+
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 1)
+		self.assertTrue(
+			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
+
 	def testDelBan(self):
 		self.testAddBan()
 		ticket = self.db.getBans(jail=self.jail)[0]