normalize usage of preferred encoding (and decode any to string);

python 3.x compatibility (used uni_decode for string representation of stdout/stderr, unified test cases)
amend for #1542

fail2ban/client/fail2banregex.py

@@ -29,7 +29,6 @@ __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halch
 __license__ = "GPL"
 
 import getopt
-import locale
 import logging
 import os
 import shlex
@@ -52,7 +51,7 @@ from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import RegexException
 
-from ..helpers import FormatterWithTraceBack, getLogger
+from ..helpers import FormatterWithTraceBack, getLogger, PREFER_ENC
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
@@ -239,7 +238,7 @@ class Fail2banRegex(object):
 		if opts.encoding:
 			self.encoding = opts.encoding
 		else:
-			self.encoding = locale.getpreferredencoding()
+			self.encoding = PREFER_ENC
 		self.raw = True if opts.raw else False
 
 	def decode_line(self, line):

fail2ban/helpers.py

@@ -21,6 +21,7 @@ __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko"
 __license__ = "GPL"
 
 import gc
+import locale
 import logging
 import os
 import re
@@ -32,6 +33,9 @@ from threading import Lock
 from .server.mytime import MyTime
 
 
+PREFER_ENC = locale.getpreferredencoding()
+
+
 def formatExceptionInfo():
 	""" Consistently format exception information """
 	cla, exc = sys.exc_info()[:2]
@@ -144,6 +148,36 @@ def splitwords(s):
 	return filter(bool, map(str.strip, re.split('[ ,\n]+', s)))
 
 
+#
+# Following "uni_decode" function unified python independent any to string converting
+#
+# Typical example resp. work-case for understanding the coding/decoding issues:
+#
+#   [isinstance('', str), isinstance(b'', str), isinstance(u'', str)]
+#   [True, True, False]; # -- python2
+#	  [True, False, True]; # -- python3
+#
+if sys.version_info >= (3,):
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, bytes):
+				return x.decode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict': 
+				raise
+			return uni_decode(x, enc, 'replace')
+else:
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, unicode):
+				return x.encode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict':
+				raise
+			return uni_decode(x, enc, 'replace')
+
 class BgService(object):
 	"""Background servicing
 

fail2ban/server/database.py

@@ -22,7 +22,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
 __license__ = "GPL"
 
 import json
-import locale
 import shutil
 import sqlite3
 import sys
@@ -32,7 +31,7 @@ from threading import RLock
 
 from .mytime import MyTime
 from .ticket import FailTicket
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -41,7 +40,7 @@ if sys.version_info >= (3,):
 	def _json_dumps_safe(x):
 		try:
 			x = json.dumps(x, ensure_ascii=False).encode(
-				locale.getpreferredencoding(), 'replace')
+				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
@@ -50,7 +49,7 @@ if sys.version_info >= (3,):
 	def _json_loads_safe(x):
 		try:
 			x = json.loads(x.decode(
-				locale.getpreferredencoding(), 'replace'))
+				PREFER_ENC, 'replace'))
 		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}
@@ -62,14 +61,14 @@ else:
 		elif isinstance(x, list):
 			return [_normalize(element) for element in x]
 		elif isinstance(x, unicode):
-			return x.encode(locale.getpreferredencoding())
+			return x.encode(PREFER_ENC)
 		else:
 			return x
 
 	def _json_dumps_safe(x):
 		try:
 			x = json.dumps(_normalize(x), ensure_ascii=False).decode(
-				locale.getpreferredencoding(), 'replace')
+				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
 			x = '{}'
@@ -78,7 +77,7 @@ else:
 	def _json_loads_safe(x):
 		try:
 			x = _normalize(json.loads(x.decode(
-				locale.getpreferredencoding(), 'replace')))
+				PREFER_ENC, 'replace')))
 		except Exception as e: # pragma: no cover
 			logSys.error('json loads failed: %s', e)
 			x = {}

fail2ban/server/filter.py

@@ -24,7 +24,6 @@ __license__ = "GPL"
 import codecs
 import datetime
 import fcntl
-import locale
 import logging
 import os
 import re
@@ -40,7 +39,7 @@ from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n
 from .mytime import MyTime
 from .failregex import FailRegex, Regex, RegexException
 from .action import CommandAction
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -87,7 +86,7 @@ class Filter(JailThread):
 		## External command
 		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
-		self.__encoding = locale.getpreferredencoding()
+		self.__encoding = PREFER_ENC
 		## Error counter (protected, so can be used in filter implementations)
 		## if it reached 100 (at once), run-cycle will go idle
 		self._errors = 0
@@ -329,7 +328,7 @@ class Filter(JailThread):
 
 	def setLogEncoding(self, encoding):
 		if encoding.lower() == "auto":
-			encoding = locale.getpreferredencoding()
+			encoding = PREFER_ENC
 		codecs.lookup(encoding) # Raise LookupError if invalid codec
 		self.__encoding = encoding
 		logSys.info("  encoding: %s" % encoding)
@@ -452,29 +451,6 @@ class Filter(JailThread):
 
 		return False
 
-	if sys.version_info >= (3,):
-		@staticmethod
-		def uni_decode(x, enc, errors='strict'):
-			try:
-				if isinstance(x, bytes):
-					return x.decode(enc, errors)
-				return x
-			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-				if errors != 'strict': 
-					raise
-				return uni_decode(x, enc, 'replace')
-	else:
-		@staticmethod
-		def uni_decode(x, enc, errors='strict'):
-			try:
-				if isinstance(x, unicode):
-					return x.encode(enc, errors)
-				return x
-			except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-				if errors != 'strict':
-					raise
-				return uni_decode(x, enc, 'replace')
-
 	def processLine(self, line, date=None, returnRawHost=False,
 		checkAllRegex=False, checkFindTime=False):
 		"""Split the time portion from log msg and return findFailures on them

fail2ban/server/filtersystemd.py

@@ -34,7 +34,7 @@ from .failmanager import FailManagerEmpty
 from .filter import JournalFilter, Filter
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, logging, splitwords
+from ..helpers import getLogger, logging, splitwords, uni_decode
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -174,10 +174,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def getJournalMatch(self):
 		return self.__matches
 
-	def uni_decode(self, x):
-		v = Filter.uni_decode(x, self.getLogEncoding())
-		return v
-
 	##
 	# Format journal log entry into syslog style
 	#
@@ -186,16 +182,16 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 	def formatJournalEntry(self, logentry):
 		# Be sure, all argument of line tuple should have the same type:
-		uni_decode = self.uni_decode
+		enc = self.getLogEncoding()
 		logelements = []
 		v = logentry.get('_HOSTNAME')
 		if v:
-			logelements.append(uni_decode(v))
+			logelements.append(uni_decode(v, enc))
 		v = logentry.get('SYSLOG_IDENTIFIER')
 		if not v:
 			v = logentry.get('_COMM')
 		if v:
-			logelements.append(uni_decode(v))
+			logelements.append(uni_decode(v, enc))
 			v = logentry.get('SYSLOG_PID')
 			if not v:
 				v = logentry.get('_PID')
@@ -210,9 +206,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				logelements.append("[%12.6f]" % monotonic.total_seconds())
 		msg = logentry.get('MESSAGE','')
 		if isinstance(msg, list):
-			logelements.append(" ".join(uni_decode(v) for v in msg))
+			logelements.append(" ".join(uni_decode(v, enc) for v in msg))
 		else:
-			logelements.append(uni_decode(msg))
+			logelements.append(uni_decode(msg, enc))
 
 		logline = " ".join(logelements)
 

fail2ban/server/utils.py

@@ -21,8 +21,14 @@ __author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester"
 __license__ = "GPL"
 
-import logging, os, fcntl, subprocess, time, signal
+import fcntl
-from ..helpers import getLogger
+import logging
+import os
+import signal
+import subprocess
+import sys
+import time
+from ..helpers import getLogger, uni_decode
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -179,7 +185,7 @@ class Utils():
 				if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
 					logSys.log(std_level, "%s -- stdout:", realCmd)
 					for l in stdout.splitlines():
-						logSys.log(std_level, " -- stdout: %r", l)
+						logSys.log(std_level, " -- stdout: %r", uni_decode(l))
 				popen.stdout.close()
 			if popen.stderr:
 				try:
@@ -191,7 +197,7 @@ class Utils():
 				if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
 					logSys.log(std_level, "%s -- stderr:", realCmd)
 					for l in stderr.splitlines():
-						logSys.log(std_level, " -- stderr: %r", l)
+						logSys.log(std_level, " -- stderr: %r", uni_decode(l))
 				popen.stderr.close()
 
 		success = False

fail2ban/tests/actiontestcase.py

@@ -399,11 +399,11 @@ class CommandActionTest(LogCaptureTestCase):
 
 	def testCaptureStdOutErr(self):
 		CommandAction.executeCmd('echo "How now brown cow"')
-		self.assertLogged("stdout: 'How now brown cow'\n", "stdout: b'How now brown cow'\n")
+		self.assertLogged("stdout: 'How now brown cow'\n")
 		CommandAction.executeCmd(
 			'echo "The rain in Spain stays mainly in the plain" 1>&2')
 		self.assertLogged(
-			"stderr: 'The rain in Spain stays mainly in the plain'\n", "stderr: b'The rain in Spain stays mainly in the plain'\n")
+			"stderr: 'The rain in Spain stays mainly in the plain'\n")
 
 	def testCallingMap(self):
 		mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'),

fail2ban/tests/filtertestcase.py

@@ -38,11 +38,11 @@ except ImportError:
 
 from ..server.jail import Jail
 from ..server.filterpoll import FilterPoll
-from ..server.filter import Filter, FileFilter, FileContainer, locale
+from ..server.filter import Filter, FileFilter, FileContainer
 from ..server.failmanager import FailManagerEmpty
 from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
-from ..server.utils import Utils
+from ..server.utils import Utils, uni_decode
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase
 from .dummyjail import DummyJail
 
@@ -311,8 +311,7 @@ class BasicFilter(unittest.TestCase):
 					b'Fail for "g\xc3\xb6ran" from 192.0.2.1'
 				):
 					# join should work if all arguments have the same type:
-					enc = locale.getpreferredencoding()
+					"".join([uni_decode(v) for v in (a1, a2, a3)])
-					"".join([Filter.uni_decode(v, enc) for v in (a1, a2, a3)])
 
 
 class IgnoreIP(LogCaptureTestCase):

fail2ban/tests/misctestcase.py

@@ -35,7 +35,7 @@ from StringIO import StringIO
 
 from utils import LogCaptureTestCase, logSys as DefLogSys
 
-from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger
+from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, uni_decode
 from ..helpers import splitwords
 from ..server.datedetector import DateDetector
 from ..server.datetemplate import DatePatternRegex
@@ -74,16 +74,14 @@ class HelpersTest(unittest.TestCase):
 
 if sys.version_info >= (2,7):
 	def _sh_call(cmd):
-		import subprocess, locale
+		import subprocess
 		ret = subprocess.check_output(cmd, shell=True)
-		if sys.version_info >= (3,):
+		return uni_decode(ret).rstrip()
-			ret = ret.decode(locale.getpreferredencoding(), 'replace')
-		return str(ret).rstrip()
 else:
 	def _sh_call(cmd):
 		import subprocess
 		ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read()
-		return str(ret).rstrip()
+		return uni_decode(ret).rstrip()
 
 def _getSysPythonVersion():
 	return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'")

fail2ban/tests/servertestcase.py

@@ -28,7 +28,6 @@ import unittest
 import time
 import tempfile
 import os
-import locale
 import sys
 import platform
 
@@ -40,7 +39,7 @@ from ..server.jail import Jail
 from ..server.jailthread import JailThread
 from ..server.utils import Utils
 from .utils import LogCaptureTestCase
-from ..helpers import getLogger
+from ..helpers import getLogger, PREFER_ENC
 from .. import version
 
 try:
@@ -354,7 +353,7 @@ class Transmitter(TransmitterBase):
 	def testJailLogEncoding(self):
 		self.setGetTest("logencoding", "UTF-8", jail=self.jailName)
 		self.setGetTest("logencoding", "ascii", jail=self.jailName)
-		self.setGetTest("logencoding", "auto", locale.getpreferredencoding(),
+		self.setGetTest("logencoding", "auto", PREFER_ENC,
 			jail=self.jailName)
 		self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)