diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index f36ca7b0..ea2f3a7f 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -29,7 +29,6 @@ __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halch __license__ = "GPL" import getopt -import locale import logging import os import shlex @@ -52,7 +51,7 @@ from .filterreader import FilterReader from ..server.filter import Filter, FileContainer from ..server.failregex import RegexException -from ..helpers import FormatterWithTraceBack, getLogger +from ..helpers import FormatterWithTraceBack, getLogger, PREFER_ENC # Gets the instance of the logger. logSys = getLogger("fail2ban") @@ -239,7 +238,7 @@ class Fail2banRegex(object): if opts.encoding: self.encoding = opts.encoding else: - self.encoding = locale.getpreferredencoding() + self.encoding = PREFER_ENC self.raw = True if opts.raw else False def decode_line(self, line): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 7660e64a..2738f5e6 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -21,6 +21,7 @@ __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko" __license__ = "GPL" import gc +import locale import logging import os import re @@ -32,6 +33,9 @@ from threading import Lock from .server.mytime import MyTime +PREFER_ENC = locale.getpreferredencoding() + + def formatExceptionInfo(): """ Consistently format exception information """ cla, exc = sys.exc_info()[:2] @@ -144,6 +148,36 @@ def splitwords(s): return filter(bool, map(str.strip, re.split('[ ,\n]+', s))) +# +# Following "uni_decode" function unified python independent any to string converting +# +# Typical example resp. work-case for understanding the coding/decoding issues: +# +# [isinstance('', str), isinstance(b'', str), isinstance(u'', str)] +# [True, True, False]; # -- python2 +# [True, False, True]; # -- python3 +# +if sys.version_info >= (3,): + def uni_decode(x, enc=PREFER_ENC, errors='strict'): + try: + if isinstance(x, bytes): + return x.decode(enc, errors) + return x + except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable + if errors != 'strict': + raise + return uni_decode(x, enc, 'replace') +else: + def uni_decode(x, enc=PREFER_ENC, errors='strict'): + try: + if isinstance(x, unicode): + return x.encode(enc, errors) + return x + except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable + if errors != 'strict': + raise + return uni_decode(x, enc, 'replace') + class BgService(object): """Background servicing diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 639331bf..120619f9 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -22,7 +22,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import json -import locale import shutil import sqlite3 import sys @@ -32,7 +31,7 @@ from threading import RLock from .mytime import MyTime from .ticket import FailTicket -from ..helpers import getLogger +from ..helpers import getLogger, PREFER_ENC # Gets the instance of the logger. logSys = getLogger(__name__) @@ -41,7 +40,7 @@ if sys.version_info >= (3,): def _json_dumps_safe(x): try: x = json.dumps(x, ensure_ascii=False).encode( - locale.getpreferredencoding(), 'replace') + PREFER_ENC, 'replace') except Exception as e: # pragma: no cover logSys.error('json dumps failed: %s', e) x = '{}' @@ -50,7 +49,7 @@ if sys.version_info >= (3,): def _json_loads_safe(x): try: x = json.loads(x.decode( - locale.getpreferredencoding(), 'replace')) + PREFER_ENC, 'replace')) except Exception as e: # pragma: no cover logSys.error('json loads failed: %s', e) x = {} @@ -62,14 +61,14 @@ else: elif isinstance(x, list): return [_normalize(element) for element in x] elif isinstance(x, unicode): - return x.encode(locale.getpreferredencoding()) + return x.encode(PREFER_ENC) else: return x def _json_dumps_safe(x): try: x = json.dumps(_normalize(x), ensure_ascii=False).decode( - locale.getpreferredencoding(), 'replace') + PREFER_ENC, 'replace') except Exception as e: # pragma: no cover logSys.error('json dumps failed: %s', e) x = '{}' @@ -78,7 +77,7 @@ else: def _json_loads_safe(x): try: x = _normalize(json.loads(x.decode( - locale.getpreferredencoding(), 'replace'))) + PREFER_ENC, 'replace'))) except Exception as e: # pragma: no cover logSys.error('json loads failed: %s', e) x = {} diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ab6e7b15..08f3e77d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -24,7 +24,6 @@ __license__ = "GPL" import codecs import datetime import fcntl -import locale import logging import os import re @@ -40,7 +39,7 @@ from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction -from ..helpers import getLogger +from ..helpers import getLogger, PREFER_ENC # Gets the instance of the logger. logSys = getLogger(__name__) @@ -87,7 +86,7 @@ class Filter(JailThread): ## External command self.__ignoreCommand = False ## Default or preferred encoding (to decode bytes from file or journal): - self.__encoding = locale.getpreferredencoding() + self.__encoding = PREFER_ENC ## Error counter (protected, so can be used in filter implementations) ## if it reached 100 (at once), run-cycle will go idle self._errors = 0 @@ -329,7 +328,7 @@ class Filter(JailThread): def setLogEncoding(self, encoding): if encoding.lower() == "auto": - encoding = locale.getpreferredencoding() + encoding = PREFER_ENC codecs.lookup(encoding) # Raise LookupError if invalid codec self.__encoding = encoding logSys.info(" encoding: %s" % encoding) @@ -452,29 +451,6 @@ class Filter(JailThread): return False - if sys.version_info >= (3,): - @staticmethod - def uni_decode(x, enc, errors='strict'): - try: - if isinstance(x, bytes): - return x.decode(enc, errors) - return x - except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable - if errors != 'strict': - raise - return uni_decode(x, enc, 'replace') - else: - @staticmethod - def uni_decode(x, enc, errors='strict'): - try: - if isinstance(x, unicode): - return x.encode(enc, errors) - return x - except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable - if errors != 'strict': - raise - return uni_decode(x, enc, 'replace') - def processLine(self, line, date=None, returnRawHost=False, checkAllRegex=False, checkFindTime=False): """Split the time portion from log msg and return findFailures on them diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 0b6c4074..f9f9395c 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -34,7 +34,7 @@ from .failmanager import FailManagerEmpty from .filter import JournalFilter, Filter from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger, logging, splitwords +from ..helpers import getLogger, logging, splitwords, uni_decode # Gets the instance of the logger. logSys = getLogger(__name__) @@ -174,10 +174,6 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalMatch(self): return self.__matches - def uni_decode(self, x): - v = Filter.uni_decode(x, self.getLogEncoding()) - return v - ## # Format journal log entry into syslog style # @@ -186,16 +182,16 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def formatJournalEntry(self, logentry): # Be sure, all argument of line tuple should have the same type: - uni_decode = self.uni_decode + enc = self.getLogEncoding() logelements = [] v = logentry.get('_HOSTNAME') if v: - logelements.append(uni_decode(v)) + logelements.append(uni_decode(v, enc)) v = logentry.get('SYSLOG_IDENTIFIER') if not v: v = logentry.get('_COMM') if v: - logelements.append(uni_decode(v)) + logelements.append(uni_decode(v, enc)) v = logentry.get('SYSLOG_PID') if not v: v = logentry.get('_PID') @@ -210,9 +206,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements.append("[%12.6f]" % monotonic.total_seconds()) msg = logentry.get('MESSAGE','') if isinstance(msg, list): - logelements.append(" ".join(uni_decode(v) for v in msg)) + logelements.append(" ".join(uni_decode(v, enc) for v in msg)) else: - logelements.append(uni_decode(msg)) + logelements.append(uni_decode(msg, enc)) logline = " ".join(logelements) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 738394c3..a8a00257 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -21,8 +21,14 @@ __author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester" __license__ = "GPL" -import logging, os, fcntl, subprocess, time, signal -from ..helpers import getLogger +import fcntl +import logging +import os +import signal +import subprocess +import sys +import time +from ..helpers import getLogger, uni_decode # Gets the instance of the logger. logSys = getLogger(__name__) @@ -179,7 +185,7 @@ class Utils(): if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): logSys.log(std_level, "%s -- stdout:", realCmd) for l in stdout.splitlines(): - logSys.log(std_level, " -- stdout: %r", l) + logSys.log(std_level, " -- stdout: %r", uni_decode(l)) popen.stdout.close() if popen.stderr: try: @@ -191,7 +197,7 @@ class Utils(): if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): logSys.log(std_level, "%s -- stderr:", realCmd) for l in stderr.splitlines(): - logSys.log(std_level, " -- stderr: %r", l) + logSys.log(std_level, " -- stderr: %r", uni_decode(l)) popen.stderr.close() success = False diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 166450bb..cacb5e17 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -399,11 +399,11 @@ class CommandActionTest(LogCaptureTestCase): def testCaptureStdOutErr(self): CommandAction.executeCmd('echo "How now brown cow"') - self.assertLogged("stdout: 'How now brown cow'\n", "stdout: b'How now brown cow'\n") + self.assertLogged("stdout: 'How now brown cow'\n") CommandAction.executeCmd( 'echo "The rain in Spain stays mainly in the plain" 1>&2') self.assertLogged( - "stderr: 'The rain in Spain stays mainly in the plain'\n", "stderr: b'The rain in Spain stays mainly in the plain'\n") + "stderr: 'The rain in Spain stays mainly in the plain'\n") def testCallingMap(self): mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'), diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index f7394a8e..15495655 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -38,11 +38,11 @@ except ImportError: from ..server.jail import Jail from ..server.filterpoll import FilterPoll -from ..server.filter import Filter, FileFilter, FileContainer, locale +from ..server.filter import Filter, FileFilter, FileContainer from ..server.failmanager import FailManagerEmpty from ..server.ipdns import DNSUtils, IPAddr from ..server.mytime import MyTime -from ..server.utils import Utils +from ..server.utils import Utils, uni_decode from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase from .dummyjail import DummyJail @@ -311,8 +311,7 @@ class BasicFilter(unittest.TestCase): b'Fail for "g\xc3\xb6ran" from 192.0.2.1' ): # join should work if all arguments have the same type: - enc = locale.getpreferredencoding() - "".join([Filter.uni_decode(v, enc) for v in (a1, a2, a3)]) + "".join([uni_decode(v) for v in (a1, a2, a3)]) class IgnoreIP(LogCaptureTestCase): diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 5b6b93a6..8fb393ea 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -35,7 +35,7 @@ from StringIO import StringIO from utils import LogCaptureTestCase, logSys as DefLogSys -from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger +from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, uni_decode from ..helpers import splitwords from ..server.datedetector import DateDetector from ..server.datetemplate import DatePatternRegex @@ -74,16 +74,14 @@ class HelpersTest(unittest.TestCase): if sys.version_info >= (2,7): def _sh_call(cmd): - import subprocess, locale + import subprocess ret = subprocess.check_output(cmd, shell=True) - if sys.version_info >= (3,): - ret = ret.decode(locale.getpreferredencoding(), 'replace') - return str(ret).rstrip() + return uni_decode(ret).rstrip() else: def _sh_call(cmd): import subprocess ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read() - return str(ret).rstrip() + return uni_decode(ret).rstrip() def _getSysPythonVersion(): return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'") diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 3ca8049b..d044da4b 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -28,7 +28,6 @@ import unittest import time import tempfile import os -import locale import sys import platform @@ -40,7 +39,7 @@ from ..server.jail import Jail from ..server.jailthread import JailThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..helpers import getLogger +from ..helpers import getLogger, PREFER_ENC from .. import version try: @@ -354,7 +353,7 @@ class Transmitter(TransmitterBase): def testJailLogEncoding(self): self.setGetTest("logencoding", "UTF-8", jail=self.jailName) self.setGetTest("logencoding", "ascii", jail=self.jailName) - self.setGetTest("logencoding", "auto", locale.getpreferredencoding(), + self.setGetTest("logencoding", "auto", PREFER_ENC, jail=self.jailName) self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)