From 227550684ae75cee1f7f8aa3be8482a19052e671 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 29 Jun 2018 22:02:41 +0200 Subject: [PATCH] **interim** try to fix several conversion errors --- fail2ban/helpers.py | 24 +++++++++++++++++++++++- fail2ban/server/database.py | 4 ++-- fail2ban/tests/fail2banclienttestcase.py | 4 ++-- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f7b0f1eb..3c4e6ee6 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -36,9 +36,31 @@ from .server.mytime import MyTime PREFER_ENC = locale.getpreferredencoding() # correct preferred encoding if lang not set in environment: if PREFER_ENC.startswith('ANSI_'): # pragma: no cover - if all((os.getenv(v) in (None, "") for v in ('LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'LANG'))): + if sys.stdout and not sys.stdout.encoding.startswith('ANSI_'): + PREFER_ENC = sys.stdout.encoding + elif all((os.getenv(v) in (None, "") for v in ('LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'LANG'))): PREFER_ENC = 'UTF-8'; +# py-2.x: try to minimize influence of sporadic conversion errors on python 2.x, +# caused by implicit converting of string/unicode (e. g. `str(u"\uFFFD")` produces an error +# if default encoding is 'ascii'); +if sys.version_info < (3,): # python >= 2.6 + # correct default (global system) encoding (mostly UTF-8): + def __resetDefaultEncoding(encoding): + global PREFER_ENC + ode = sys.getdefaultencoding().upper() + if ode == 'ASCII' or ode != PREFER_ENC.upper(): + # setdefaultencoding is normally deleted after site initialized, so hack-in using load of sys-module: + from imp import load_dynamic as __ldm + _sys = __ldm('_sys', 'sys') + _sys.setdefaultencoding(encoding) + # override to PREFER_ENC: + __resetDefaultEncoding(PREFER_ENC) + del __resetDefaultEncoding + +# todo: rewrite explicit (and implicit) str-conversions via encode/decode with IO-encoding (sys.stdout.encoding), +# e. g. inside tags-replacement by command-actions, etc. + def formatExceptionInfo(): """ Consistently format exception information """ diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index a06db21c..d9866b97 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -83,8 +83,8 @@ else: def _json_loads_safe(x): try: - x = _normalize(json.loads(x.decode( - PREFER_ENC, 'replace'))) + x = json.loads(x.decode( + PREFER_ENC, 'replace')) except Exception as e: # pragma: no cover logSys.error('json loads failed: %s', e) x = {} diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index cde8f4f5..68bdad80 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1223,8 +1223,8 @@ class Fail2banServerTest(Fail2banClientServerBase): _write_file(lgfn, "w+", str(int(MyTime.time())) + ' failure "125-000-001" - 192.0.2.1', str(int(MyTime.time())) + ' failure "125-000-002" - 192.0.2.1', - str(int(MyTime.time())) + ' failure "125-000-003" - 192.0.2.1', - str(int(MyTime.time())) + ' failure "125-000-004" - 192.0.2.1', + str(int(MyTime.time())) + ' failure "125-000-003" - 192.0.2.1 (\xf2\xf0\xe5\xf2\xe8\xe9)', + str(int(MyTime.time())) + ' failure "125-000-004" - 192.0.2.1 (\xf2\xf0\xe5\xf2\xe8\xe9)', str(int(MyTime.time())) + ' failure "125-000-005" - 192.0.2.1', ) # check all sessions are banned (and blacklisted in map-file):