From 66367876bbc780688c5755e3654bb5b13ea58390 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 27 Feb 2013 18:09:55 +0000 Subject: [PATCH] Add ability to set log encoding for jail --- client/beautifier.py | 3 +++ client/jailreader.py | 3 +++ common/protocol.py | 2 ++ config/jail.conf | 7 ++++++ server/filter.py | 46 +++++++++++++++++++++++++++++++------ server/server.py | 6 +++++ server/transmitter.py | 6 +++++ testcases/servertestcase.py | 9 +++++++- 8 files changed, 74 insertions(+), 8 deletions(-) diff --git a/client/beautifier.py b/client/beautifier.py index 7e48016c..1653ff63 100644 --- a/client/beautifier.py +++ b/client/beautifier.py @@ -110,6 +110,9 @@ class Beautifier: for path in response[:-1]: msg = msg + "|- " + path + "\n" msg = msg + "`- " + response[len(response)-1] + elif inC[2] == "logencoding": + msg = "Current log encoding is set to:\n" + msg = msg + response elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"): if len(response) == 0: msg = "No IP address/network is ignored" diff --git a/client/jailreader.py b/client/jailreader.py index f66dc010..be22a78f 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -61,6 +61,7 @@ class JailReader(ConfigReader): def getOptions(self): opts = [["bool", "enabled", "false"], ["string", "logpath", "/var/log/messages"], + ["string", "logencoding", "auto"], ["string", "backend", "auto"], ["int", "maxretry", 3], ["int", "findtime", 600], @@ -110,6 +111,8 @@ class JailReader(ConfigReader): logSys.error("No file found for " + path) for p in pathList: stream.append(["set", self.__name, "addlogpath", p]) + elif opt == "logencoding": + stream.append(["set", self.__name, "logencoding", self.__opts[opt]]) elif opt == "backend": backend = self.__opts[opt] elif opt == "maxretry": diff --git a/common/protocol.py b/common/protocol.py index 99a2fe09..99d608a8 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -56,6 +56,7 @@ protocol = [ ["set delignoreip ", "removes from the ignore list of "], ["set addlogpath ", "adds to the monitoring list of "], ["set dellogpath ", "removes from the monitoring list of "], +["set logencoding ", "sets the of the log files for "], ["set addfailregex ", "adds the regular expression which must match failures for "], ["set delfailregex ", "removes the regular expression at for failregex"], ["set addignoreregex ", "adds the regular expression which should match pattern to exclude for "], @@ -77,6 +78,7 @@ protocol = [ ["set actionunban ", "sets the unban command of the action for "], ['', "JAIL INFORMATION", ""], ["get logpath", "gets the list of the monitored files for "], +["get logencoding ", "gets the of the log files for "], ["get ignoreip", "gets the list of ignored IP addresses for "], ["get failregex", "gets the list of regular expressions which matches the failures for "], ["get ignoreregex", "gets the list of regular expressions which matches patterns to ignore for "], diff --git a/config/jail.conf b/config/jail.conf index a0093f68..e56023d7 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -55,6 +55,13 @@ backend = auto # but it will be logged as info. usedns = warn +# "logencoding" specifies the encoding of the log files handled by the jail +# This is used to decode the lines from the log file. +# Typical examples: "ascii", "utf-8" +# +# auto: will use the system locale setting +logencoding = auto + # This jail corresponds to the standard configuration in Fail2ban 0.6. # The mail-whois action send a notification e-mail with a whois request diff --git a/server/filter.py b/server/filter.py index 3f507cd1..57305431 100644 --- a/server/filter.py +++ b/server/filter.py @@ -35,7 +35,7 @@ from datedetector import DateDetector from mytime import MyTime from failregex import FailRegex, Regex, RegexException -import logging, re, os, fcntl, time, sys +import logging, re, os, fcntl, time, sys, locale, codecs # Gets the instance of the logger. logSys = logging.getLogger("fail2ban.filter") @@ -392,6 +392,7 @@ class FileFilter(Filter): Filter.__init__(self, jail, **kwargs) ## The log file path. self.__logPath = [] + self.setLogEncoding("auto") ## # Add a log file path @@ -402,7 +403,7 @@ class FileFilter(Filter): if self.containsLogPath(path): logSys.error(path + " already exists") else: - container = FileContainer(path, tail) + container = FileContainer(path, self.getLogEncoding(), tail) self.__logPath.append(container) logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -451,6 +452,28 @@ class FileFilter(Filter): return True return False + ## + # Set the log file encoding + # + # @param encoding the encoding used with log files + + def setLogEncoding(self, encoding): + if encoding.lower() == "auto": + encoding = locale.getpreferredencoding() + codecs.lookup(encoding) # Raise LookupError if invalid codec + for log in self.getLogPath(): + log.setEncoding(encoding) + self.__encoding = encoding + logSys.info("Set jail log file encoding to %s" % encoding) + + ## + # Get the log file encoding + # + # @return log encoding value + + def getLogEncoding(self): + return self.__encoding + def getFileContainer(self, path): for log in self.__logPath: if log.getFileName() == path: @@ -510,8 +533,9 @@ except ImportError: class FileContainer: - def __init__(self, filename, tail = False): + def __init__(self, filename, encoding, tail = False): self.__filename = filename + self.setEncoding(encoding) self.__tail = tail self.__handler = None # Try to open the file. Raises an exception if an error occured. @@ -534,6 +558,13 @@ class FileContainer: def getFileName(self): return self.__filename + def setEncoding(self, encoding): + codecs.lookup(encoding) # Raises LookupError if invalid + self.__encoding = encoding + + def getEncoding(self): + return self.__encoding + def open(self): self.__handler = open(self.__filename, 'rb') # Set the file descriptor to be FD_CLOEXEC @@ -557,11 +588,12 @@ class FileContainer: return "" line = self.__handler.readline() try: - line = line.decode('utf-8', 'strict') + line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warn("Error decoding line to utf-8: %s" % `line`) - if sys.version_info >= (3,): # In python3, must be unicode - line = line.decode('utf-8', 'ignore') + logSys.warn("Error decoding line from '%s' with '%s': %s" % + (self.getFileName(), self.getEncoding(), `line`)) + if sys.version_info >= (3,): # In python3, must be decoded + line = line.decode(self.getEncoding(), 'ignore') return line def close(self): diff --git a/server/server.py b/server/server.py index 4a086078..0db03d09 100644 --- a/server/server.py +++ b/server/server.py @@ -181,6 +181,12 @@ class Server: return [m.getFileName() for m in self.__jails.getFilter(name).getLogPath()] + def setLogEncoding(self, name, encoding): + return self.__jails.getFilter(name).setLogEncoding(encoding) + + def getLogEncoding(self, name): + return self.__jails.getFilter(name).getLogEncoding() + def setFindTime(self, name, value): self.__jails.getFilter(name).setFindTime(value) diff --git a/server/transmitter.py b/server/transmitter.py index a02b94a2..1c1b1e2c 100644 --- a/server/transmitter.py +++ b/server/transmitter.py @@ -139,6 +139,10 @@ class Transmitter: value = command[2] self.__server.delLogPath(name, value) return self.__server.getLogPath(name) + elif command[1] == "logencoding": + value = command[2] + self.__server.setLogEncoding(name, value) + return self.__server.getLogEncoding(name) elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value) @@ -234,6 +238,8 @@ class Transmitter: # Filter elif command[1] == "logpath": return self.__server.getLogPath(name) + elif command[1] == "logencoding": + return self.__server.getLogEncoding(name) elif command[1] == "ignoreip": return self.__server.getIgnoreIP(name) elif command[1] == "failregex": diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py index 00f56b81..f52fecca 100644 --- a/testcases/servertestcase.py +++ b/testcases/servertestcase.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest, socket, time, tempfile, os +import unittest, socket, time, tempfile, os, locale from server.server import Server class StartStop(unittest.TestCase): @@ -258,6 +258,13 @@ class Transmitter(unittest.TestCase): self.setGetTest("maxretry", "-2", -2, jail=self.jailName) self.setGetTestNOK("maxretry", "Duck", jail=self.jailName) + def testJailLogEncoding(self): + self.setGetTest("logencoding", "UTF-8", jail=self.jailName) + self.setGetTest("logencoding", "ascii", jail=self.jailName) + self.setGetTest("logencoding", "auto", locale.getpreferredencoding(), + jail=self.jailName) + self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName) + def testJailLogPath(self): self.jailAddDelTest( "logpath",