Add ability to set log encoding for jail

client/beautifier.py

@@ -110,6 +110,9 @@ class Beautifier:
 					for path in response[:-1]:
 						msg = msg + "|- " + path + "\n"
 					msg = msg + "`- " + response[len(response)-1]
+			elif inC[2] == "logencoding":
+				msg = "Current log encoding is set to:\n"
+				msg = msg + response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"

client/jailreader.py

@@ -61,6 +61,7 @@ class JailReader(ConfigReader):
 	def getOptions(self):
 		opts = [["bool", "enabled", "false"],
 				["string", "logpath", "/var/log/messages"],
+				["string", "logencoding", "auto"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
 				["int", "findtime", 600],
@@ -110,6 +111,8 @@ class JailReader(ConfigReader):
 						logSys.error("No file found for " + path)
 					for p in pathList:
 						stream.append(["set", self.__name, "addlogpath", p])
+			elif opt == "logencoding":
+				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
 			elif opt == "backend":
 				backend = self.__opts[opt]
 			elif opt == "maxretry":

common/protocol.py

@@ -56,6 +56,7 @@ protocol = [
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
 ["set <JAIL> addlogpath <FILE>", "adds <FILE> to the monitoring list of <JAIL>"], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
+["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
@@ -77,6 +78,7 @@ protocol = [
 ["set <JAIL> actionunban <ACT> <CMD>", "sets the unban command <CMD> of the action <ACT> for <JAIL>"], 
 ['', "JAIL INFORMATION", ""],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
+["get <JAIL> logencoding <ENCODING>", "gets the <ENCODING> of the log files for <JAIL>"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],

config/jail.conf

@@ -55,6 +55,13 @@ backend = auto
 #        but it will be logged as info.
 usedns = warn
 
+# "logencoding" specifies the encoding of the log files handled by the jail
+#   This is used to decode the lines from the log file.
+#   Typical examples:  "ascii", "utf-8"
+#
+#   auto:   will use the system locale setting
+logencoding = auto
+
 
 # This jail corresponds to the standard configuration in Fail2ban 0.6.
 # The mail-whois action send a notification e-mail with a whois request

server/filter.py

@@ -35,7 +35,7 @@ from datedetector import DateDetector
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
 
-import logging, re, os, fcntl, time, sys
+import logging, re, os, fcntl, time, sys, locale, codecs
 
 # Gets the instance of the logger.
 logSys = logging.getLogger("fail2ban.filter")
@@ -392,6 +392,7 @@ class FileFilter(Filter):
 		Filter.__init__(self, jail, **kwargs)
 		## The log file path.
 		self.__logPath = []
+		self.setLogEncoding("auto")
 
 	##
 	# Add a log file path
@@ -402,7 +403,7 @@ class FileFilter(Filter):
 		if self.containsLogPath(path):
 			logSys.error(path + " already exists")
 		else:
-			container = FileContainer(path, tail)
+			container = FileContainer(path, self.getLogEncoding(), tail)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@@ -451,6 +452,28 @@ class FileFilter(Filter):
 				return True
 		return False
 
+	##
+	# Set the log file encoding
+	#
+	# @param encoding the encoding used with log files
+
+	def setLogEncoding(self, encoding):
+		if encoding.lower() == "auto":
+			encoding = locale.getpreferredencoding()
+		codecs.lookup(encoding) # Raise LookupError if invalid codec
+		for log in self.getLogPath():
+			log.setEncoding(encoding)
+		self.__encoding = encoding
+		logSys.info("Set jail log file encoding to %s" % encoding)
+
+	##
+	# Get the log file encoding
+	#
+	# @return log encoding value
+
+	def getLogEncoding(self):
+		return self.__encoding
+
 	def getFileContainer(self, path):
 		for log in self.__logPath:
 			if log.getFileName() == path:
@@ -510,8 +533,9 @@ except ImportError:
 
 class FileContainer:
 
-	def __init__(self, filename, tail = False):
+	def __init__(self, filename, encoding, tail = False):
 		self.__filename = filename
+		self.setEncoding(encoding)
 		self.__tail = tail
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
@@ -534,6 +558,13 @@ class FileContainer:
 	def getFileName(self):
 		return self.__filename
 
+	def setEncoding(self, encoding):
+		codecs.lookup(encoding) # Raises LookupError if invalid
+		self.__encoding = encoding
+
+	def getEncoding(self):
+		return self.__encoding
+
 	def open(self):
 		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
@@ -557,11 +588,12 @@ class FileContainer:
 			return ""
 		line = self.__handler.readline()
 		try:
-			line = line.decode('utf-8', 'strict')
+			line = line.decode(self.getEncoding(), 'strict')
 		except UnicodeDecodeError:
-			logSys.warn("Error decoding line to utf-8: %s" % `line`)
+			logSys.warn("Error decoding line from '%s' with '%s': %s" %
-			if sys.version_info >= (3,): # In python3, must be unicode
+				(self.getFileName(), self.getEncoding(), `line`))
-				line = line.decode('utf-8', 'ignore')
+			if sys.version_info >= (3,): # In python3, must be decoded
+				line = line.decode(self.getEncoding(), 'ignore')
 		return line
 
 	def close(self):

server/server.py

@@ -181,6 +181,12 @@ class Server:
 		return [m.getFileName()
 				for m in self.__jails.getFilter(name).getLogPath()]
 	
+	def setLogEncoding(self, name, encoding):
+		return self.__jails.getFilter(name).setLogEncoding(encoding)
+	
+	def getLogEncoding(self, name):
+		return self.__jails.getFilter(name).getLogEncoding()
+	
 	def setFindTime(self, name, value):
 		self.__jails.getFilter(name).setFindTime(value)
 	

server/transmitter.py

@@ -139,6 +139,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.delLogPath(name, value)
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			value = command[2]
+			self.__server.setLogEncoding(name, value)
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value)
@@ -234,6 +238,8 @@ class Transmitter:
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "ignoreip":
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "failregex":

testcases/servertestcase.py

@@ -27,7 +27,7 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import unittest, socket, time, tempfile, os
+import unittest, socket, time, tempfile, os, locale
 from server.server import Server
 
 class StartStop(unittest.TestCase):
@@ -258,6 +258,13 @@ class Transmitter(unittest.TestCase):
 		self.setGetTest("maxretry", "-2", -2, jail=self.jailName)
 		self.setGetTestNOK("maxretry", "Duck", jail=self.jailName)
 
+	def testJailLogEncoding(self):
+		self.setGetTest("logencoding", "UTF-8", jail=self.jailName)
+		self.setGetTest("logencoding", "ascii", jail=self.jailName)
+		self.setGetTest("logencoding", "auto", locale.getpreferredencoding(),
+			jail=self.jailName)
+		self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)
+
 	def testJailLogPath(self):
 		self.jailAddDelTest(
 			"logpath",