Merge branch 'py3' of https://github.com/kwirk/fail2ban into 0.9

* 'py3' of https://github.com/kwirk/fail2ban: (38 commits) DOC: Add python3 to requirements ENH: Clarify use of bytes in csocket and asyncserver for python3 DOC: Revert dnsToIp error change, seperate log message for socket.error TST: Tweak python3 open statement to resolve python2.5 SyntaxError TST: Revert changes for filter testcase open statement DOC: Revert setup.py messages to use print statement Add *.bak files generated by 2to3 to gitignore TST: Fix up fail2ban python3 scripts TST: Fix issues in tests which assumed dictionary's order ENH: setup.py now automatically runs 2to3 for python3.x TST: Remove Travis CI unsupported versions of python from Travis config add fail2ban-2to3 to MANIFEST file ENH: Add python3 versions to Travis CI config BF: Handle expected errors for python3.{0,1} when changing log target Minor tweaks to fail2ban-regex for encoding Added ability to set log file encoding with fail2ban-regex Add ability to set log encoding for jail Move handling of unicode decoding to FileContainer readline Fix incorrect exit code from fail2ban-2to3 Remove redundant reassignment of variable ... Conflicts: fail2ban/tests/servertestcase.py -- both branches added a new unittest at the same point
2013-04-16 23:24:49 -04:00 · 2013-04-16 23:24:49 -04:00 · 4869186c8f
parent f5572c8ade f14cb7302a
commit 4869186c8f
24 changed files with 246 additions and 82 deletions
--- a/.gitignore
+++ b/.gitignore
@ -6,3 +6,4 @@ htmlcov
 .coverage
 *.orig
 *.rej
+*.bak
--- a/.travis.yml
+++ b/.travis.yml
@ -5,12 +5,16 @@ python:
  - "2.5"
  - "2.6"
  - "2.7"
+  - "3.2"
+  - "3.3"
 before_install:
  - sudo apt-get update -qq
 install:
  - pip install pyinotify
  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; fi
  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi
+before_script:
+    - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi
 script:
  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi
  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi
--- a/1
+++ b/1
@ -5,6 +5,7 @@ THANKS
 COPYING
 DEVELOP
 doc/run-rootless.txt
+fail2ban-2to3
 bin/fail2ban-client
 bin/fail2ban-server
 bin/fail2ban-testcases
--- a/2
+++ b/2
@ -19,7 +19,7 @@ Installation:
 -------------

 Required:
-   >=python-2.3 (http://www.python.org)
+   >=python-2.3 or >=python-3.0 (http://www.python.org)

 Optional:
   pyinotify:
--- a/bin/fail2ban-regex
+++ b/bin/fail2ban-regex
@ -22,7 +22,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko"
 __license__ = "GPL"

-import getopt, sys, time, logging, os
+import getopt, sys, time, logging, os, locale
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError

 from fail2ban.version import version
@ -70,6 +70,7 @@ class Fail2banRegex:
 		self.__ignoreregex = list()
 		self.__failregex = list()
 		self.__verbose = False
+		self.encoding = locale.getpreferredencoding()
 		# Setup logging
 		logging.getLogger("fail2ban").handlers = []
 		self.__hdlr = logging.StreamHandler(Fail2banRegex.test)
@ -103,6 +104,8 @@ class Fail2banRegex:
 		print "This tools can test regular expressions for \"fail2ban\"."
 		print
 		print "Options:"
+		print "    -e ENCODING, --encoding=ENCODING"
+		print "                            set the file encoding. default:system locale"
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print "    -v, --verbose           verbose output"
@ -135,6 +138,8 @@ class Fail2banRegex:
 				sys.exit(0)
 			elif opt[0] in ["-v", "--verbose"]:
 				self.__verbose = True
+			elif opt[0] in ["-e", "--encoding"]:
+				self.encoding = opt[1]
 			elif opt[0] in ["-l", "--maxlines"]:
 				try:
 					self.__filter.setMaxLines(int(opt[1]))
@ -320,8 +325,8 @@ if __name__ == "__main__":
 	fail2banRegex = Fail2banRegex()
 	# Reads the command line options.
 	try:
-		cmdOpts = 'hVcvl:'
-		cmdLongOpts = ['help', 'version', 'verbose', 'maxlines=']
+		cmdOpts = 'hVcvl:e:'
+		cmdLongOpts = ['help', 'version', 'verbose', 'maxlines=', 'encoding=']
 		optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts)
 	except getopt.GetoptError:
 		fail2banRegex.dispUsage()
@ -348,10 +353,16 @@ if __name__ == "__main__":

 		if fail2banRegex.logIsFile(cmd_log):
 			try:
-				hdlr = open(cmd_log)
+				hdlr = open(cmd_log, 'rb')
 				print "Use log file   : " + cmd_log
+				print "Use encoding   : " + fail2banRegex.encoding
 				print
 				for line in hdlr:
+					try:
+						line = line.decode(fail2banRegex.encoding, 'strict')
+					except UnicodeDecodeError:
+						if sys.version_info >= (3,): # Python 3 must be decoded
+							line = line.decode(fail2banRegex.encoding, 'ignore')
 					fail2banRegex.testIgnoreRegex(line)
 					fail2banRegex.testRegex(line)
 			except IOError, e:
--- a/bin/fail2ban-server
+++ b/bin/fail2ban-server
@ -97,10 +97,10 @@ class Fail2banServer:
 			if opt[0] == "-x":
 				self.__conf["force"] = True
 			if opt[0] in ["-h", "--help"]:
-	 			self.dispUsage()
+				self.dispUsage()
 				sys.exit(0)
 			if opt[0] in ["-V", "--version"]:
-	 			self.dispVersion()
+				self.dispVersion()
 				sys.exit(0)

 	def start(self, argv):
--- a/config/jail.conf
+++ b/config/jail.conf
@ -58,6 +58,13 @@ backend = auto
 #        but it will be logged as info.
 usedns = warn

+# "logencoding" specifies the encoding of the log files handled by the jail
+#   This is used to decode the lines from the log file.
+#   Typical examples:  "ascii", "utf-8"
+#
+#   auto:   will use the system locale setting
+logencoding = auto
+

 # This jail corresponds to the standard configuration in Fail2ban 0.6.
 # The mail-whois action send a notification e-mail with a whois request
--- a/14
+++ b/14
@ -0,0 +1,14 @@
+#!/bin/bash
+# This script carries out conversion of fail2ban to python3
+# A backup of any converted files are created with ".bak"
+# extension
+
+set -eu
+
+if 2to3 -w --no-diffs bin/* fail2ban;then
+  echo "Success!" >&2
+  exit 0
+else
+  echo "Fail!" >&2
+  exit 1
+fi
--- a/18
+++ b/18
@ -0,0 +1,18 @@
+#!/bin/bash
+# Simple helper script to exercise unittests using all available
+# (under /usr/bin and /usr/local/bin python3.*)
+
+set -eu
+
+failed=
+for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg}
+do
+	[ -e "$python" ] || continue
+	echo "Testing using $python"
+	$python bin/fail2ban-testcases "$@" || failed+=" $python"
+done
+
+if [ ! -z "$failed" ]; then
+	echo "E: Failed with $failed"
+	exit 1
+fi
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@ -110,6 +110,9 @@ class Beautifier:
 					for path in response[:-1]:
 						msg = msg + "|- " + path + "\n"
 					msg = msg + "`- " + response[len(response)-1]
+			elif inC[2] == "logencoding":
+				msg = "Current log encoding is set to:\n"
+				msg = msg + response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@ -29,11 +29,21 @@ __license__ = "GPL"

 #from cPickle import dumps, loads, HIGHEST_PROTOCOL
 from pickle import dumps, loads, HIGHEST_PROTOCOL
-import socket
+import socket, sys
+
+if sys.version_info >= (3,):
+	# b"" causes SyntaxError in python <= 2.5, so below implements equivalent
+	EMPTY_BYTES = bytes("", encoding="ascii")
+else:
+	# python 2.x, string type is equivalent to bytes.
+	EMPTY_BYTES = ""

 class CSocket:
 	
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding='ascii')
+	else:
+		END_STRING = "<F2B_END_COMMAND>"
 	
 	def __init__(self, sock = "/var/run/fail2ban/fail2ban.sock"):
 		# Create an INET, STREAMing socket
@ -52,7 +62,7 @@ class CSocket:
 	
 	#@staticmethod
 	def receive(sock):
-		msg = ''
+		msg = EMPTY_BYTES
 		while msg.rfind(CSocket.END_STRING) == -1:
 			chunk = sock.recv(6)
 			if chunk == '':
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@ -62,6 +62,7 @@ class JailReader(ConfigReader):
 	def getOptions(self):
 		opts = [["bool", "enabled", "false"],
 				["string", "logpath", "/var/log/messages"],
+				["string", "logencoding", "auto"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
 				["int", "maxlines", 1],
@ -117,6 +118,8 @@ class JailReader(ConfigReader):
 						logSys.error("No file found for " + path)
 					for p in pathList:
 						stream.append(["set", self.__name, "addlogpath", p])
+			elif opt == "logencoding":
+				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
 			elif opt == "backend":
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@ -57,6 +57,7 @@ protocol = [
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
 ["set <JAIL> addlogpath <FILE>", "adds <FILE> to the monitoring list of <JAIL>"], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
+["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
@ -79,6 +80,7 @@ protocol = [
 ["set <JAIL> actionunban <ACT> <CMD>", "sets the unban command <CMD> of the action <ACT> for <JAIL>"], 
 ['', "JAIL INFORMATION", ""],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
+["get <JAIL> logencoding <ENCODING>", "gets the <ENCODING> of the log files for <JAIL>"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@ -342,7 +342,7 @@ class Action:
 					return True
 				else:
 					msg = _RETCODE_HINTS.get(retcode, None)
- 					logSys.error("%s returned %x" % (realCmd, retcode))
+					logSys.error("%s returned %x" % (realCmd, retcode))
 					if msg:
 						logSys.info("HINT on %x: %s"
 									% (retcode, msg % locals()))
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@ -35,6 +35,13 @@ from fail2ban import helpers
 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)

+if sys.version_info >= (3,):
+	# b"" causes SyntaxError in python <= 2.5, so below implements equivalent
+	EMPTY_BYTES = bytes("", encoding="ascii")
+else:
+	# python 2.x, string type is equivalent to bytes.
+	EMPTY_BYTES = ""
+
 ##
 # Request handler class.
 #
@ -43,7 +50,10 @@ logSys = logging.getLogger(__name__)

 class RequestHandler(asynchat.async_chat):
 	
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding="ascii")
+	else:
+		END_STRING = "<F2B_END_COMMAND>"

 	def __init__(self, conn, transmitter):
 		asynchat.async_chat.__init__(self, conn)
@ -63,7 +73,7 @@ class RequestHandler(asynchat.async_chat):

 	def found_terminator(self):
 		# Joins the buffer items.
-		message = loads("".join(self.__buffer))
+		message = loads(EMPTY_BYTES.join(self.__buffer))
 		# Gives the message to the transmitter.
 		message = self.__transmitter.proceed(message)
 		# Serializes the response.
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@ -206,7 +206,7 @@ class DateDetector:
 		if date == None:
 			return None
 		else:
-			return time.mktime(date)
+			return time.mktime(tuple(date))

 	##
 	# Sort the template lists using the hits score. This method is not called
@ -216,7 +216,7 @@ class DateDetector:
 		self.__lock.acquire()
 		try:
 			logSys.debug("Sorting the template list")
-			self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits()), reverse=True)
+			self.__templates.sort(key=lambda x: x.getHits(), reverse=True)
 			t = self.__templates[0]
 			logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits()))
 		finally:
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@ -166,10 +166,10 @@ class DateStrptime(DateTemplate):
 				# Bug fix for #1241756
 				# If the date is greater than the current time, we suppose
 				# that the log is not from this year but from the year before
-				if time.mktime(date) > MyTime.time():
+				if time.mktime(tuple(date)) > MyTime.time():
 					logSys.debug(
 						u"Correcting deduced year from %d to %d since %f > %f" %
-						(date[0], date[0]-1, time.mktime(date), MyTime.time()))
+						(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
 					# NOTE: Possibly makes week/year day incorrect
 					date[0] -= 1
 				elif date[1] == 1 and date[2] == 1:
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@ -35,7 +35,7 @@ from datedetector import DateDetector
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException

-import logging, re, os, fcntl, time
+import logging, re, os, fcntl, time, sys, locale, codecs

 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)
@ -316,12 +316,7 @@ class Filter(JailThread):
 	def processLine(self, line):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
-		try:
-			# Decode line to UTF-8
-			l = line.decode('utf-8')
-		except UnicodeDecodeError:
-			l = line
-		timeMatch = self.dateDetector.matchTime(l)
+		timeMatch = self.dateDetector.matchTime(line)
 		if timeMatch:
 			# Lets split into time part and log part of the line
 			timeLine = timeMatch.group()
@ -329,10 +324,10 @@ class Filter(JailThread):
 			# Lets leave the beginning in as well, so if there is no
 			# anchore at the beginning of the time regexp, we don't
 			# at least allow injection. Should be harmless otherwise
-			logLine  = l[:timeMatch.start()] + l[timeMatch.end():]
+			logLine  = line[:timeMatch.start()] + line[timeMatch.end():]
 		else:
-			timeLine = self.__lastTimeLine or l
-			logLine = l
+			timeLine = self.__lastTimeLine or line
+			logLine = line
 		self.__lineBuffer = ((self.__lineBuffer +
 				[logLine])[-self.__lineBufferSize:])
 		return self.findFailure(timeLine, "".join(self.__lineBuffer))
@ -429,6 +424,7 @@ class FileFilter(Filter):
 		Filter.__init__(self, jail, **kwargs)
 		## The log file path.
 		self.__logPath = []
+		self.setLogEncoding("auto")

 	##
 	# Add a log file path
@ -439,7 +435,7 @@ class FileFilter(Filter):
 		if self.containsLogPath(path):
 			logSys.error(path + " already exists")
 		else:
-			container = FileContainer(path, tail)
+			container = FileContainer(path, self.getLogEncoding(), tail)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@ -488,6 +484,28 @@ class FileFilter(Filter):
 				return True
 		return False

+	##
+	# Set the log file encoding
+	#
+	# @param encoding the encoding used with log files
+
+	def setLogEncoding(self, encoding):
+		if encoding.lower() == "auto":
+			encoding = locale.getpreferredencoding()
+		codecs.lookup(encoding) # Raise LookupError if invalid codec
+		for log in self.getLogPath():
+			log.setEncoding(encoding)
+		self.__encoding = encoding
+		logSys.info("Set jail log file encoding to %s" % encoding)
+
+	##
+	# Get the log file encoding
+	#
+	# @return log encoding value
+
+	def getLogEncoding(self):
+		return self.__encoding
+
 	def getFileContainer(self, path):
 		for log in self.__logPath:
 			if log.getFileName() == path:
@ -525,7 +543,7 @@ class FileFilter(Filter):

 		while True:
 			line = container.readline()
-			if (line == "") or not self._isActive():
+			if not line or not self._isActive():
 				# The jail reached the bottom or has been stopped
 				break
 			self.processLineAndAdd(line)
@ -556,12 +574,13 @@ except ImportError: # pragma: no cover

 class FileContainer:

-	def __init__(self, filename, tail = False):
+	def __init__(self, filename, encoding, tail = False):
 		self.__filename = filename
+		self.setEncoding(encoding)
 		self.__tail = tail
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
-		handler = open(filename)
+		handler = open(filename, 'rb')
 		stats = os.fstat(handler.fileno())
 		self.__ino = stats.st_ino
 		try:
@ -580,8 +599,15 @@ class FileContainer:
 	def getFileName(self):
 		return self.__filename

+	def setEncoding(self, encoding):
+		codecs.lookup(encoding) # Raises LookupError if invalid
+		self.__encoding = encoding
+
+	def getEncoding(self):
+		return self.__encoding
+
 	def open(self):
-		self.__handler = open(self.__filename)
+		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
 		fd = self.__handler.fileno()
 		fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC)
@ -601,7 +627,15 @@ class FileContainer:
 	def readline(self):
 		if self.__handler == None:
 			return ""
-		return self.__handler.readline()
+		line = self.__handler.readline()
+		try:
+			line = line.decode(self.getEncoding(), 'strict')
+		except UnicodeDecodeError:
+			logSys.warn("Error decoding line from '%s' with '%s': %s" %
+				(self.getFileName(), self.getEncoding(), `line`))
+			if sys.version_info >= (3,): # In python3, must be decoded
+				line = line.decode(self.getEncoding(), 'ignore')
+		return line

 	def close(self):
 		if not self.__handler == None:
@ -636,6 +670,10 @@ class DNSUtils:
 			logSys.warn("Unable to find a corresponding IP address for %s"
 						% dns)
 			return list()
+		except socket.error, e:
+			logSys.warn("Socket error raised trying to resolve hostname %s: %s"
+						% (dns, e))
+			return list()
 	dnsToIp = staticmethod(dnsToIp)

 	#@staticmethod
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@ -181,6 +181,12 @@ class Server:
 		return [m.getFileName()
 				for m in self.__jails.getFilter(name).getLogPath()]
 	
+	def setLogEncoding(self, name, encoding):
+		return self.__jails.getFilter(name).setLogEncoding(encoding)
+	
+	def getLogEncoding(self, name):
+		return self.__jails.getFilter(name).getLogEncoding()
+	
 	def setFindTime(self, name, value):
 		self.__jails.getFilter(name).setFindTime(value)
 	
@ -288,12 +294,9 @@ class Server:
 	def status(self):
 		try:
 			self.__lock.acquire()
-			jailList = ''
-			for jail in self.__jails.getAll():
-				jailList += jail + ', '
-			length = len(jailList)
-			if not length == 0:
-				jailList = jailList[:length-2]
+			jails = list(self.__jails.getAll())
+			jails.sort()
+			jailList = ", ".join(jails)
 			ret = [("Number of jail", self.__jails.size()), 
 				   ("Jail list", jailList)]
 			return ret
@ -387,7 +390,8 @@ class Server:
 					handler.flush()
 					handler.close()
 				except (ValueError, KeyError):
-					if sys.version_info >= (2,6):
+					if (2,6) <= sys.version_info < (3,) or \
+							(3,2) <= sys.version_info:
 						raise
 					# is known to be thrown after logging was shutdown once
 					# with older Pythons -- seems to be safe to ignore there
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@ -143,6 +143,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.delLogPath(name, value)
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			value = command[2]
+			self.__server.setLogEncoding(name, value)
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value)
@ -242,6 +246,8 @@ class Transmitter:
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "ignoreip":
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "failregex":
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@ -146,7 +146,9 @@ class FilterReaderTest(unittest.TestCase):
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)

-		self.assertEquals(filterReader.convert(), output)
+		# Add sort as configreader uses dictionary and therefore order
+		# is unreliable
+		self.assertEquals(sorted(filterReader.convert()), sorted(output))

 class JailsReaderTest(unittest.TestCase):

--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@ -53,7 +53,10 @@ def open(*args):
 	if len(args) == 2:
 		# ~50kB buffer should be sufficient for all tests here.
 		args = args + (50000,)
-	return fopen(*args)
+	if sys.version_info >= (3,):
+		return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'})
+	else:
+		return fopen(*args)

 def _killfile(f, name):
 	try:
@ -97,22 +100,25 @@ def _assert_equal_entries(utest, found, output, count=None):
 		# do not check if custom count (e.g. going through them twice)
 		utest.assertEqual(repr(found[3]), repr(output[3]))

+def _ticket_tuple(ticket):
+	"""Create a tuple for easy comparison from fail ticket
+	"""
+	attempts = ticket.getAttempt()
+	date = ticket.getTime()
+	ip = ticket.getIP()
+	matches = ticket.getMatches()
+	return (ip, attempts, date, matches)
+
 def _assert_correct_last_attempt(utest, filter_, output, count=None):
 	"""Additional helper to wrap most common test case

 	Test filter to contain target ticket
 	"""
 	if isinstance(filter_, DummyJail):
-		ticket = filter_.getFailTicket()
+		found = _ticket_tuple(filter_.getFailTicket())
 	else:
 		# when we are testing without jails
-		ticket = filter_.failManager.toBan()
-
-	attempts = ticket.getAttempt()
-	date = ticket.getTime()
-	ip = ticket.getIP()
-	matches = ticket.getMatches()
-	found = (ip, attempts, date, matches)
+		found = _ticket_tuple(filter_.failManager.toBan())

 	_assert_equal_entries(utest, found, output, count)

@ -437,7 +443,7 @@ def get_monitor_failures_testcase(Filter_):
 			#return
 			# just for fun let's copy all of them again and see if that results
 			# in a new ban
- 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
+			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)

 		def test_rewrite_file(self):
@ -533,7 +539,7 @@ class GetFailures(unittest.TestCase):

 	# so that they could be reused by other tests
 	FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
-				  ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
+				  [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)

 	def setUp(self):
 		"""Call before every test case."""
@ -557,7 +563,7 @@ class GetFailures(unittest.TestCase):

 	def testGetFailures02(self):
 		output = ('141.3.81.106', 4, 1124013539.0,
-				  ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
+				  [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
 				   % m for m in 53, 54, 57, 58])

 		self.filter.addLogPath(GetFailures.FILENAME_02)
@ -590,11 +596,11 @@ class GetFailures(unittest.TestCase):
 	def testGetFailuresUseDNS(self):
 		# We should still catch failures with usedns = no ;-)
 		output_yes = ('192.0.43.10', 2, 1124013539.0,
-					  ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
-					   'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
+					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
+					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])

 		output_no = ('192.0.43.10', 1, 1124013539.0,
-					  ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
+					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])

 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,
@ -645,10 +651,14 @@ class GetFailures(unittest.TestCase):

 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)

-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-
-		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+		foundList = []
+		while True:
+			try:
+				foundList.append(
+					_ticket_tuple(self.filter.failManager.toBan())[0:3])
+			except FailManagerEmpty:
+				break
+		self.assertEqual(sorted(foundList), sorted(output))

 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
@ -676,11 +686,14 @@ class GetFailures(unittest.TestCase):

 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)

-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-
-		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+		foundList = []
+		while True:
+			try:
+				foundList.append(
+					_ticket_tuple(self.filter.failManager.toBan())[0:3])
+			except FailManagerEmpty:
+				break
+		self.assertEqual(sorted(foundList), sorted(output))

 class DNSUtilsTests(unittest.TestCase):

--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@ -27,7 +27,7 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"

-import unittest, socket, time, tempfile, os
+import unittest, socket, time, tempfile, os, locale

 from fail2ban.server.server import Server
 from fail2ban.exceptions import UnknownJailException
@ -277,6 +277,13 @@ class Transmitter(TransmitterBase):
 		self.setGetTestNOK("maxlines", "-2", jail=self.jailName)
 		self.setGetTestNOK("maxlines", "Duck", jail=self.jailName)

+	def testJailLogEncoding(self):
+		self.setGetTest("logencoding", "UTF-8", jail=self.jailName)
+		self.setGetTest("logencoding", "ascii", jail=self.jailName)
+		self.setGetTest("logencoding", "auto", locale.getpreferredencoding(),
+			jail=self.jailName)
+		self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)
+
 	def testJailLogPath(self):
 		self.jailAddDelTest(
 			"logpath",
--- a/setup.py
+++ b/setup.py
@ -23,6 +23,15 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"

 from distutils.core import setup
+try:
+	# python 3.x
+	from distutils.command.build_py import build_py_2to3 as build_py
+	from distutils.command.build_scripts \
+		import build_scripts_2to3 as build_scripts
+except ImportError:
+	# python 2.x
+	from distutils.command.build_py import build_py
+	from distutils.command.build_scripts import build_scripts
 from os.path import isfile, join, isdir
 import sys
 from glob import glob
@ -46,6 +55,7 @@ setup(
 	url = "http://www.fail2ban.org",
 	license = "GPL",
 	platforms = "Posix",
+	cmdclass = {'build_py': build_py, 'build_scripts': build_scripts},
 	scripts =	[
 					'bin/fail2ban-client',
 					'bin/fail2ban-server',
@ -107,25 +117,25 @@ for directory in elements:
 			obsoleteFiles.append(path)

 if obsoleteFiles:
-	print
-	print "Obsolete files from previous Fail2Ban versions were found on " \
-		  "your system."
-	print "Please delete them:"
-	print
+	print("")
+	print("Obsolete files from previous Fail2Ban versions were found on "
+		  "your system.")
+	print("Please delete them:")
+	print("")
 	for f in obsoleteFiles:
-		print "\t" + f
-	print
+		print("\t" + f)
+	print("")

 if isdir("/usr/lib/fail2ban"):
-	print
-	print "Fail2ban is not installed under /usr/lib anymore. The new " \
-		  "location is under /usr/share. Please remove the directory " \
-		  "/usr/lib/fail2ban and everything under this directory."
-	print
+	print("")
+	print("Fail2ban is not installed under /usr/lib anymore. The new "
+		  "location is under /usr/share. Please remove the directory "
+		  "/usr/lib/fail2ban and everything under this directory.")
+	print("")

 # Update config file
 if sys.argv[1] == "install":
-	print
-	print "Please do not forget to update your configuration files."
-	print "They are in /etc/fail2ban/."
-	print
+	print("")
+	print("Please do not forget to update your configuration files.")
+	print("They are in /etc/fail2ban/.")
+	print("")