From c926c88ea13577151d30e88ed32a610dee205ca0 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 18:42:50 +0000
Subject: [PATCH 01/38] Remove spurious space at start of line

---
 server/action.py            | 2 +-
 testcases/filtertestcase.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/action.py b/server/action.py
index 35974c70..65263751 100644
--- a/server/action.py
+++ b/server/action.py
@@ -342,7 +342,7 @@ class Action:
 					return True
 				else:
 					msg = _RETCODE_HINTS.get(retcode, None)
- 					logSys.error("%s returned %x" % (realCmd, retcode))
+					logSys.error("%s returned %x" % (realCmd, retcode))
 					if msg:
 						logSys.info("HINT on %x: %s"
 									% (retcode, msg % locals()))
diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py
index c10fa78d..3b764729 100644
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@@ -409,7 +409,7 @@ def get_monitor_failures_testcase(Filter_):
 			#return
 			# just for fun let's copy all of them again and see if that results
 			# in a new ban
- 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
+			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 
 		def test_rewrite_file(self):

From 99c92a56ca602bad7936966843d6ca20e70ee37f Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 18:47:56 +0000
Subject: [PATCH 02/38] Check for unicode and encode before md5sum in filter

---
 server/filter.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/server/filter.py b/server/filter.py
index fb79fcbe..dab8cb8a 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -526,7 +526,10 @@ class FileContainer:
 		try:
 			firstLine = handler.readline()
 			# Computes the MD5 of the first line.
-			self.__hash = md5sum(firstLine).digest()
+			if isinstance(firstLine, unicode):
+				self.__hash = md5sum(firstLine.encode('utf-8')).digest()
+			else:
+				self.__hash = md5sum(firstLine).digest()
 			# Start at the beginning of file if tail mode is off.
 			if tail:
 				handler.seek(0, 2)
@@ -546,7 +549,10 @@ class FileContainer:
 		fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC)
 		firstLine = self.__handler.readline()
 		# Computes the MD5 of the first line.
-		myHash = md5sum(firstLine).digest()
+		if isinstance(firstLine, unicode):
+			myHash = md5sum(firstLine.encode('utf-8')).digest()
+		else:
+			myHash = md5sum(firstLine).digest()
 		stats = os.fstat(self.__handler.fileno())
 		# Compare hash and inode
 		if self.__hash != myHash or self.__ino != stats.st_ino:

From 2ecf34540e601e01381bcd41ac6308c3f9299109 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 19:33:49 +0000
Subject: [PATCH 03/38] Change sort in datedetector to use key based comparison

---
 server/datedetector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/datedetector.py b/server/datedetector.py
index c013d551..d31667fb 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -204,7 +204,7 @@ class DateDetector:
 		self.__lock.acquire()
 		try:
 			logSys.debug("Sorting the template list")
-			self.__templates.sort(lambda x, y: cmp(x.getHits(), y.getHits()), reverse=True)
+			self.__templates.sort(key=lambda x: x.getHits(), reverse=True)
 			t = self.__templates[0]
 			logSys.debug("Winning template: %s with %d hits" % (t.getName(), t.getHits()))
 		finally:

From ad7119a3fe4fb3ad33716911f1f62c3e2a5670ad Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 21:01:39 +0000
Subject: [PATCH 04/38] time.mktime argument should be tuple

---
 server/datedetector.py | 2 +-
 server/datetemplate.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/server/datedetector.py b/server/datedetector.py
index d31667fb..7876580d 100644
--- a/server/datedetector.py
+++ b/server/datedetector.py
@@ -194,7 +194,7 @@ class DateDetector:
 		if date == None:
 			return None
 		else:
-			return time.mktime(date)
+			return time.mktime(tuple(date))
 
 	##
 	# Sort the template lists using the hits score. This method is not called
diff --git a/server/datetemplate.py b/server/datetemplate.py
index f663862e..14076030 100644
--- a/server/datetemplate.py
+++ b/server/datetemplate.py
@@ -162,10 +162,10 @@ class DateStrptime(DateTemplate):
 				# Bug fix for #1241756
 				# If the date is greater than the current time, we suppose
 				# that the log is not from this year but from the year before
-				if time.mktime(date) > MyTime.time():
+				if time.mktime(tuple(date)) > MyTime.time():
 					logSys.debug(
-						u"Correcting deduced year from %d to %d since %f > %f" %
-						(date[0], date[0]-1, time.mktime(date), MyTime.time()))
+						"Correcting deduced year from %d to %d since %f > %f" %
+						(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
 					date[0] -= 1
 				elif date[1] == 1 and date[2] == 1:
 					# If it is Jan 1st, it is either really Jan 1st or there

From 2bb346964432080899d1bf17223841f4b5b1cbae Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 21:03:51 +0000
Subject: [PATCH 05/38] Change filter to ignore unicode errors in python3

Also do not try to convert unicode to unicode again for python3 and
python2
---
 server/filter.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/server/filter.py b/server/filter.py
index dab8cb8a..e47458de 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -35,7 +35,7 @@ from datedetector import DateDetector
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
 
-import logging, re, os, fcntl, time
+import logging, re, os, fcntl, time, sys
 
 # Gets the instance of the logger.
 logSys = logging.getLogger("fail2ban.filter")
@@ -289,22 +289,24 @@ class Filter(JailThread):
 	def processLine(self, line):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
-		try:
-			# Decode line to UTF-8
-			l = line.decode('utf-8')
-		except UnicodeDecodeError:
-			l = line
-		timeMatch = self.dateDetector.matchTime(l)
+		if (sys.version_info >= (3,) and isinstance(line, bytes)) or \
+			(sys.version_info < (3,) and isinstance(line, str)):
+			try:
+				# Decode line to UTF-8
+				line = line.decode('utf-8')
+			except UnicodeDecodeError:
+				line = line
+		timeMatch = self.dateDetector.matchTime(line)
 		if timeMatch:
 			# Lets split into time part and log part of the line
 			timeLine = timeMatch.group()
 			# Lets leave the beginning in as well, so if there is no
 			# anchore at the beginning of the time regexp, we don't
 			# at least allow injection. Should be harmless otherwise
-			logLine  = l[:timeMatch.start()] + l[timeMatch.end():]
+			logLine  = line[:timeMatch.start()] + line[timeMatch.end():]
 		else:
-			timeLine = l
-			logLine = l
+			timeLine = line
+			logLine = line
 		return self.findFailure(timeLine, logLine)
 
 	def processLineAndAdd(self, line):
@@ -520,7 +522,10 @@ class FileContainer:
 		self.__tail = tail
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
-		handler = open(filename)
+		if sys.version_info >= (3,):
+			handler = open(filename, errors='ignore')
+		else:
+			handler = open(filename)
 		stats = os.fstat(handler.fileno())
 		self.__ino = stats.st_ino
 		try:

From 53be2ade863d0ead9ca53566fad2ef8684f80399 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 21:05:23 +0000
Subject: [PATCH 06/38] Wrap open method to allow python3 to ignore unicode
 errors

---
 testcases/filtertestcase.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py
index 3b764729..8ab3fca8 100644
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@@ -27,6 +27,7 @@ import os
 import sys
 import time
 import tempfile
+import functools
 
 from server.jail import Jail
 from server.filterpoll import FilterPoll
@@ -38,6 +39,11 @@ from server.failmanager import FailManagerEmpty
 # Useful helpers
 #
 
+if sys.version_info >= (3,):
+	open_ = functools.partial(open, errors='ignore')
+else:
+	open_ = open
+
 def _killfile(f, name):
 	try:
 		f.close()
@@ -104,9 +110,9 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line
 		# polling filter could detect the change
 		time.sleep(1)
 	if isinstance(fin, str):
-		fin = open(fin, 'r')
+		fin = open_(fin, 'r')
 	if isinstance(fout, str):
-		fout = open(fout, mode)
+		fout = open_(fout, mode)
 	# Skip
 	for i in xrange(skip):
 		_ = fin.readline()
@@ -184,7 +190,7 @@ class LogFileMonitor(unittest.TestCase):
 		"""Call before every test case."""
 		self.filter = self.name = 'NA'
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
-		self.file = open(self.name, 'a')
+		self.file = open_(self.name, 'a')
 		self.filter = FilterPoll(None)
 		self.filter.addLogPath(self.name)
 		self.filter.setActive(True)
@@ -226,7 +232,7 @@ class LogFileMonitor(unittest.TestCase):
 		# we are not signaling as modified whenever
 		# it gets away
 		self.assertTrue(self.notModified())
-		f = open(self.name, 'a')
+		f = open_(self.name, 'a')
 		self.assertTrue(self.isModified())
 		self.assertTrue(self.notModified())
 		_sleep_4_poll()
@@ -329,7 +335,7 @@ def get_monitor_failures_testcase(Filter_):
 			"""Call before every test case."""
 			self.filter = self.name = 'NA'
 			_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
-			self.file = open(self.name, 'a')
+			self.file = open_(self.name, 'a')
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
 			self.filter.addLogPath(self.name)
@@ -454,7 +460,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 
 			# create a bogus file in the same directory and see if that doesn't affect
-			open(self.name + '.bak2', 'w').write('')
+			open_(self.name + '.bak2', 'w').write('')
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100)
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)

From 5d0d362e3ff039467149e69ae17c7dcdf8b2a952 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 22:06:16 +0000
Subject: [PATCH 07/38] Ensure jail names return in alphabetical order in
 status

Primarily to support testing as order can switch
---
 server/server.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/server/server.py b/server/server.py
index 0e1b6c97..4a086078 100644
--- a/server/server.py
+++ b/server/server.py
@@ -282,12 +282,9 @@ class Server:
 	def status(self):
 		try:
 			self.__lock.acquire()
-			jailList = ''
-			for jail in self.__jails.getAll():
-				jailList += jail + ', '
-			length = len(jailList)
-			if not length == 0:
-				jailList = jailList[:length-2]
+			jails = list(self.__jails.getAll())
+			jails.sort()
+			jailList = ", ".join(jails)
 			ret = [("Number of jail", self.__jails.size()), 
 				   ("Jail list", jailList)]
 			return ret

From df255063ae3527edf29ec741528a1d434ad042d1 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Feb 2013 22:19:26 +0000
Subject: [PATCH 08/38] python3: Default open encoding for files to UTF-8

Also tidy up unicode check in processLine, which is handled by `2to3`
---
 server/filter.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/server/filter.py b/server/filter.py
index e47458de..b3b99e30 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -289,8 +289,7 @@ class Filter(JailThread):
 	def processLine(self, line):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
-		if (sys.version_info >= (3,) and isinstance(line, bytes)) or \
-			(sys.version_info < (3,) and isinstance(line, str)):
+		if not isinstance(line, unicode):
 			try:
 				# Decode line to UTF-8
 				line = line.decode('utf-8')
@@ -523,7 +522,7 @@ class FileContainer:
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
 		if sys.version_info >= (3,):
-			handler = open(filename, errors='ignore')
+			handler = open(filename, encoding='utf-8', errors='ignore')
 		else:
 			handler = open(filename)
 		stats = os.fstat(handler.fileno())
@@ -548,7 +547,11 @@ class FileContainer:
 		return self.__filename
 
 	def open(self):
-		self.__handler = open(self.__filename)
+		if sys.version_info >= (3,):
+			self.__handler = open(
+				self.__filename, encoding='utf-8', errors='ignore')
+		else:
+			self.__handler = open(self.__filename)
 		# Set the file descriptor to be FD_CLOEXEC
 		fd = self.__handler.fileno()
 		fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC)

From 6f4da8f3c4ceb771c785df55cbb4ca0d3b4fc5cd Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 10:04:36 +0000
Subject: [PATCH 09/38] Remove spurious space in fail2ban-server

---
 fail2ban-server | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fail2ban-server b/fail2ban-server
index 81db58bd..87ddd042 100755
--- a/fail2ban-server
+++ b/fail2ban-server
@@ -104,10 +104,10 @@ class Fail2banServer:
 			if opt[0] == "-x":
 				self.__conf["force"] = True
 			if opt[0] in ["-h", "--help"]:
-	 			self.dispUsage()
+				self.dispUsage()
 				sys.exit(0)
 			if opt[0] in ["-V", "--version"]:
-	 			self.dispVersion()
+				self.dispVersion()
 				sys.exit(0)
 		
 	def start(self, argv):

From 46a2b6e428c1e54293182b343ecb83eb2b5314d1 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 10:10:15 +0000
Subject: [PATCH 10/38] Fix up for client/server socket for python3

---
 client/csocket.py     | 12 +++++++++---
 server/asyncserver.py | 10 ++++++++--
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/client/csocket.py b/client/csocket.py
index 6e014e23..3963bbe5 100644
--- a/client/csocket.py
+++ b/client/csocket.py
@@ -29,11 +29,14 @@ __license__ = "GPL"
 
 #from cPickle import dumps, loads, HIGHEST_PROTOCOL
 from pickle import dumps, loads, HIGHEST_PROTOCOL
-import socket
+import socket, sys
 
 class CSocket:
 	
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = b"<F2B_END_COMMAND>"
+	else:
+		END_STRING = "<F2B_END_COMMAND>"
 	
 	def __init__(self, sock = "/var/run/fail2ban/fail2ban.sock"):
 		# Create an INET, STREAMing socket
@@ -52,7 +55,10 @@ class CSocket:
 	
 	#@staticmethod
 	def receive(sock):
-		msg = ''
+		if sys.version_info >= (3,):
+			msg = b''
+		else:
+			msg = ''
 		while msg.rfind(CSocket.END_STRING) == -1:
 			chunk = sock.recv(6)
 			if chunk == '':
diff --git a/server/asyncserver.py b/server/asyncserver.py
index 64ec8f39..0e06a790 100644
--- a/server/asyncserver.py
+++ b/server/asyncserver.py
@@ -42,7 +42,10 @@ logSys = logging.getLogger("fail2ban.server")
 
 class RequestHandler(asynchat.async_chat):
 	
-	END_STRING = "<F2B_END_COMMAND>"
+	if sys.version_info >= (3,):
+		END_STRING = b"<F2B_END_COMMAND>"
+	else:
+		END_STRING = "<F2B_END_COMMAND>"
 
 	def __init__(self, conn, transmitter):
 		asynchat.async_chat.__init__(self, conn)
@@ -62,7 +65,10 @@ class RequestHandler(asynchat.async_chat):
 
 	def found_terminator(self):
 		# Joins the buffer items.
-		message = loads("".join(self.__buffer))
+		if sys.version_info >= (3,):
+			message = loads(b"".join(self.__buffer))
+		else:
+			message = loads("".join(self.__buffer))
 		# Gives the message to the transmitter.
 		message = self.__transmitter.proceed(message)
 		# Serializes the response.

From e28a698c0eb7976c4ab51c44b3f5f4e94e90f6d3 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 10:11:34 +0000
Subject: [PATCH 11/38] Change filter testcases python3 open wrapper to utf-8

---
 testcases/filtertestcase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py
index 8ab3fca8..e8f19260 100644
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@@ -40,7 +40,7 @@ from server.failmanager import FailManagerEmpty
 #
 
 if sys.version_info >= (3,):
-	open_ = functools.partial(open, errors='ignore')
+	open_ = functools.partial(open, encoding='utf-8', errors='ignore')
 else:
 	open_ = open
 

From 0dd3a81ba276bb61a9332f9612bd13a276d4689e Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 18:04:40 +0000
Subject: [PATCH 12/38] Fix use of python3 bytes in client/server socket for
 python2.5

---
 client/csocket.py     | 4 ++--
 server/asyncserver.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/csocket.py b/client/csocket.py
index 3963bbe5..283d6357 100644
--- a/client/csocket.py
+++ b/client/csocket.py
@@ -34,7 +34,7 @@ import socket, sys
 class CSocket:
 	
 	if sys.version_info >= (3,):
-		END_STRING = b"<F2B_END_COMMAND>"
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding='ascii')
 	else:
 		END_STRING = "<F2B_END_COMMAND>"
 	
@@ -56,7 +56,7 @@ class CSocket:
 	#@staticmethod
 	def receive(sock):
 		if sys.version_info >= (3,):
-			msg = b''
+			msg = bytes("", encoding='ascii')
 		else:
 			msg = ''
 		while msg.rfind(CSocket.END_STRING) == -1:
diff --git a/server/asyncserver.py b/server/asyncserver.py
index 0e06a790..a671d877 100644
--- a/server/asyncserver.py
+++ b/server/asyncserver.py
@@ -43,7 +43,7 @@ logSys = logging.getLogger("fail2ban.server")
 class RequestHandler(asynchat.async_chat):
 	
 	if sys.version_info >= (3,):
-		END_STRING = b"<F2B_END_COMMAND>"
+		END_STRING = bytes("<F2B_END_COMMAND>", encoding="ascii")
 	else:
 		END_STRING = "<F2B_END_COMMAND>"
 
@@ -66,7 +66,7 @@ class RequestHandler(asynchat.async_chat):
 	def found_terminator(self):
 		# Joins the buffer items.
 		if sys.version_info >= (3,):
-			message = loads(b"".join(self.__buffer))
+			message = loads(bytes("", encoding="ascii").join(self.__buffer))
 		else:
 			message = loads("".join(self.__buffer))
 		# Gives the message to the transmitter.

From c8c9ed4dfcf5526defc84a18279f38d79fb7b2ca Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 18:13:48 +0000
Subject: [PATCH 13/38] Fix DNSUtils dnsToIp not catching general socket.error
 exception

---
 server/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/filter.py b/server/filter.py
index b3b99e30..2669cc7f 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -605,7 +605,7 @@ class DNSUtils:
 		"""
 		try:
 			return socket.gethostbyname_ex(dns)[2]
-		except socket.gaierror:
+		except socket.error:
 			logSys.warn("Unable to find a corresponding IP address for %s"
 						% dns)
 			return list()

From dbc40d6210f1b67e18834cf9c84d822013c52897 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 18:27:40 +0000
Subject: [PATCH 14/38] Added helper scripts to carry out 2to3 and testcases on
 python3

---
 fail2ban-2to3                  | 17 +++++++++++++++++
 fail2ban-testcases-all-python3 | 18 ++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100755 fail2ban-2to3
 create mode 100755 fail2ban-testcases-all-python3

diff --git a/fail2ban-2to3 b/fail2ban-2to3
new file mode 100755
index 00000000..f0e292d6
--- /dev/null
+++ b/fail2ban-2to3
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This script carries out conversion of fail2ban to python3
+# A backup of any converted files are created with ".bak"
+# extension
+
+set -eu
+
+nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases"
+
+find . \
+    -name "*.py" \
+    -exec 2to3 -w --no-diffs $nonPyFiles {} + \
+    || echo "Fail!" >&2 && exit 1
+
+echo "Success!" >&2
+
+exit 0
diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3
new file mode 100755
index 00000000..094f2938
--- /dev/null
+++ b/fail2ban-testcases-all-python3
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Simple helper script to exercise unittests using all available
+# (under /usr/bin and /usr/local/bin python2.*)
+
+set -eu
+
+failed=
+for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg}
+do
+	[ -e "$python" ] || continue
+	echo "Testing using $python"
+	$python ./fail2ban-testcases "$@" || failed+=" $python"
+done
+
+if [ ! -z "$failed" ]; then
+	echo "E: Failed with $failed"
+	exit 1
+fi

From 418d845f9b5579bc11e9f3a0bcba8b9ea62a7fac Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 18:59:07 +0000
Subject: [PATCH 15/38] fail2ban-regex open files as utf-8 for python3

---
 fail2ban-regex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fail2ban-regex b/fail2ban-regex
index f9bc72c1..8bd2ff63 100755
--- a/fail2ban-regex
+++ b/fail2ban-regex
@@ -346,7 +346,10 @@ if __name__ == "__main__":
 
 		if fail2banRegex.logIsFile(cmd_log):
 			try:
-				hdlr = open(cmd_log)
+				if sys.version_info >= (3,):
+					hdlr = open(cmd_log, encoding='utf-8', errors='ignore')
+				else:
+					hdlr = open(cmd_log)
 				print "Use log file   : " + cmd_log
 				print
 				for line in hdlr:

From 31b173f0320f471899b9329ef941d6b92ba6fa74 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 19:13:36 +0000
Subject: [PATCH 16/38] Remove functools from filter testcases for python2.4
 compatibility

---
 testcases/filtertestcase.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py
index e8f19260..d9e859a3 100644
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@@ -27,7 +27,6 @@ import os
 import sys
 import time
 import tempfile
-import functools
 
 from server.jail import Jail
 from server.filterpoll import FilterPoll
@@ -40,7 +39,8 @@ from server.failmanager import FailManagerEmpty
 #
 
 if sys.version_info >= (3,):
-	open_ = functools.partial(open, encoding='utf-8', errors='ignore')
+	def open_(filename, mode):
+		return open(filename, mode, encoding='utf-8', errors='ignore')
 else:
 	open_ = open
 

From 3a3d07ef39d7cf8fefefb373e9acab879b59bd77 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 19:22:50 +0000
Subject: [PATCH 17/38] Undo removal of unicode prefix in
 server/datetemplate.py

---
 server/datetemplate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/datetemplate.py b/server/datetemplate.py
index 14076030..ba4477f6 100644
--- a/server/datetemplate.py
+++ b/server/datetemplate.py
@@ -164,7 +164,7 @@ class DateStrptime(DateTemplate):
 				# that the log is not from this year but from the year before
 				if time.mktime(tuple(date)) > MyTime.time():
 					logSys.debug(
-						"Correcting deduced year from %d to %d since %f > %f" %
+						u"Correcting deduced year from %d to %d since %f > %f" %
 						(date[0], date[0]-1, time.mktime(tuple(date)), MyTime.time()))
 					date[0] -= 1
 				elif date[1] == 1 and date[2] == 1:

From 78d86bc38dbd4ca3d70ece3a268506c307d7444a Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 19:24:07 +0000
Subject: [PATCH 18/38] Minor typo in fail2ban-testcases-all-python3

---
 fail2ban-testcases-all-python3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3
index 094f2938..fd3ff871 100755
--- a/fail2ban-testcases-all-python3
+++ b/fail2ban-testcases-all-python3
@@ -1,6 +1,6 @@
 #!/bin/bash
 # Simple helper script to exercise unittests using all available
-# (under /usr/bin and /usr/local/bin python2.*)
+# (under /usr/bin and /usr/local/bin python3.*)
 
 set -eu
 

From 184e0eccb6b054799f51faa63c05a1661e083e21 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 19:28:41 +0000
Subject: [PATCH 19/38] Remove redundant reassignment of variable

---
 server/filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/filter.py b/server/filter.py
index 2669cc7f..6a2cf342 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -294,7 +294,7 @@ class Filter(JailThread):
 				# Decode line to UTF-8
 				line = line.decode('utf-8')
 			except UnicodeDecodeError:
-				line = line
+				pass
 		timeMatch = self.dateDetector.matchTime(line)
 		if timeMatch:
 			# Lets split into time part and log part of the line

From 7e1819ed65954da7f56f77e8835d6eef759c9338 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 24 Feb 2013 19:34:05 +0000
Subject: [PATCH 20/38] Fix incorrect exit code from fail2ban-2to3

---
 fail2ban-2to3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban-2to3 b/fail2ban-2to3
index f0e292d6..a6a61090 100755
--- a/fail2ban-2to3
+++ b/fail2ban-2to3
@@ -10,7 +10,7 @@ nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases"
 find . \
     -name "*.py" \
     -exec 2to3 -w --no-diffs $nonPyFiles {} + \
-    || echo "Fail!" >&2 && exit 1
+    || (echo "Fail!" >&2 && exit 1)
 
 echo "Success!" >&2
 

From d23d365be2b2027ab745e2222d3ae48550256339 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Mon, 25 Feb 2013 22:45:16 +0000
Subject: [PATCH 21/38] Move handling of unicode decoding to FileContainer
 readline

Also print warning for unicode decode failure, leaving as str in python2
and ignoring erroneous characters in python3
---
 server/filter.py            | 38 +++++++++++++------------------------
 testcases/filtertestcase.py | 10 +++++-----
 2 files changed, 18 insertions(+), 30 deletions(-)

diff --git a/server/filter.py b/server/filter.py
index 6a2cf342..3f507cd1 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -289,12 +289,6 @@ class Filter(JailThread):
 	def processLine(self, line):
 		"""Split the time portion from log msg and return findFailures on them
 		"""
-		if not isinstance(line, unicode):
-			try:
-				# Decode line to UTF-8
-				line = line.decode('utf-8')
-			except UnicodeDecodeError:
-				pass
 		timeMatch = self.dateDetector.matchTime(line)
 		if timeMatch:
 			# Lets split into time part and log part of the line
@@ -485,7 +479,7 @@ class FileFilter(Filter):
 
 		while True:
 			line = container.readline()
-			if (line == "") or not self._isActive():
+			if not line or not self._isActive():
 				# The jail reached the bottom or has been stopped
 				break
 			self.processLineAndAdd(line)
@@ -521,19 +515,13 @@ class FileContainer:
 		self.__tail = tail
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
-		if sys.version_info >= (3,):
-			handler = open(filename, encoding='utf-8', errors='ignore')
-		else:
-			handler = open(filename)
+		handler = open(filename, 'rb')
 		stats = os.fstat(handler.fileno())
 		self.__ino = stats.st_ino
 		try:
 			firstLine = handler.readline()
 			# Computes the MD5 of the first line.
-			if isinstance(firstLine, unicode):
-				self.__hash = md5sum(firstLine.encode('utf-8')).digest()
-			else:
-				self.__hash = md5sum(firstLine).digest()
+			self.__hash = md5sum(firstLine).digest()
 			# Start at the beginning of file if tail mode is off.
 			if tail:
 				handler.seek(0, 2)
@@ -547,20 +535,13 @@ class FileContainer:
 		return self.__filename
 
 	def open(self):
-		if sys.version_info >= (3,):
-			self.__handler = open(
-				self.__filename, encoding='utf-8', errors='ignore')
-		else:
-			self.__handler = open(self.__filename)
+		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
 		fd = self.__handler.fileno()
 		fcntl.fcntl(fd, fcntl.F_SETFD, fd | fcntl.FD_CLOEXEC)
 		firstLine = self.__handler.readline()
 		# Computes the MD5 of the first line.
-		if isinstance(firstLine, unicode):
-			myHash = md5sum(firstLine.encode('utf-8')).digest()
-		else:
-			myHash = md5sum(firstLine).digest()
+		myHash = md5sum(firstLine).digest()
 		stats = os.fstat(self.__handler.fileno())
 		# Compare hash and inode
 		if self.__hash != myHash or self.__ino != stats.st_ino:
@@ -574,7 +555,14 @@ class FileContainer:
 	def readline(self):
 		if self.__handler == None:
 			return ""
-		return self.__handler.readline()
+		line = self.__handler.readline()
+		try:
+			line = line.decode('utf-8', 'strict')
+		except UnicodeDecodeError:
+			logSys.warn("Error decoding line to utf-8: %s" % `line`)
+			if sys.version_info >= (3,): # In python3, must be unicode
+				line = line.decode('utf-8', 'ignore')
+		return line
 
 	def close(self):
 		if not self.__handler == None:
diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py
index d9e859a3..58384449 100644
--- a/testcases/filtertestcase.py
+++ b/testcases/filtertestcase.py
@@ -508,7 +508,7 @@ class GetFailures(unittest.TestCase):
 
 	# so that they could be reused by other tests
 	FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,
-				  ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
+				  [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3)
 
 	def setUp(self):
 		"""Call before every test case."""
@@ -532,7 +532,7 @@ class GetFailures(unittest.TestCase):
 
 	def testGetFailures02(self):
 		output = ('141.3.81.106', 4, 1124013539.0,
-				  ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
+				  [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n'
 				   % m for m in 53, 54, 57, 58])
 
 		self.filter.addLogPath(GetFailures.FILENAME_02)
@@ -565,11 +565,11 @@ class GetFailures(unittest.TestCase):
 	def testGetFailuresUseDNS(self):
 		# We should still catch failures with usedns = no ;-)
 		output_yes = ('192.0.43.10', 2, 1124013539.0,
-					  ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
-					   'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
+					  [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n',
+					   u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
 
 		output_no = ('192.0.43.10', 1, 1124013539.0,
-					  ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
+					  [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:192.0.43.10 port 51332 ssh2\n'])
 
 		# Actually no exception would be raised -- it will be just set to 'no'
 		#self.assertRaises(ValueError,

From 66367876bbc780688c5755e3654bb5b13ea58390 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Wed, 27 Feb 2013 18:09:55 +0000
Subject: [PATCH 22/38] Add ability to set log encoding for jail

---
 client/beautifier.py        |  3 +++
 client/jailreader.py        |  3 +++
 common/protocol.py          |  2 ++
 config/jail.conf            |  7 ++++++
 server/filter.py            | 46 +++++++++++++++++++++++++++++++------
 server/server.py            |  6 +++++
 server/transmitter.py       |  6 +++++
 testcases/servertestcase.py |  9 +++++++-
 8 files changed, 74 insertions(+), 8 deletions(-)

diff --git a/client/beautifier.py b/client/beautifier.py
index 7e48016c..1653ff63 100644
--- a/client/beautifier.py
+++ b/client/beautifier.py
@@ -110,6 +110,9 @@ class Beautifier:
 					for path in response[:-1]:
 						msg = msg + "|- " + path + "\n"
 					msg = msg + "`- " + response[len(response)-1]
+			elif inC[2] == "logencoding":
+				msg = "Current log encoding is set to:\n"
+				msg = msg + response
 			elif inC[2] in ("ignoreip", "addignoreip", "delignoreip"):
 				if len(response) == 0:
 					msg = "No IP address/network is ignored"
diff --git a/client/jailreader.py b/client/jailreader.py
index f66dc010..be22a78f 100644
--- a/client/jailreader.py
+++ b/client/jailreader.py
@@ -61,6 +61,7 @@ class JailReader(ConfigReader):
 	def getOptions(self):
 		opts = [["bool", "enabled", "false"],
 				["string", "logpath", "/var/log/messages"],
+				["string", "logencoding", "auto"],
 				["string", "backend", "auto"],
 				["int", "maxretry", 3],
 				["int", "findtime", 600],
@@ -110,6 +111,8 @@ class JailReader(ConfigReader):
 						logSys.error("No file found for " + path)
 					for p in pathList:
 						stream.append(["set", self.__name, "addlogpath", p])
+			elif opt == "logencoding":
+				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
 			elif opt == "backend":
 				backend = self.__opts[opt]
 			elif opt == "maxretry":
diff --git a/common/protocol.py b/common/protocol.py
index 99a2fe09..99d608a8 100644
--- a/common/protocol.py
+++ b/common/protocol.py
@@ -56,6 +56,7 @@ protocol = [
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
 ["set <JAIL> addlogpath <FILE>", "adds <FILE> to the monitoring list of <JAIL>"], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
+["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
@@ -77,6 +78,7 @@ protocol = [
 ["set <JAIL> actionunban <ACT> <CMD>", "sets the unban command <CMD> of the action <ACT> for <JAIL>"], 
 ['', "JAIL INFORMATION", ""],
 ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"],
+["get <JAIL> logencoding <ENCODING>", "gets the <ENCODING> of the log files for <JAIL>"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],
diff --git a/config/jail.conf b/config/jail.conf
index a0093f68..e56023d7 100644
--- a/config/jail.conf
+++ b/config/jail.conf
@@ -55,6 +55,13 @@ backend = auto
 #        but it will be logged as info.
 usedns = warn
 
+# "logencoding" specifies the encoding of the log files handled by the jail
+#   This is used to decode the lines from the log file.
+#   Typical examples:  "ascii", "utf-8"
+#
+#   auto:   will use the system locale setting
+logencoding = auto
+
 
 # This jail corresponds to the standard configuration in Fail2ban 0.6.
 # The mail-whois action send a notification e-mail with a whois request
diff --git a/server/filter.py b/server/filter.py
index 3f507cd1..57305431 100644
--- a/server/filter.py
+++ b/server/filter.py
@@ -35,7 +35,7 @@ from datedetector import DateDetector
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
 
-import logging, re, os, fcntl, time, sys
+import logging, re, os, fcntl, time, sys, locale, codecs
 
 # Gets the instance of the logger.
 logSys = logging.getLogger("fail2ban.filter")
@@ -392,6 +392,7 @@ class FileFilter(Filter):
 		Filter.__init__(self, jail, **kwargs)
 		## The log file path.
 		self.__logPath = []
+		self.setLogEncoding("auto")
 
 	##
 	# Add a log file path
@@ -402,7 +403,7 @@ class FileFilter(Filter):
 		if self.containsLogPath(path):
 			logSys.error(path + " already exists")
 		else:
-			container = FileContainer(path, tail)
+			container = FileContainer(path, self.getLogEncoding(), tail)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@@ -451,6 +452,28 @@ class FileFilter(Filter):
 				return True
 		return False
 
+	##
+	# Set the log file encoding
+	#
+	# @param encoding the encoding used with log files
+
+	def setLogEncoding(self, encoding):
+		if encoding.lower() == "auto":
+			encoding = locale.getpreferredencoding()
+		codecs.lookup(encoding) # Raise LookupError if invalid codec
+		for log in self.getLogPath():
+			log.setEncoding(encoding)
+		self.__encoding = encoding
+		logSys.info("Set jail log file encoding to %s" % encoding)
+
+	##
+	# Get the log file encoding
+	#
+	# @return log encoding value
+
+	def getLogEncoding(self):
+		return self.__encoding
+
 	def getFileContainer(self, path):
 		for log in self.__logPath:
 			if log.getFileName() == path:
@@ -510,8 +533,9 @@ except ImportError:
 
 class FileContainer:
 
-	def __init__(self, filename, tail = False):
+	def __init__(self, filename, encoding, tail = False):
 		self.__filename = filename
+		self.setEncoding(encoding)
 		self.__tail = tail
 		self.__handler = None
 		# Try to open the file. Raises an exception if an error occured.
@@ -534,6 +558,13 @@ class FileContainer:
 	def getFileName(self):
 		return self.__filename
 
+	def setEncoding(self, encoding):
+		codecs.lookup(encoding) # Raises LookupError if invalid
+		self.__encoding = encoding
+
+	def getEncoding(self):
+		return self.__encoding
+
 	def open(self):
 		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
@@ -557,11 +588,12 @@ class FileContainer:
 			return ""
 		line = self.__handler.readline()
 		try:
-			line = line.decode('utf-8', 'strict')
+			line = line.decode(self.getEncoding(), 'strict')
 		except UnicodeDecodeError:
-			logSys.warn("Error decoding line to utf-8: %s" % `line`)
-			if sys.version_info >= (3,): # In python3, must be unicode
-				line = line.decode('utf-8', 'ignore')
+			logSys.warn("Error decoding line from '%s' with '%s': %s" %
+				(self.getFileName(), self.getEncoding(), `line`))
+			if sys.version_info >= (3,): # In python3, must be decoded
+				line = line.decode(self.getEncoding(), 'ignore')
 		return line
 
 	def close(self):
diff --git a/server/server.py b/server/server.py
index 4a086078..0db03d09 100644
--- a/server/server.py
+++ b/server/server.py
@@ -181,6 +181,12 @@ class Server:
 		return [m.getFileName()
 				for m in self.__jails.getFilter(name).getLogPath()]
 	
+	def setLogEncoding(self, name, encoding):
+		return self.__jails.getFilter(name).setLogEncoding(encoding)
+	
+	def getLogEncoding(self, name):
+		return self.__jails.getFilter(name).getLogEncoding()
+	
 	def setFindTime(self, name, value):
 		self.__jails.getFilter(name).setFindTime(value)
 	
diff --git a/server/transmitter.py b/server/transmitter.py
index a02b94a2..1c1b1e2c 100644
--- a/server/transmitter.py
+++ b/server/transmitter.py
@@ -139,6 +139,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.delLogPath(name, value)
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			value = command[2]
+			self.__server.setLogEncoding(name, value)
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "addfailregex":
 			value = command[2]
 			self.__server.addFailRegex(name, value)
@@ -234,6 +238,8 @@ class Transmitter:
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
+		elif command[1] == "logencoding":
+			return self.__server.getLogEncoding(name)
 		elif command[1] == "ignoreip":
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "failregex":
diff --git a/testcases/servertestcase.py b/testcases/servertestcase.py
index 00f56b81..f52fecca 100644
--- a/testcases/servertestcase.py
+++ b/testcases/servertestcase.py
@@ -27,7 +27,7 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import unittest, socket, time, tempfile, os
+import unittest, socket, time, tempfile, os, locale
 from server.server import Server
 
 class StartStop(unittest.TestCase):
@@ -258,6 +258,13 @@ class Transmitter(unittest.TestCase):
 		self.setGetTest("maxretry", "-2", -2, jail=self.jailName)
 		self.setGetTestNOK("maxretry", "Duck", jail=self.jailName)
 
+	def testJailLogEncoding(self):
+		self.setGetTest("logencoding", "UTF-8", jail=self.jailName)
+		self.setGetTest("logencoding", "ascii", jail=self.jailName)
+		self.setGetTest("logencoding", "auto", locale.getpreferredencoding(),
+			jail=self.jailName)
+		self.setGetTestNOK("logencoding", "Monkey", jail=self.jailName)
+
 	def testJailLogPath(self):
 		self.jailAddDelTest(
 			"logpath",

From 578d9bed1bdb965a34c32a8a73f4471ff7e3c6b9 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Wed, 27 Feb 2013 18:13:22 +0000
Subject: [PATCH 23/38] Added ability to set log file encoding with
 fail2ban-regex

---
 fail2ban-regex | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/fail2ban-regex b/fail2ban-regex
index 8bd2ff63..20a67975 100755
--- a/fail2ban-regex
+++ b/fail2ban-regex
@@ -22,7 +22,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
-import getopt, sys, time, logging, os
+import getopt, sys, time, logging, os, locale
 
 # Inserts our own modules path first in the list
 # fix for bug #343821
@@ -77,6 +77,7 @@ class Fail2banRegex:
 		self.__ignoreregex = list()
 		self.__failregex = list()
 		self.__verbose = False
+		self.encoding = locale.getpreferredencoding()
 		# Setup logging
 		logging.getLogger("fail2ban").handlers = []
 		self.__hdlr = logging.StreamHandler(Fail2banRegex.test)
@@ -110,6 +111,7 @@ class Fail2banRegex:
 		print "This tools can test regular expressions for \"fail2ban\"."
 		print
 		print "Options:"
+		print "    -e, --encoding          set the file encoding"
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print "    -v, --verbose           verbose output"
@@ -141,6 +143,8 @@ class Fail2banRegex:
 				sys.exit(0)
 			elif opt[0] in ["-v", "--verbose"]:
 				self.__verbose = True
+			elif opt[0] in ["-e", "--encoding"]:
+				self.encoding = opt[1]
 
 	#@staticmethod
 	def logIsFile(value):
@@ -318,8 +322,8 @@ if __name__ == "__main__":
 	fail2banRegex = Fail2banRegex()
 	# Reads the command line options.
 	try:
-		cmdOpts = 'hVcv'
-		cmdLongOpts = ['help', 'version', 'verbose']
+		cmdOpts = 'e:hVcv'
+		cmdLongOpts = ['encoding=', 'help', 'version', 'verbose']
 		optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts)
 	except getopt.GetoptError:
 		fail2banRegex.dispUsage()
@@ -346,13 +350,15 @@ if __name__ == "__main__":
 
 		if fail2banRegex.logIsFile(cmd_log):
 			try:
-				if sys.version_info >= (3,):
-					hdlr = open(cmd_log, encoding='utf-8', errors='ignore')
-				else:
-					hdlr = open(cmd_log)
+				hdlr = open(cmd_log, 'rb')
 				print "Use log file   : " + cmd_log
 				print
 				for line in hdlr:
+					try:
+						line = line.decode(fail2banRegex.encoding, 'strict')
+					except UnicodeDecodeError:
+						if sys.version_info >= (3,): # Python 3 must be decoded
+							line = line.decode(fail2banRegex.encoding, 'ignore')
 					fail2banRegex.testIgnoreRegex(line)
 					fail2banRegex.testRegex(line)
 			except IOError, e:

From a4a24048d4ab49620a6f98c168939272650471a7 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Wed, 27 Feb 2013 18:31:47 +0000
Subject: [PATCH 24/38] Minor tweaks to fail2ban-regex for encoding

---
 fail2ban-regex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fail2ban-regex b/fail2ban-regex
index 20a67975..f9746ce4 100755
--- a/fail2ban-regex
+++ b/fail2ban-regex
@@ -111,7 +111,8 @@ class Fail2banRegex:
 		print "This tools can test regular expressions for \"fail2ban\"."
 		print
 		print "Options:"
-		print "    -e, --encoding          set the file encoding"
+		print "    -e ENCODING, --encoding=ENCODING"
+		print "                            set the file encoding. default:system locale"
 		print "    -h, --help              display this help message"
 		print "    -V, --version           print the version"
 		print "    -v, --verbose           verbose output"
@@ -352,6 +353,7 @@ if __name__ == "__main__":
 			try:
 				hdlr = open(cmd_log, 'rb')
 				print "Use log file   : " + cmd_log
+				print "Use encoding   : " + fail2banRegex.encoding
 				print
 				for line in hdlr:
 					try:

From 8ddc9de928a812665ef774ffa1c7713d5f9ba17d Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Mar 2013 19:04:58 +0000
Subject: [PATCH 25/38] BF: Handle expected errors for python3.{0,1} when
 changing log target

---
 server/server.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/server.py b/server/server.py
index 0db03d09..de837b86 100644
--- a/server/server.py
+++ b/server/server.py
@@ -383,7 +383,8 @@ class Server:
 					handler.flush()
 					handler.close()
 				except ValueError:
-					if sys.version_info >= (2,6):
+					if (2,6) <= sys.version_info < (3,) or \
+							(3,2) <= sys.version_info:
 						raise
 					# is known to be thrown after logging was shutdown once
 					# with older Pythons -- seems to be safe to ignore there

From 21888dfe69ef03d1efe1e453ea72c1b9229e8848 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 23 Mar 2013 19:31:29 +0000
Subject: [PATCH 26/38] ENH: Add python3 versions to Travis CI config

---
 .travis.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index e655e7c1..b309255b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,13 @@ python:
   - "2.5"
   - "2.6"
   - "2.7"
+  - "3.0"
+  - "3.1"
+  - "3.2"
+  - "3.3"
 install:
   - "pip install pyinotify"
+before_script:
+    - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi
 script:
     - python ./fail2ban-testcases

From d30f6a2d66201e8d37d04fbc78262d02b84753a0 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 30 Mar 2013 21:40:42 +0000
Subject: [PATCH 27/38] add fail2ban-2to3 to MANIFEST file

---
 MANIFEST | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST b/MANIFEST
index eef145b6..785edd5e 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -7,6 +7,7 @@ fail2ban-client
 fail2ban-server
 fail2ban-testcases
 fail2ban-regex
+fail2ban-2to3
 client/configreader.py
 client/configparserinc.py
 client/jailreader.py

From 5acd035f7224f2dfa564916d969619dcba04884f Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 30 Mar 2013 21:51:22 +0000
Subject: [PATCH 28/38] TST: Remove Travis CI unsupported versions of python
 from Travis config

---
 .travis.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b309255b..b68b6c3e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,8 +5,6 @@ python:
   - "2.5"
   - "2.6"
   - "2.7"
-  - "3.0"
-  - "3.1"
   - "3.2"
   - "3.3"
 install:

From a33bf5baca961a2b13d330fb905de11802fbc50b Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Tue, 9 Apr 2013 19:40:54 +0100
Subject: [PATCH 29/38] ENH: setup.py now automatically runs 2to3 for python3.x

---
 setup.py | 47 +++++++++++++++++++++++++++++------------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/setup.py b/setup.py
index 784999a2..8aca6852 100755
--- a/setup.py
+++ b/setup.py
@@ -23,9 +23,18 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 from distutils.core import setup
+try:
+	# python 3.x
+	from distutils.command.build_py import build_py_2to3 as build_py
+	from distutils.command.build_scripts \
+		import build_scripts_2to3 as build_scripts
+except ImportError:
+	# python 2.x
+	from distutils.command.build_py import build_py
+	from distutils.command.build_scripts import build_scripts
 from common.version import version
 from os.path import isfile, join, isdir
-from sys import argv
+import sys
 from glob import glob
 
 longdesc = '''
@@ -45,6 +54,7 @@ setup(
 	url = "http://www.fail2ban.org",
 	license = "GPL",
 	platforms = "Posix",
+	cmdclass = {'build_py': build_py, 'build_scripts': build_scripts},
 	scripts =	[
 					'fail2ban-client',
 					'fail2ban-server',
@@ -100,25 +110,26 @@ for directory in elements:
 			obsoleteFiles.append(path)
 
 if obsoleteFiles:
-	print
-	print "Obsolete files from previous Fail2Ban versions were found on " \
-		  "your system."
-	print "Please delete them:"
-	print
+	sys.stdout.write("\n")
+	sys.stdout.write("Obsolete files from previous Fail2Ban versions " \
+		  "were found on your system.\n")
+	sys.stdout.write("Please delete them:\n")
+	sys.stdout.write("\n")
 	for f in obsoleteFiles:
-		print "\t" + f
-	print
+		sys.stdout.write("\t" + f)
+	sys.stdout.write("\n")
 
 if isdir("/usr/lib/fail2ban"):
-	print
-	print "Fail2ban is not installed under /usr/lib anymore. The new " \
-		  "location is under /usr/share. Please remove the directory " \
-		  "/usr/lib/fail2ban and everything under this directory."
-	print
+	sys.stdout.write("\n")
+	sys.stdout.write("Fail2ban is not installed under /usr/lib anymore. " \
+		  "The new location is under /usr/share. Please remove the " \
+		  "directory /usr/lib/fail2ban and everything under this directory.\n")
+	sys.stdout.write("\n")
 
 # Update config file
-if argv[1] == "install":
-	print
-	print "Please do not forget to update your configuration files."
-	print "They are in /etc/fail2ban/."
-	print
+if sys.argv[1] == "install":
+	sys.stdout.write("\n")
+	sys.stdout.write("Please do not forget to update your configuration "
+          "files.\n")
+	sys.stdout.write("They are in /etc/fail2ban/.\n")
+	sys.stdout.write("\n")

From d061b2b549445164ad4c3b197e482bf10f220b22 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 13 Apr 2013 16:55:22 +0100
Subject: [PATCH 30/38] TST: Fix issues in tests which assumed dictionary's
 order

---
 fail2ban/tests/clientreadertestcase.py |  4 ++-
 fail2ban/tests/filtertestcase.py       | 44 ++++++++++++++++----------
 2 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 99d9a50e..d721ef00 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -146,7 +146,9 @@ class FilterReaderTest(unittest.TestCase):
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
 
-		self.assertEquals(filterReader.convert(), output)
+		# Add sort as configreader uses dictionary and therefore order
+		# is unreliable
+		self.assertEquals(sorted(filterReader.convert()), sorted(output))
 
 class JailsReaderTest(unittest.TestCase):
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 460f2b45..6578a261 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -94,22 +94,25 @@ def _assert_equal_entries(utest, found, output, count=None):
 		# do not check if custom count (e.g. going through them twice)
 		utest.assertEqual(repr(found[3]), repr(output[3]))
 
+def _ticket_tuple(ticket):
+	"""Create a tuple for easy comparison from fail ticket
+	"""
+	attempts = ticket.getAttempt()
+	date = ticket.getTime()
+	ip = ticket.getIP()
+	matches = ticket.getMatches()
+	return (ip, attempts, date, matches)
+
 def _assert_correct_last_attempt(utest, filter_, output, count=None):
 	"""Additional helper to wrap most common test case
 
 	Test filter to contain target ticket
 	"""
 	if isinstance(filter_, DummyJail):
-		ticket = filter_.getFailTicket()
+		found = _ticket_tuple(filter_.getFailTicket())
 	else:
 		# when we are testing without jails
-		ticket = filter_.failManager.toBan()
-
-	attempts = ticket.getAttempt()
-	date = ticket.getTime()
-	ip = ticket.getIP()
-	matches = ticket.getMatches()
-	found = (ip, attempts, date, matches)
+		found = _ticket_tuple(filter_.failManager.toBan())
 
 	_assert_equal_entries(utest, found, output, count)
 
@@ -642,10 +645,14 @@ class GetFailures(unittest.TestCase):
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
 
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-
-		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+		foundList = []
+		while True:
+			try:
+				foundList.append(
+					_ticket_tuple(self.filter.failManager.toBan())[0:3])
+			except FailManagerEmpty:
+				break
+		self.assertEqual(sorted(foundList), sorted(output))
 
 	def testGetFailuresMultiLineIgnoreRegex(self):
 		output = [("192.0.43.10", 2, 1124013599.0)]
@@ -673,11 +680,14 @@ class GetFailures(unittest.TestCase):
 
 		self.filter.getFailures(GetFailures.FILENAME_MULTILINE)
 
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-		_assert_correct_last_attempt(self, self.filter, output.pop())
-
-		self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
+		foundList = []
+		while True:
+			try:
+				foundList.append(
+					_ticket_tuple(self.filter.failManager.toBan())[0:3])
+			except FailManagerEmpty:
+				break
+		self.assertEqual(sorted(foundList), sorted(output))
 
 class DNSUtilsTests(unittest.TestCase):
 

From 9241ded37663988412091b0cb2622eb408bfb61c Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 13 Apr 2013 16:58:05 +0100
Subject: [PATCH 31/38] TST: Fix up fail2ban python3 scripts

---
 fail2ban-2to3                  | 17 +++++++----------
 fail2ban-testcases-all-python3 |  2 +-
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/fail2ban-2to3 b/fail2ban-2to3
index a6a61090..2015ed5b 100755
--- a/fail2ban-2to3
+++ b/fail2ban-2to3
@@ -5,13 +5,10 @@
 
 set -eu
 
-nonPyFiles="fail2ban-client fail2ban-server fail2ban-regex fail2ban-testcases"
-
-find . \
-    -name "*.py" \
-    -exec 2to3 -w --no-diffs $nonPyFiles {} + \
-    || (echo "Fail!" >&2 && exit 1)
-
-echo "Success!" >&2
-
-exit 0
+if 2to3 -w --no-diffs bin/* fail2ban;then
+  echo "Success!" >&2
+  exit 0
+else
+  echo "Fail!" >&2
+  exit 1
+fi
diff --git a/fail2ban-testcases-all-python3 b/fail2ban-testcases-all-python3
index fd3ff871..9445f396 100755
--- a/fail2ban-testcases-all-python3
+++ b/fail2ban-testcases-all-python3
@@ -9,7 +9,7 @@ for python in /usr/{,local/}bin/python3.[0-9]{,.*}{,-dbg}
 do
 	[ -e "$python" ] || continue
 	echo "Testing using $python"
-	$python ./fail2ban-testcases "$@" || failed+=" $python"
+	$python bin/fail2ban-testcases "$@" || failed+=" $python"
 done
 
 if [ ! -z "$failed" ]; then

From 70bcb0e32ff912e440a9e9a78523b66329b3191b Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 13 Apr 2013 17:05:19 +0100
Subject: [PATCH 32/38] Add *.bak files generated by 2to3 to gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index c2e979e5..b697c3dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ htmlcov
 .coverage
 *.orig
 *.rej
+*.bak

From 36097ffc3e6bd434300dae4778e114efdb8010ea Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 14 Apr 2013 10:18:22 +0100
Subject: [PATCH 33/38] DOC: Revert setup.py messages to use print statement

---
 setup.py | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/setup.py b/setup.py
index 38392406..57b75225 100755
--- a/setup.py
+++ b/setup.py
@@ -117,26 +117,25 @@ for directory in elements:
 			obsoleteFiles.append(path)
 
 if obsoleteFiles:
-	sys.stdout.write("\n")
-	sys.stdout.write("Obsolete files from previous Fail2Ban versions " \
-		  "were found on your system.\n")
-	sys.stdout.write("Please delete them:\n")
-	sys.stdout.write("\n")
+	print("")
+	print("Obsolete files from previous Fail2Ban versions were found on "
+		  "your system.")
+	print("Please delete them:")
+	print("")
 	for f in obsoleteFiles:
-		sys.stdout.write("\t" + f)
-	sys.stdout.write("\n")
+		print("\t" + f)
+	print("")
 
 if isdir("/usr/lib/fail2ban"):
-	sys.stdout.write("\n")
-	sys.stdout.write("Fail2ban is not installed under /usr/lib anymore. " \
-		  "The new location is under /usr/share. Please remove the " \
-		  "directory /usr/lib/fail2ban and everything under this directory.\n")
-	sys.stdout.write("\n")
+	print("")
+	print("Fail2ban is not installed under /usr/lib anymore. The new "
+		  "location is under /usr/share. Please remove the directory "
+		  "/usr/lib/fail2ban and everything under this directory.")
+	print("")
 
 # Update config file
 if sys.argv[1] == "install":
-	sys.stdout.write("\n")
-	sys.stdout.write("Please do not forget to update your configuration "
-          "files.\n")
-	sys.stdout.write("They are in /etc/fail2ban/.\n")
-	sys.stdout.write("\n")
+	print("")
+	print("Please do not forget to update your configuration files.")
+	print("They are in /etc/fail2ban/.")
+	print("")

From d28788c87b4f02f5ec6ab2f7d928a58da0e06ff6 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 14 Apr 2013 10:24:44 +0100
Subject: [PATCH 34/38] TST: Revert changes for filter testcase open statement

Also merging python3 support
---
 fail2ban/tests/filtertestcase.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 6578a261..bc043373 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -22,6 +22,7 @@
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
+from __builtin__ import open as fopen
 import unittest
 import os
 import sys
@@ -44,13 +45,18 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836
 # adding a sufficiently large buffer might help to guarantee that
 # writes happen atomically.
-# Overload also for python3 to use utf-8 encoding by default
-if sys.version_info >= (3,):
-	def open_(filename, mode):
-		return open(filename, mode, 50000, encoding='utf-8', errors='ignore')
-else:
-	def open_(filename, mode):
-		return open(filename, mode, 50000)
+def open(*args):
+	"""Overload built in open so we could assure sufficiently large buffer
+
+	Explicit .flush would be needed to assure that changes leave the buffer
+	"""
+	if len(args) == 2:
+		# ~50kB buffer should be sufficient for all tests here.
+		args = args + (50000,)
+	if sys.version_info >= (3,):
+		return fopen(*args, encoding='utf-8', errors='ignore')
+	else:
+		return fopen(*args)
 
 def _killfile(f, name):
 	try:
@@ -126,7 +132,7 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line
 		# polling filter could detect the change
 		time.sleep(1)
 	if isinstance(fin, str): # pragma: no branch - only used with str in test cases
-		fin = open_(fin, 'r')
+		fin = open(fin, 'r')
 	# Skip
 	for i in xrange(skip):
 		_ = fin.readline()
@@ -141,7 +147,7 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line
 		i += 1
 	# Write: all at once and flush
 	if isinstance(fout, str):
-		fout = open_(fout, mode)
+		fout = open(fout, mode)
 	fout.write('\n'.join(lines))
 	fout.flush()
 	# to give other threads possibly some time to crunch
@@ -209,7 +215,7 @@ class LogFileMonitor(unittest.TestCase):
 		"""Call before every test case."""
 		self.filter = self.name = 'NA'
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
-		self.file = open_(self.name, 'a')
+		self.file = open(self.name, 'a')
 		self.filter = FilterPoll(None)
 		self.filter.addLogPath(self.name)
 		self.filter.setActive(True)
@@ -251,7 +257,7 @@ class LogFileMonitor(unittest.TestCase):
 		# we are not signaling as modified whenever
 		# it gets away
 		self.assertTrue(self.notModified())
-		f = open_(self.name, 'a')
+		f = open(self.name, 'a')
 		self.assertTrue(self.isModified())
 		self.assertTrue(self.notModified())
 		_sleep_4_poll()
@@ -360,7 +366,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.filter = self.name = 'NA'
 			self.name = '%s-%d' % (testclass_name, self.count)
 			MonitorFailures.count += 1 # so we have unique filenames across tests
-			self.file = open_(self.name, 'a')
+			self.file = open(self.name, 'a')
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
 			self.filter.addLogPath(self.name)
@@ -483,7 +489,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 
 			# create a bogus file in the same directory and see if that doesn't affect
-			open_(self.name + '.bak2', 'w').write('')
+			open(self.name + '.bak2', 'w').write('')
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100)
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)

From 88187fc1612300510fdcdaead289b533d0617b7b Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 14 Apr 2013 11:00:08 +0100
Subject: [PATCH 35/38] TST: Tweak python3 open statement to resolve python2.5
 SyntaxError

---
 fail2ban/tests/filtertestcase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index bc043373..fa9340ca 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -54,7 +54,7 @@ def open(*args):
 		# ~50kB buffer should be sufficient for all tests here.
 		args = args + (50000,)
 	if sys.version_info >= (3,):
-		return fopen(*args, encoding='utf-8', errors='ignore')
+		return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'})
 	else:
 		return fopen(*args)
 

From a8f2e02eea89885390b894624bd4a7d0b5618da9 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Mon, 15 Apr 2013 20:44:13 +0100
Subject: [PATCH 36/38] DOC: Revert dnsToIp error change, seperate log message
 for socket.error

---
 fail2ban/server/filter.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index b5f910f3..5d6a569c 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -664,10 +664,14 @@ class DNSUtils:
 		"""
 		try:
 			return socket.gethostbyname_ex(dns)[2]
-		except socket.error:
+		except socket.gaierror:
 			logSys.warn("Unable to find a corresponding IP address for %s"
 						% dns)
 			return list()
+		except socket.error, e:
+			logSys.warn("Socket error raised trying to resolve hostname %s: %s"
+						% (dns, e))
+			return list()
 	dnsToIp = staticmethod(dnsToIp)
 
 	#@staticmethod

From 4db77ebf41dd4f1991094d3f568e9b9ab4dfa0ab Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Mon, 15 Apr 2013 20:47:37 +0100
Subject: [PATCH 37/38] ENH: Clarify use of bytes in csocket and asyncserver
 for python3

---
 fail2ban/client/csocket.py     | 12 ++++++++----
 fail2ban/server/asyncserver.py | 12 ++++++++----
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py
index 283d6357..346c7a90 100644
--- a/fail2ban/client/csocket.py
+++ b/fail2ban/client/csocket.py
@@ -31,6 +31,13 @@ __license__ = "GPL"
 from pickle import dumps, loads, HIGHEST_PROTOCOL
 import socket, sys
 
+if sys.version_info >= (3,):
+	# b"" causes SyntaxError in python <= 2.5, so below implements equivalent
+	EMPTY_BYTES = bytes("", encoding="ascii")
+else:
+	# python 2.x, string type is equivalent to bytes.
+	EMPTY_BYTES = ""
+
 class CSocket:
 	
 	if sys.version_info >= (3,):
@@ -55,10 +62,7 @@ class CSocket:
 	
 	#@staticmethod
 	def receive(sock):
-		if sys.version_info >= (3,):
-			msg = bytes("", encoding='ascii')
-		else:
-			msg = ''
+		msg = EMPTY_BYTES
 		while msg.rfind(CSocket.END_STRING) == -1:
 			chunk = sock.recv(6)
 			if chunk == '':
diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py
index 1f46d936..9d506a66 100644
--- a/fail2ban/server/asyncserver.py
+++ b/fail2ban/server/asyncserver.py
@@ -35,6 +35,13 @@ from fail2ban import helpers
 # Gets the instance of the logger.
 logSys = logging.getLogger(__name__)
 
+if sys.version_info >= (3,):
+	# b"" causes SyntaxError in python <= 2.5, so below implements equivalent
+	EMPTY_BYTES = bytes("", encoding="ascii")
+else:
+	# python 2.x, string type is equivalent to bytes.
+	EMPTY_BYTES = ""
+
 ##
 # Request handler class.
 #
@@ -66,10 +73,7 @@ class RequestHandler(asynchat.async_chat):
 
 	def found_terminator(self):
 		# Joins the buffer items.
-		if sys.version_info >= (3,):
-			message = loads(bytes("", encoding="ascii").join(self.__buffer))
-		else:
-			message = loads("".join(self.__buffer))
+		message = loads(EMPTY_BYTES.join(self.__buffer))
 		# Gives the message to the transmitter.
 		message = self.__transmitter.proceed(message)
 		# Serializes the response.

From f14cb7302a7682ffc9bb491640807fed3f3e5801 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Mon, 15 Apr 2013 21:00:45 +0100
Subject: [PATCH 38/38] DOC: Add python3 to requirements

---
 README | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README b/README
index 4a6460d5..e8c0dcd2 100644
--- a/README
+++ b/README
@@ -19,7 +19,7 @@ Installation:
 -------------
 
 Required:
-   >=python-2.3 (http://www.python.org)
+   >=python-2.3 or >=python-3.0 (http://www.python.org)
 
 Optional:
    pyinotify: