Merge branch 'py-3.6-compat' into 0.10

.travis.yml

@@ -11,6 +11,7 @@ python:
   - 3.3
   - 3.4
   - 3.5
+  - 3.6
   - pypy3
 before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi

fail2ban/server/actions.py

@@ -150,7 +150,7 @@ class Actions(JailThread, Mapping):
 				# reload actions after all parameters set via stream:
 				for name, initOpts in self._reload_actions.iteritems():
 					if name in self._actions:
-						self._actions[name].reload(**initOpts if initOpts else {})
+						self._actions[name].reload(**(initOpts if initOpts else {}))
 				# remove obsolete actions (untouched by reload process):
 				delacts = OrderedDict((name, action) for name, action in self._actions.iteritems()
 					if name not in self._reload_actions)

fail2ban/server/datedetector.py

@@ -284,7 +284,7 @@ class DateDetector(object):
 			if preMatch is not None:
 				# get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}:
 				template = _getAnchoredTemplate(template,
-					wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch))
+					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: s, preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
 

fail2ban/tests/action_d/test_smtp.py

@@ -30,18 +30,22 @@ else:
 
 from ..dummyjail import DummyJail
 
-from ..utils import CONFIG_DIR, asyncserver
-
+from ..utils import CONFIG_DIR, asyncserver, Utils, uni_decode
 
 class TestSMTPServer(smtpd.SMTPServer):
 
-	def process_message(self, peer, mailfrom, rcpttos, data):
+	def __init__(self, *args):
+		smtpd.SMTPServer.__init__(self, *args)
+		self.ready = False
+
+	def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
 		self.peer = peer
 		self.mailfrom = mailfrom
 		self.rcpttos = rcpttos
 		self.org_data = data
-		# replace new line (with tab or space) for possible mime translations (word wrap):
-		self.data = re.sub(r"\n[\t ]", " ", data)
+		# replace new line (with tab or space) for possible mime translations (word wrap),
+		self.data = re.sub(r"\n[\t ]", " ", uni_decode(data))
+		self.ready = True
 
 
 class SMTPActionTest(unittest.TestCase):
@@ -78,8 +82,14 @@ class SMTPActionTest(unittest.TestCase):
 		self._active = False
 		self._loop_thread.join()
 
+	def _exec_and_wait(self, doaction, timeout=3, short=False):
+		if short: timeout /= 25
+		self.smtpd.ready = False
+		doaction()
+		Utils.wait_for(lambda: self.smtpd.ready, timeout)
+
 	def testStart(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -87,7 +97,7 @@ class SMTPActionTest(unittest.TestCase):
 			in self.smtpd.data)
 
 	def testStop(self):
-		self.action.stop()
+		self._exec_and_wait(self.action.stop)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -105,7 +115,7 @@ class SMTPActionTest(unittest.TestCase):
 		if restored:
 			aInfo['restored'] = 1
 
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo), short=restored)
 		if restored: # no mail, should raises attribute error:
 			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
 			return
@@ -118,15 +128,15 @@ class SMTPActionTest(unittest.TestCase):
 			"%i attempts" % aInfo['failures'], self.smtpd.data)
 
 		self.action.matches = "matches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['matches'], self.smtpd.data)
 
 		self.action.matches = "ipjailmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipjailmatches'], self.smtpd.data)
 
 		self.action.matches = "ipmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
 	
 	def testBan(self):
@@ -136,14 +146,14 @@ class SMTPActionTest(unittest.TestCase):
 		self._testBan(restored=True)
 
 	def testOptions(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 
 		self.action.fromname = "Test"
 		self.action.fromaddr = "test@example.com"
 		self.action.toaddr = "test@example.com, test2@example.com"
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "test@example.com")
 		self.assertTrue("From: %s <%s>" %
 			(self.action.fromname, self.action.fromaddr) in self.smtpd.data)

fail2ban/tests/servertestcase.py

@@ -992,9 +992,10 @@ class LoggingTests(LogCaptureTestCase):
 			badThread = _BadThread()
 			badThread.start()
 			badThread.join()
-			self.assertLogged("Unhandled exception")
+			self.assertTrue( Utils.wait_for( lambda: len(x) and self._is_logged("Unhandled exception"), 3) )
 		finally:
 			sys.__excepthook__ = prev_exchook
+		self.assertLogged("Unhandled exception")
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
 
@@ -1646,7 +1647,7 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				r' echo mail \1 ) | cat', realCmd)
 			# replace abuse retrieving (possible no-network):
 			realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+',
-				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
+				lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd)
 			# execute action:
 			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
 

fail2ban/tests/utils.py

@@ -37,7 +37,7 @@ import unittest
 from cStringIO import StringIO
 from functools import wraps
 
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode
 from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils