Merge branch '0.10' into patch-1

.travis.yml

@@ -11,6 +11,7 @@ python:
   - 3.3
   - 3.4
   - 3.5
+  - 3.6
   - pypy3
 before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi

config/filter.d/postfix-rbl.conf

@@ -12,7 +12,7 @@ before = common.conf
 
 _daemon = postfix(-\w+)?/smtpd
 
-failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$
+failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[<HOST>\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
 
 ignoreregex =
 

fail2ban/server/actions.py

@@ -150,7 +150,7 @@ class Actions(JailThread, Mapping):
 				# reload actions after all parameters set via stream:
 				for name, initOpts in self._reload_actions.iteritems():
 					if name in self._actions:
-						self._actions[name].reload(**initOpts if initOpts else {})
+						self._actions[name].reload(**(initOpts if initOpts else {}))
 				# remove obsolete actions (untouched by reload process):
 				delacts = OrderedDict((name, action) for name, action in self._actions.iteritems()
 					if name not in self._reload_actions)

fail2ban/server/datedetector.py

@@ -284,7 +284,7 @@ class DateDetector(object):
 			if preMatch is not None:
 				# get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}:
 				template = _getAnchoredTemplate(template,
-					wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch))
+					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: s, preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
 

fail2ban/tests/action_d/test_smtp.py

@@ -30,18 +30,22 @@ else:
 
 from ..dummyjail import DummyJail
 
-from ..utils import CONFIG_DIR, asyncserver
-
+from ..utils import CONFIG_DIR, asyncserver, Utils, uni_decode
 
 class TestSMTPServer(smtpd.SMTPServer):
 
-	def process_message(self, peer, mailfrom, rcpttos, data):
+	def __init__(self, *args):
+		smtpd.SMTPServer.__init__(self, *args)
+		self.ready = False
+
+	def process_message(self, peer, mailfrom, rcpttos, data, **kwargs):
 		self.peer = peer
 		self.mailfrom = mailfrom
 		self.rcpttos = rcpttos
 		self.org_data = data
-		# replace new line (with tab or space) for possible mime translations (word wrap):
-		self.data = re.sub(r"\n[\t ]", " ", data)
+		# replace new line (with tab or space) for possible mime translations (word wrap),
+		self.data = re.sub(r"\n[\t ]", " ", uni_decode(data))
+		self.ready = True
 
 
 class SMTPActionTest(unittest.TestCase):
@@ -78,8 +82,14 @@ class SMTPActionTest(unittest.TestCase):
 		self._active = False
 		self._loop_thread.join()
 
+	def _exec_and_wait(self, doaction, timeout=3, short=False):
+		if short: timeout /= 25
+		self.smtpd.ready = False
+		doaction()
+		Utils.wait_for(lambda: self.smtpd.ready, timeout)
+
 	def testStart(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -87,7 +97,7 @@ class SMTPActionTest(unittest.TestCase):
 			in self.smtpd.data)
 
 	def testStop(self):
-		self.action.stop()
+		self._exec_and_wait(self.action.stop)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		self.assertTrue(
@@ -105,7 +115,7 @@ class SMTPActionTest(unittest.TestCase):
 		if restored:
 			aInfo['restored'] = 1
 
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo), short=restored)
 		if restored: # no mail, should raises attribute error:
 			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
 			return
@@ -118,15 +128,15 @@ class SMTPActionTest(unittest.TestCase):
 			"%i attempts" % aInfo['failures'], self.smtpd.data)
 
 		self.action.matches = "matches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['matches'], self.smtpd.data)
 
 		self.action.matches = "ipjailmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipjailmatches'], self.smtpd.data)
 
 		self.action.matches = "ipmatches"
-		self.action.ban(aInfo)
+		self._exec_and_wait(lambda: self.action.ban(aInfo))
 		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
 	
 	def testBan(self):
@@ -136,14 +146,14 @@ class SMTPActionTest(unittest.TestCase):
 		self._testBan(restored=True)
 
 	def testOptions(self):
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 
 		self.action.fromname = "Test"
 		self.action.fromaddr = "test@example.com"
 		self.action.toaddr = "test@example.com, test2@example.com"
-		self.action.start()
+		self._exec_and_wait(self.action.start)
 		self.assertEqual(self.smtpd.mailfrom, "test@example.com")
 		self.assertTrue("From: %s <%s>" %
 			(self.action.fromname, self.action.fromaddr) in self.smtpd.data)

fail2ban/tests/files/logs/postfix-rbl

@@ -3,3 +3,6 @@ Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.examp
 
 # failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" }
 Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from=<spammer@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badguy.example.com>
+
+# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" }
+Feb  7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from=<spammer@example.com> to=<goodguy@example.com> proto=SMTP helo=<WIN-5N8GBBS0R5I>

fail2ban/tests/servertestcase.py

@@ -992,9 +992,10 @@ class LoggingTests(LogCaptureTestCase):
 			badThread = _BadThread()
 			badThread.start()
 			badThread.join()
-			self.assertLogged("Unhandled exception")
+			self.assertTrue( Utils.wait_for( lambda: len(x) and self._is_logged("Unhandled exception"), 3) )
 		finally:
 			sys.__excepthook__ = prev_exchook
+		self.assertLogged("Unhandled exception")
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)
 
@@ -1646,7 +1647,8 @@ class ServerConfigReaderTests(LogCaptureTestCase):
 				r' echo mail \1 ) | cat', realCmd)
 			# replace abuse retrieving (possible no-network), just replace first occurrence of 'dig...':
 			realCmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+',
-				'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd, 1)
+				lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"',
+					realCmd, 1)
 			# execute action:
 			return _actions.CommandAction.executeCmd(realCmd, timeout=timeout)
 

fail2ban/tests/utils.py

@@ -37,7 +37,7 @@ import unittest
 from cStringIO import StringIO
 from functools import wraps
 
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode
 from ..server.ipdns import DNSUtils
 from ..server.mytime import MyTime
 from ..server.utils import Utils