From 63f791688608bf116b3f4a1cca1f479de8b98ef9 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 6 Mar 2016 15:46:52 +0100 Subject: [PATCH 1/3] fix test cases by testing with multi-threaded execution (wait for threaded execution done) (cherry picked from commit 1ec6782f321bf0fca43a2a598cc33f70b39e7f35) # Conflicts: # fail2ban/tests/observertestcase.py (not yet available in 0.10) --- fail2ban/tests/action_d/test_smtp.py | 29 ++++++++++++++++++---------- fail2ban/tests/servertestcase.py | 3 ++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 1dba454c..0aeacbae 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -30,11 +30,14 @@ else: from ..dummyjail import DummyJail -from ..utils import CONFIG_DIR, asyncserver - +from ..utils import CONFIG_DIR, asyncserver, Utils class TestSMTPServer(smtpd.SMTPServer): + def __init__(self, *args): + smtpd.SMTPServer.__init__(self, *args) + self.ready = False + def process_message(self, peer, mailfrom, rcpttos, data): self.peer = peer self.mailfrom = mailfrom @@ -42,6 +45,7 @@ class TestSMTPServer(smtpd.SMTPServer): self.org_data = data # replace new line (with tab or space) for possible mime translations (word wrap): self.data = re.sub(r"\n[\t ]", " ", data) + self.ready = True class SMTPActionTest(unittest.TestCase): @@ -78,8 +82,13 @@ class SMTPActionTest(unittest.TestCase): self._active = False self._loop_thread.join() + def _exec_and_wait(self, doaction): + self.smtpd.ready = False + doaction() + Utils.wait_for(lambda: self.smtpd.ready, 3) + def testStart(self): - self.action.start() + self._exec_and_wait(self.action.start) self.assertEqual(self.smtpd.mailfrom, "fail2ban") self.assertEqual(self.smtpd.rcpttos, ["root"]) self.assertTrue( @@ -87,7 +96,7 @@ class SMTPActionTest(unittest.TestCase): in self.smtpd.data) def testStop(self): - self.action.stop() + self._exec_and_wait(self.action.stop) self.assertEqual(self.smtpd.mailfrom, "fail2ban") self.assertEqual(self.smtpd.rcpttos, ["root"]) self.assertTrue( @@ -105,7 +114,7 @@ class SMTPActionTest(unittest.TestCase): if restored: aInfo['restored'] = 1 - self.action.ban(aInfo) + self._exec_and_wait(lambda: self.action.ban(aInfo)) if restored: # no mail, should raises attribute error: self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom) return @@ -118,15 +127,15 @@ class SMTPActionTest(unittest.TestCase): "%i attempts" % aInfo['failures'], self.smtpd.data) self.action.matches = "matches" - self.action.ban(aInfo) + self._exec_and_wait(lambda: self.action.ban(aInfo)) self.assertIn(aInfo['matches'], self.smtpd.data) self.action.matches = "ipjailmatches" - self.action.ban(aInfo) + self._exec_and_wait(lambda: self.action.ban(aInfo)) self.assertIn(aInfo['ipjailmatches'], self.smtpd.data) self.action.matches = "ipmatches" - self.action.ban(aInfo) + self._exec_and_wait(lambda: self.action.ban(aInfo)) self.assertIn(aInfo['ipmatches'], self.smtpd.data) def testBan(self): @@ -136,14 +145,14 @@ class SMTPActionTest(unittest.TestCase): self._testBan(restored=True) def testOptions(self): - self.action.start() + self._exec_and_wait(self.action.start) self.assertEqual(self.smtpd.mailfrom, "fail2ban") self.assertEqual(self.smtpd.rcpttos, ["root"]) self.action.fromname = "Test" self.action.fromaddr = "test@example.com" self.action.toaddr = "test@example.com, test2@example.com" - self.action.start() + self._exec_and_wait(self.action.start) self.assertEqual(self.smtpd.mailfrom, "test@example.com") self.assertTrue("From: %s <%s>" % (self.action.fromname, self.action.fromaddr) in self.smtpd.data) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index aada699c..9423ee12 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -992,9 +992,10 @@ class LoggingTests(LogCaptureTestCase): badThread = _BadThread() badThread.start() badThread.join() - self.assertLogged("Unhandled exception") + self.assertTrue( Utils.wait_for( lambda: len(x) and self._is_logged("Unhandled exception"), 3) ) finally: sys.__excepthook__ = prev_exchook + self.assertLogged("Unhandled exception") self.assertEqual(len(x), 1) self.assertEqual(x[0][0], RuntimeError) From 01db0b502854f23ae9bd23906ccc0e48b80f0465 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 21 Jan 2017 17:54:01 +0100 Subject: [PATCH 2/3] small performance fix after merge with 0.10 (cherry picked from commit 8e2711681cf0f65ca5d3ef6831246ec5d4973c21) --- fail2ban/tests/action_d/test_smtp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 0aeacbae..8334f948 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -82,10 +82,11 @@ class SMTPActionTest(unittest.TestCase): self._active = False self._loop_thread.join() - def _exec_and_wait(self, doaction): + def _exec_and_wait(self, doaction, timeout=3, short=False): + if short: timeout /= 25 self.smtpd.ready = False doaction() - Utils.wait_for(lambda: self.smtpd.ready, 3) + Utils.wait_for(lambda: self.smtpd.ready, timeout) def testStart(self): self._exec_and_wait(self.action.start) @@ -114,7 +115,7 @@ class SMTPActionTest(unittest.TestCase): if restored: aInfo['restored'] = 1 - self._exec_and_wait(lambda: self.action.ban(aInfo)) + self._exec_and_wait(lambda: self.action.ban(aInfo), short=restored) if restored: # no mail, should raises attribute error: self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom) return From cf53a834f7b1ed623e5ff81cb6a4ffedcc1d48e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Feb 2017 18:46:00 +0100 Subject: [PATCH 3/3] python-3.6 compatibility: - dynamical string replacement within call of regexp.sub fixed with lambda-replacement (otherwise "sre_constants.error: bad escape \s at position"); - optional arguments (3.6 has more agrs by calling of SMTPServer.process_message); - implicit convert byte to string, because python3.6 fails on binary data (test_smtp). --- fail2ban/server/datedetector.py | 2 +- fail2ban/tests/action_d/test_smtp.py | 8 ++++---- fail2ban/tests/servertestcase.py | 2 +- fail2ban/tests/utils.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index cccbf71a..dd5d198f 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -284,7 +284,7 @@ class DateDetector(object): if preMatch is not None: # get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}: template = _getAnchoredTemplate(template, - wrap=lambda s: RE_DATE_PREMATCH.sub(s, preMatch)) + wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: s, preMatch)) # append date detector template (ignore duplicate if some was added before default): self._appendTemplate(template, ignoreDup=ignoreDup) diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 8334f948..bf8e83a2 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -30,7 +30,7 @@ else: from ..dummyjail import DummyJail -from ..utils import CONFIG_DIR, asyncserver, Utils +from ..utils import CONFIG_DIR, asyncserver, Utils, uni_decode class TestSMTPServer(smtpd.SMTPServer): @@ -38,13 +38,13 @@ class TestSMTPServer(smtpd.SMTPServer): smtpd.SMTPServer.__init__(self, *args) self.ready = False - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): self.peer = peer self.mailfrom = mailfrom self.rcpttos = rcpttos self.org_data = data - # replace new line (with tab or space) for possible mime translations (word wrap): - self.data = re.sub(r"\n[\t ]", " ", data) + # replace new line (with tab or space) for possible mime translations (word wrap), + self.data = re.sub(r"\n[\t ]", " ", uni_decode(data)) self.ready = True diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 9423ee12..2f1f5451 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1647,7 +1647,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): r' echo mail \1 ) | cat', realCmd) # replace abuse retrieving (possible no-network): realCmd = re.sub(r'[^\n]+\bADDRESSES=\$\(dig\s[^\n]+', - 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd) + lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', realCmd) # execute action: return _actions.CommandAction.executeCmd(realCmd, timeout=timeout) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index ae543e3d..78a42d09 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -37,7 +37,7 @@ import unittest from cStringIO import StringIO from functools import wraps -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode from ..server.ipdns import DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils