From cee09e29eccdfe7546169bd0c50e58be0a08bc07 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 15 Jul 2016 03:19:45 -0400 Subject: [PATCH 01/60] BF: do not rely on long relative path to upstairs config - symlink dereferenced copied during install (#1485) * BF: do not rely on long relative path to upstairs config - symlink common.conf * Added missing files to MANIFEST --- MANIFEST | 6 ++++++ fail2ban/tests/config/filter.d/common.conf | 1 + fail2ban/tests/config/filter.d/zzz-generic-example.conf | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 120000 fail2ban/tests/config/filter.d/common.conf diff --git a/MANIFEST b/MANIFEST index bf0ae4c9..c27878e8 100644 --- a/MANIFEST +++ b/MANIFEST @@ -16,6 +16,8 @@ config/action.d/firewallcmd-allports.conf config/action.d/firewallcmd-ipset.conf config/action.d/firewallcmd-multiport.conf config/action.d/firewallcmd-new.conf +config/action.d/firewallcmd-rich-logging.conf +config/action.d/firewallcmd-rich-rules.conf config/action.d/hostsdeny.conf config/action.d/ipfilter.conf config/action.d/ipfw.conf @@ -124,6 +126,7 @@ config/filter.d/sendmail-auth.conf config/filter.d/sendmail-reject.conf config/filter.d/sendmail-spam.conf config/filter.d/sieve.conf +config/filter.d/slapd.conf config/filter.d/sogo-auth.conf config/filter.d/solid-pop3d.conf config/filter.d/squid.conf @@ -205,6 +208,7 @@ fail2ban/tests/config/fail2ban.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local +fail2ban/tests/config/filter.d/zzz-generic-example.conf fail2ban/tests/config/jail.conf fail2ban/tests/config/paths-common.conf fail2ban/tests/config/paths-debian.conf @@ -306,6 +310,7 @@ fail2ban/tests/files/logs/sendmail-auth fail2ban/tests/files/logs/sendmail-reject fail2ban/tests/files/logs/sendmail-spam fail2ban/tests/files/logs/sieve +fail2ban/tests/files/logs/slapd fail2ban/tests/files/logs/sogo-auth fail2ban/tests/files/logs/solid-pop3d fail2ban/tests/files/logs/squid @@ -320,6 +325,7 @@ fail2ban/tests/files/logs/vsftpd fail2ban/tests/files/logs/webmin-auth fail2ban/tests/files/logs/wuftpd fail2ban/tests/files/logs/xinetd-fail +fail2ban/tests/files/logs/zzz-generic-example fail2ban/tests/files/testcase01.log fail2ban/tests/files/testcase02.log fail2ban/tests/files/testcase03.log diff --git a/fail2ban/tests/config/filter.d/common.conf b/fail2ban/tests/config/filter.d/common.conf new file mode 120000 index 00000000..83e92474 --- /dev/null +++ b/fail2ban/tests/config/filter.d/common.conf @@ -0,0 +1 @@ +../../../../config/filter.d/common.conf \ No newline at end of file diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf index a59ccb1e..e2ae91b0 100644 --- a/fail2ban/tests/config/filter.d/zzz-generic-example.conf +++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf @@ -6,8 +6,9 @@ [INCLUDES] # Read common prefixes. If any customizations available -- read them from -# common.local -before = ../../../../config/filter.d/common.conf +# common.local. common.conf is a symlink to the original common.conf and +# should be copied (dereferenced) during installation +before = common.conf [Definition] From f73746d846587c3d450b18af60620b7f09471feb Mon Sep 17 00:00:00 2001 From: rhardy613 Date: Sun, 31 Jul 2016 13:50:52 -0400 Subject: [PATCH 02/60] Fix ASSP filter to work with current release of ASSP ASSP V1 development stopped at the end of 2014 and it is now deprecated. All users were urged to upgrade to ASSP V2 which is still actively developed. For some reason fail2ban 0.9.5 (and trunk) still have code which only understands ASSP V1 logs. This means the filter ignores brute force attacks against ASSP. --- config/filter.d/assp.conf | 22 +++++++++++----------- fail2ban/tests/files/logs/assp | 33 ++++++++------------------------- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 2aa8958c..0bfba6dc 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -1,24 +1,24 @@ -# Fail2Ban filter for Anti-Spam SMTP Proxy Server also known as ASSP +# Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP) Version 2.5.1 (or later) # -# Honmepage: http://www.magicvillage.de/~Fritz_Borgstedt/assp/0003D91C-8000001C/ -# ProjektSite: http://sourceforge.net/projects/assp/?source=directory +# Homepage: http://sourceforge.net/projects/assp/ +# ProjectSite: http://sourceforge.net/projects/assp/?source=directory # # [Definition] -__assp_actions = (?:dropping|refusing) - -failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ - ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ - ^ Blocking - too much AUTH errors \(\d{,3}\);$ +failregex = \<\S+@\S+\.\S+\> to: \S+@\S+\.\S+ relay attempt blocked for: \S+$ + \[SMTP Error\] 535 5\.7\.8 Error: authentication failed.*$ ignoreregex = # DEV Notes: # -# Examples: Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); -# Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded +# Examples: +# Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org +# Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: # # Author: Enrico Labedzki (enrico.labedzki@deiwos.de) +# Updated: Robert Hardy (rhardy@webcon.ca) diff --git a/fail2ban/tests/files/logs/assp b/fail2ban/tests/files/logs/assp index 2c658eb9..71c28221 100644 --- a/fail2ban/tests/files/logs/assp +++ b/fail2ban/tests/files/logs/assp @@ -1,25 +1,8 @@ -# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } -Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } -Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# failJSON: { "time": "2013-04-07T07:10:37", "match": true , "host": "68.171.223.68" } -Apr-07-13 07:10:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# failJSON: { "time": "2013-04-07T07:12:37", "match": true , "host": "68.171.223.68" } -Apr-07-13 07:12:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# failJSON: { "time": "2013-04-07T07:14:36", "match": true , "host": "68.171.223.68" } -Apr-07-13 07:14:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; -# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (8); -# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (9); -# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (10); -# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; -# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; -# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; -# failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" } -Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; - +# failJSON: { "time": "2016-07-29T16:49:52", "match": true , "host": "0.0.0.0" } +Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org +# failJSON: { "time": "2016-07-30T17:07:25", "match": true , "host": "0.0.0.0" } +Jul-30-16 17:07:25 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# failJSON: { "time": "2016-07-30T17:11:05", "match": true , "host": "0.0.0.0" } +Jul-30-16 17:11:05 m1-13060-05386 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# failJSON: { "time": "2016-07-31T06:45:59", "match": true , "host": "0.0.0.0" } +Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: From 3e330604c7db446ce74eb4225b122c264e562be8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 Jul 2016 19:12:33 +0200 Subject: [PATCH 03/60] several test cases rewritten using new assertIn, assertNotIn (better as own from unittest, because support generators beautifying, etc.) + new forward compatibility method assertRaisesRegexp; + methods assertIn, assertNotIn, assertRaisesRegexp are test covered now; + easy-fix for distributions compatible test cases (e.g. fedora default backend is 'systemd'), (closes gh-1353, closes gh-1490) cherry picked from 9d56079756ca6aef901a69041786ab137d0b5ae3 (0.10 branch) --- fail2ban/tests/action_d/test_badips.py | 2 +- fail2ban/tests/action_d/test_smtp.py | 8 +-- fail2ban/tests/actionstestcase.py | 6 +- fail2ban/tests/clientreadertestcase.py | 18 +++--- fail2ban/tests/databasetestcase.py | 21 ++++++- fail2ban/tests/misctestcase.py | 87 ++++++++++++++++++++++++-- fail2ban/tests/servertestcase.py | 6 +- fail2ban/tests/utils.py | 50 ++++++++++++--- 8 files changed, 164 insertions(+), 34 deletions(-) diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index 3f71b7a3..b0f8b3c3 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -49,7 +49,7 @@ if sys.version_info >= (2,7): def testCategory(self): categories = self.action.getCategories() - self.assertTrue("ssh" in categories) + self.assertIn("ssh", categories) self.assertTrue(len(categories) >= 10) self.assertRaises( diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 35ac2393..b8328743 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -101,21 +101,21 @@ class SMTPActionTest(unittest.TestCase): self.assertEqual(self.smtpd.rcpttos, ["root"]) subject = "Subject: [Fail2Ban] %s: banned %s" % ( self.jail.name, aInfo['ip']) - self.assertTrue(subject in self.smtpd.data.replace("\n", "")) + self.assertIn(subject, self.smtpd.data.replace("\n", "")) self.assertTrue( "%i attempts" % aInfo['failures'] in self.smtpd.data) self.action.matches = "matches" self.action.ban(aInfo) - self.assertTrue(aInfo['matches'] in self.smtpd.data) + self.assertIn(aInfo['matches'], self.smtpd.data) self.action.matches = "ipjailmatches" self.action.ban(aInfo) - self.assertTrue(aInfo['ipjailmatches'] in self.smtpd.data) + self.assertIn(aInfo['ipjailmatches'], self.smtpd.data) self.action.matches = "ipmatches" self.action.ban(aInfo) - self.assertTrue(aInfo['ipmatches'] in self.smtpd.data) + self.assertIn(aInfo['ipmatches'], self.smtpd.data) def testOptions(self): self.action.start() diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index 0ceb35d5..3b9d2d01 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -65,12 +65,12 @@ class ExecuteActions(LogCaptureTestCase): def testActionsManipulation(self): self.__actions.add('test') self.assertTrue(self.__actions['test']) - self.assertTrue('test' in self.__actions) - self.assertFalse('nonexistant action' in self.__actions) + self.assertIn('test', self.__actions) + self.assertNotIn('nonexistant action', self.__actions) self.__actions.add('test1') del self.__actions['test'] del self.__actions['test1'] - self.assertFalse('test' in self.__actions) + self.assertNotIn('test', self.__actions) self.assertEqual(len(self.__actions), 0) self.__actions.setBanTime(127) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0a3734e5..ee362e3d 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -525,12 +525,12 @@ class JailsReaderTest(LogCaptureTestCase): self.assertTrue(actionReader.read()) actionReader.getOptions({}) # populate _opts if not actionName.endswith('-common'): - self.assertTrue('Definition' in actionReader.sections(), + self.assertIn('Definition', actionReader.sections(), msg="Action file %r is lacking [Definition] section" % actionConfig) # all must have some actionban defined self.assertTrue(actionReader._opts.get('actionban', '').strip(), msg="Action file %r is lacking actionban" % actionConfig) - self.assertTrue('Init' in actionReader.sections(), + self.assertIn('Init', actionReader.sections(), msg="Action file %r is lacking [Init] section" % actionConfig) def testReadStockJailConf(self): @@ -582,7 +582,7 @@ class JailsReaderTest(LogCaptureTestCase): self.assertTrue(len(actName)) self.assertTrue(isinstance(actOpt, dict)) if actName == 'iptables-multiport': - self.assertTrue('port' in actOpt) + self.assertIn('port', actOpt) actionReader = ActionReader( actName, jail, {}, basedir=CONFIG_DIR) @@ -632,11 +632,13 @@ class JailsReaderTest(LogCaptureTestCase): # and we know even some of them by heart for j in ['sshd', 'recidive']: - # by default we have 'auto' backend ATM - self.assertTrue(['add', j, 'auto'] in comm_commands) + # by default we have 'auto' backend ATM, but some distributions can overwrite it, + # (e.g. fedora default is 'systemd') therefore let check it without backend... + self.assertIn(['add', j], + (cmd[:2] for cmd in comm_commands if len(cmd) == 3 and cmd[0] == 'add')) # and warn on useDNS - self.assertTrue(['set', j, 'usedns', 'warn'] in comm_commands) - self.assertTrue(['start', j] in comm_commands) + self.assertIn(['set', j, 'usedns', 'warn'], comm_commands) + self.assertIn(['start', j], comm_commands) # last commands should be the 'start' commands self.assertEqual(comm_commands[-1][0], 'start') @@ -655,7 +657,7 @@ class JailsReaderTest(LogCaptureTestCase): action_name = action.getName() if '' in str(commands): # Verify that it is among cInfo - self.assertTrue('blocktype' in action._initOpts) + self.assertIn('blocktype', action._initOpts) # Verify that we have a call to set it up blocktype_present = False target_command = ['set', jail_name, 'action', action_name, 'blocktype'] diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 3d156eda..e934ba45 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -123,7 +123,7 @@ class DatabaseTest(LogCaptureTestCase): self.db.addLog(self.jail, self.fileContainer) - self.assertTrue(filename in self.db.getLogPaths(self.jail)) + self.assertIn(filename, self.db.getLogPaths(self.jail)) os.remove(filename) def testUpdateLog(self): @@ -318,6 +318,25 @@ class DatabaseTest(LogCaptureTestCase): actions._Actions__checkBan() self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) + def testDelAndAddJail(self): + self.testAddJail() # Add jail + # Delete jail (just disabled it): + self.db.delJail(self.jail) + jails = self.db.getJailNames() + self.assertIn(len(jails) == 1 and self.jail.name, jails) + jails = self.db.getJailNames(enabled=False) + self.assertIn(len(jails) == 1 and self.jail.name, jails) + jails = self.db.getJailNames(enabled=True) + self.assertTrue(len(jails) == 0) + # Add it again - should just enable it: + self.db.addJail(self.jail) + jails = self.db.getJailNames() + self.assertIn(len(jails) == 1 and self.jail.name, jails) + jails = self.db.getJailNames(enabled=True) + self.assertIn(len(jails) == 1 and self.jail.name, jails) + jails = self.db.getJailNames(enabled=False) + self.assertTrue(len(jails) == 0) + def testPurge(self): if Fail2BanDb is None: # pragma: no cover return diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 48074d53..5f7d853d 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -23,6 +23,7 @@ __license__ = "GPL" import logging import os +import re import sys import unittest import tempfile @@ -32,6 +33,8 @@ import datetime from glob import glob from StringIO import StringIO +from utils import LogCaptureTestCase, logSys as DefLogSys + from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger from ..helpers import splitwords from ..server.datetemplate import DatePatternRegex @@ -130,7 +133,7 @@ class SetupTest(unittest.TestCase): % (sys.executable, self.setup)) -class TestsUtilsTest(unittest.TestCase): +class TestsUtilsTest(LogCaptureTestCase): def testmbasename(self): self.assertEqual(mbasename("sample.py"), 'sample') @@ -165,12 +168,88 @@ class TestsUtilsTest(unittest.TestCase): if not ('fail2ban-testcases' in s): # we must be calling it from setup or nosetests but using at least # nose's core etc - self.assertTrue('>' in s, msg="no '>' in %r" % s) + self.assertIn('>', s) elif not ('coverage' in s): # There is only "fail2ban-testcases" in this case, no true traceback - self.assertFalse('>' in s, msg="'>' present in %r" % s) + self.assertNotIn('>', s) - self.assertTrue(':' in s, msg="no ':' in %r" % s) + self.assertIn(':', s) + + def _testAssertionErrorRE(self, regexp, fun, *args, **kwargs): + self.assertRaisesRegexp(AssertionError, regexp, fun, *args, **kwargs) + + def testExtendedAssertRaisesRE(self): + ## test _testAssertionErrorRE several fail cases: + def _key_err(msg): + raise KeyError(msg) + self.assertRaises(KeyError, + self._testAssertionErrorRE, r"^failed$", + _key_err, 'failed') + self.assertRaises(AssertionError, + self._testAssertionErrorRE, r"^failed$", + self.fail, '__failed__') + self._testAssertionErrorRE(r'failed.* does not match .*__failed__', + lambda: self._testAssertionErrorRE(r"^failed$", + self.fail, '__failed__') + ) + ## no exception in callable: + self.assertRaises(AssertionError, + self._testAssertionErrorRE, r"", int, 1) + self._testAssertionErrorRE(r'0 AssertionError not raised X.* does not match .*AssertionError not raised', + lambda: self._testAssertionErrorRE(r"^0 AssertionError not raised X$", + lambda: self._testAssertionErrorRE(r"", int, 1)) + ) + + def testExtendedAssertMethods(self): + ## assertIn, assertNotIn positive case: + self.assertIn('a', ['a', 'b', 'c', 'd']) + self.assertIn('a', ('a', 'b', 'c', 'd',)) + self.assertIn('a', 'cba') + self.assertIn('a', (c for c in 'cba' if c != 'b')) + self.assertNotIn('a', ['b', 'c', 'd']) + self.assertNotIn('a', ('b', 'c', 'd',)) + self.assertNotIn('a', 'cbd') + self.assertNotIn('a', (c.upper() for c in 'cba' if c != 'b')) + ## assertIn, assertNotIn negative case: + self._testAssertionErrorRE(r"'a' unexpectedly found in 'cba'", + self.assertNotIn, 'a', 'cba') + self._testAssertionErrorRE(r"1 unexpectedly found in \[0, 1, 2\]", + self.assertNotIn, 1, xrange(3)) + self._testAssertionErrorRE(r"'A' unexpectedly found in \['C', 'A'\]", + self.assertNotIn, 'A', (c.upper() for c in 'cba' if c != 'b')) + self._testAssertionErrorRE(r"'a' was not found in 'xyz'", + self.assertIn, 'a', 'xyz') + self._testAssertionErrorRE(r"5 was not found in \[0, 1, 2\]", + self.assertIn, 5, xrange(3)) + self._testAssertionErrorRE(r"'A' was not found in \['C', 'B'\]", + self.assertIn, 'A', (c.upper() for c in 'cba' if c != 'a')) + ## assertLogged, assertNotLogged positive case: + logSys = DefLogSys + self.pruneLog() + logSys.debug('test "xyz"') + self.assertLogged('test "xyz"') + self.assertLogged('test', 'xyz', all=True) + self.assertNotLogged('test', 'zyx', all=False) + self.assertNotLogged('test_zyx', 'zyx', all=True) + self.assertLogged('test', 'zyx', all=False) + self.pruneLog() + logSys.debug('xxxx "xxx"') + self.assertNotLogged('test "xyz"') + self.assertNotLogged('test', 'xyz', all=False) + self.assertNotLogged('test', 'xyz', 'zyx', all=True) + ## assertLogged, assertNotLogged negative case: + self.pruneLog() + logSys.debug('test "xyz"') + self._testAssertionErrorRE(r"All of the .* were found present in the log", + self.assertNotLogged, 'test "xyz"') + self._testAssertionErrorRE(r"was found in the log", + self.assertNotLogged, 'test', 'xyz', all=True) + self._testAssertionErrorRE(r"was not found in the log", + self.assertLogged, 'test', 'zyx', all=True) + self._testAssertionErrorRE(r"None among .* was found in the log", + self.assertLogged, 'test_zyx', 'zyx', all=False) + self._testAssertionErrorRE(r"All of the .* were found present in the log", + self.assertNotLogged, 'test', 'xyz', all=False) def testFormatterWithTraceBack(self): strout = StringIO() diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 07e10c7d..21e2d784 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -228,7 +228,7 @@ class Transmitter(TransmitterBase): time.sleep(1) self.assertEqual( self.transm.proceed(["stop", self.jailName]), (0, None)) - self.assertTrue(self.jailName not in self.server._Server__jails) + self.assertNotIn(self.jailName, self.server._Server__jails) def testStartStopAllJail(self): self.server.addJail("TestJail2", "auto") @@ -242,8 +242,8 @@ class Transmitter(TransmitterBase): time.sleep(0.1) self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None)) time.sleep(1) - self.assertTrue(self.jailName not in self.server._Server__jails) - self.assertTrue("TestJail2" not in self.server._Server__jails) + self.assertNotIn(self.jailName, self.server._Server__jails) + self.assertNotIn("TestJail2", self.server._Server__jails) def testJailIdle(self): self.assertEqual( diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8fc78683..7bccf6e2 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,6 +22,7 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" +import itertools import logging import os import re @@ -208,16 +209,45 @@ def gatherTests(regexps=None, no_network=False): return tests -# forwards compatibility of unittest.TestCase for some early python versions -if not hasattr(unittest.TestCase, 'assertIn'): - def __assertIn(self, a, b, msg=None): - if a not in b: # pragma: no cover - self.fail(msg or "%r was not found in %r" % (a, b)) - unittest.TestCase.assertIn = __assertIn - def __assertNotIn(self, a, b, msg=None): - if a in b: # pragma: no cover - self.fail(msg or "%r was found in %r" % (a, b)) - unittest.TestCase.assertNotIn = __assertNotIn +# +# Forwards compatibility of unittest.TestCase for some early python versions +# + +if not hasattr(unittest.TestCase, 'assertRaisesRegexp'): + def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs): + try: + fun(*args, **kwargs) + except exccls as e: + if re.search(regexp, e.message) is None: + self.fail('\"%s\" does not match \"%s\"' % (regexp, e.message)) + else: + self.fail('%s not raised' % getattr(exccls, '__name__')) + unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp + +# always custom following methods, because we use atm better version of both (support generators) +if True: ## if not hasattr(unittest.TestCase, 'assertIn'): + def assertIn(self, a, b, msg=None): + bb = b + wrap = False + if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + b, bb = itertools.tee(b) + wrap = True + if a not in b: + if wrap: bb = list(bb) + msg = msg or "%r was not found in %r" % (a, bb) + self.fail(msg) + unittest.TestCase.assertIn = assertIn + def assertNotIn(self, a, b, msg=None): + bb = b + wrap = False + if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + b, bb = itertools.tee(b) + wrap = True + if a in b: + if wrap: bb = list(bb) + msg = msg or "%r unexpectedly found in %r" % (a, bb) + self.fail(msg) + unittest.TestCase.assertNotIn = assertNotIn class LogCaptureTestCase(unittest.TestCase): From 8b1225f17761d40ea841b386353ef7b3db1e3be8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 1 Aug 2016 14:07:37 +0200 Subject: [PATCH 04/60] several amend fixes after cherry pick from 10th branch --- fail2ban/server/database.py | 14 ++++++++++--- fail2ban/tests/utils.py | 39 +++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 560fbfe5..9f562511 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -293,8 +293,12 @@ class Fail2BanDb(object): Jail to be added to the database. """ cur.execute( - "INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)", + "INSERT OR IGNORE INTO jails(name, enabled) VALUES(?, 1)", (jail.name,)) + if cur.rowcount <= 0: + cur.execute( + "UPDATE jails SET enabled = 1 WHERE name = ? AND enabled != 1", + (jail.name,)) @commitandrollback def delJail(self, cur, jail): @@ -317,7 +321,7 @@ class Fail2BanDb(object): cur.execute("UPDATE jails SET enabled=0") @commitandrollback - def getJailNames(self, cur): + def getJailNames(self, cur, enabled=None): """Get name of jails in database. Currently only used for testing purposes. @@ -327,7 +331,11 @@ class Fail2BanDb(object): set Set of jail names. """ - cur.execute("SELECT name FROM jails") + if enabled is None: + cur.execute("SELECT name FROM jails") + else: + cur.execute("SELECT name FROM jails WHERE enabled=%s" % + (int(enabled),)) return set(row[0] for row in cur.fetchmany()) @commitandrollback diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 7bccf6e2..e091c935 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -271,6 +271,7 @@ class LogCaptureTestCase(unittest.TestCase): def tearDown(self): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() + self.pruneLog() logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level @@ -278,7 +279,7 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() - def assertLogged(self, *s): + def assertLogged(self, *s, **kwargs): """Assert that one of the strings was logged Preferable to assertTrue(self._is_logged(..))) @@ -288,14 +289,23 @@ class LogCaptureTestCase(unittest.TestCase): ---------- s : string or list/set/tuple of strings Test should succeed if string (or any of the listed) is present in the log + all : boolean (default False) if True should fail if any of s not logged """ logged = self._log.getvalue() - for s_ in s: - if s_ in logged: - return - raise AssertionError("None among %r was found in the log: %r" % (s, logged)) + if not kwargs.get('all', False): + # at least one entry should be found: + for s_ in s: + if s_ in logged: + return + if True: # pragma: no cover + self.fail("None among %r was found in the log: ===\n%s===" % (s, logged)) + else: + # each entry should be found: + for s_ in s: + if s_ not in logged: # pragma: no cover + self.fail("%r was not found in the log: ===\n%s===" % (s_, logged)) - def assertNotLogged(self, *s): + def assertNotLogged(self, *s, **kwargs): """Assert that strings were not logged Parameters @@ -303,13 +313,22 @@ class LogCaptureTestCase(unittest.TestCase): s : string or list/set/tuple of strings Test should succeed if the string (or at least one of the listed) is not present in the log + all : boolean (default False) if True should fail if any of s logged """ logged = self._log.getvalue() - for s_ in s: - if s_ not in logged: - return - raise AssertionError("All of the %r were found present in the log: %r" % (s, logged)) + if not kwargs.get('all', False): + for s_ in s: + if s_ not in logged: + return + if True: # pragma: no cover + self.fail("All of the %r were found present in the log: ===\n%s===" % (s, logged)) + else: + for s_ in s: + if s_ in logged: # pragma: no cover + self.fail("%r was found in the log: ===\n%s===" % (s_, logged)) + def pruneLog(self): + self._log.truncate(0) def getLog(self): return self._log.getvalue() From eb6e3c52ae10c1609bc6a585803883265c674b04 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 1 Aug 2016 18:04:00 +0200 Subject: [PATCH 05/60] ChangeLog entries for the last fix (cherry pick from 0.10) --- ChangeLog | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59b6b1af..3f1052a8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,13 +6,31 @@ Fail2Ban: Changelog =================== -ver. 0.9.5 (2016/07/15) - old-not-obsolete +ver. 0.9.6 (2016/XX/XX) - wanna-be-released ----------- 0.9.x line is no longer heavily developed. If you are interested in new features (e.g. IPv6 support), please consider 0.10 branch and its releases. +### Fixes +* Misleading add resp. enable of (already available) jail in database, that + induced a subsequent error: last position of log file will be never retrieved (gh-795) +* Fixed a distribution related bug within testReadStockJailConfForceEnabled + (e.g. test-cases faults on Fedora, see gh-1353) + +### New Features + +### Enhancements +* Several test cases rewritten using new methods assertIn, assertNotIn +* New forward compatibility method assertRaisesRegexp (normally python >= 2.7). + Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged + are test covered now + + +ver. 0.9.5 (2016/07/15) - old-not-obsolete +----------- + ### Fixes * `filter.d/monit.conf` - Extended failregex with new monit "access denied" version (gh-1355) From c0994b0c6c2d4760f137ee3fdc1bf06c37a190dd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 4 Aug 2016 10:23:05 -0400 Subject: [PATCH 06/60] DOC: minor typo (thanks John Bernard) Closes #1496 --- config/action.d/badips.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf index 70b46546..6f9513f6 100644 --- a/config/action.d/badips.conf +++ b/config/action.d/badips.conf @@ -1,6 +1,6 @@ # Fail2ban reporting to badips.com # -# Note: This reports and IP only and does not actually ban traffic. Use +# Note: This reports an IP only and does not actually ban traffic. Use # another action in the same jail if you want bans to occur. # # Set the category to the appropriate value before use. From 890a3dcbb96d47fead173943a005d6b2a4c90dee Mon Sep 17 00:00:00 2001 From: rhardy613 Date: Fri, 5 Aug 2016 17:26:47 -0400 Subject: [PATCH 07/60] Fix ASSP filter to work with current release of ASSP ASSP V1 development stopped at the end of 2014 and it is now deprecated. All users were urged to upgrade to ASSP V2 which is still actively developed. For some reason fail2ban 0.9.5 (and trunk) still have code which only understands ASSP V1 logs. This means the filter ignores brute force attacks against ASSP. Now updated with anchored patterns tested against 6 months of log data. --- config/filter.d/assp.conf | 21 ++++++++++++++------- fail2ban/tests/files/logs/assp | 14 +++++++++++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 0bfba6dc..e740ad63 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -7,18 +7,25 @@ [Definition] -failregex = \<\S+@\S+\.\S+\> to: \S+@\S+\.\S+ relay attempt blocked for: \S+$ - \[SMTP Error\] 535 5\.7\.8 Error: authentication failed.*$ +failregex = ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))*: \S+$ + ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)*$ ignoreregex = # DEV Notes: # -# Examples: -# Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org -# Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 -# Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 -# Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: +# Examples matches: +# Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org +# Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 +# Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: +# Jan-05-16 08:38:49 m1-01129-09140 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 relay attempt blocked for (parsing): +# Jun-12-16 16:43:37 m1-64217-12013 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 to: user2@example.com relay attempt blocked for (parsing): +# Jan-22-16 22:25:51 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid authentication mechanism +# Mar-19-16 13:42:20 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid base64 data in continued response +# Jul-18-16 16:54:21 [Worker_2] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server +# Jul-18-16 17:14:23 m1-76453-02949 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server + # # Author: Enrico Labedzki (enrico.labedzki@deiwos.de) # Updated: Robert Hardy (rhardy@webcon.ca) diff --git a/fail2ban/tests/files/logs/assp b/fail2ban/tests/files/logs/assp index 71c28221..6e9c1c35 100644 --- a/fail2ban/tests/files/logs/assp +++ b/fail2ban/tests/files/logs/assp @@ -5,4 +5,16 @@ Jul-30-16 17:07:25 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: au # failJSON: { "time": "2016-07-30T17:11:05", "match": true , "host": "0.0.0.0" } Jul-30-16 17:11:05 m1-13060-05386 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 # failJSON: { "time": "2016-07-31T06:45:59", "match": true , "host": "0.0.0.0" } -Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: +Jul-31-16 06:45:59 [Worker_1] [TLS-in] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: +# failJSON: { "time": "2016-01-05T08:38:49", "match": true , "host": "0.0.0.0" } +Jan-05-16 08:38:49 m1-01129-09140 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 relay attempt blocked for (parsing): +# failJSON: { "time": "2016-06-12T16:43:37", "match": true , "host": "0.0.0.0" } +Jun-12-16 16:43:37 m1-64217-12013 [Worker_1] [TLS-in] [TLS-out] [RelayAttempt] 0.0.0.0 to: user2@example.com relay attempt blocked for (parsing): +# failJSON: { "time": "2016-01-22T22:25:51", "match": true , "host": "0.0.0.0" } +Jan-22-16 22:25:51 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid authentication mechanism +# failJSON: { "time": "2016-03-19T13:42:20", "match": true , "host": "0.0.0.0" } +Mar-19-16 13:42:20 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Invalid base64 data in continued response +# failJSON: { "time": "2016-07-18T16:54:21", "match": true , "host": "0.0.0.0" } +Jul-18-16 16:54:21 [Worker_2] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server +# failJSON: { "time": "2016-07-18T17:14:23", "match": true , "host": "0.0.0.0" } +Jul-18-16 17:14:23 m1-76453-02949 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: Connection lost to authentication server From 66fe5a77cef8f4337a7a39b70c46b3afb49ecd22 Mon Sep 17 00:00:00 2001 From: rhardy613 Date: Fri, 5 Aug 2016 23:18:51 -0400 Subject: [PATCH 08/60] Fix ASSP filter to work with both ASSP V1 and V2 ASSP V1 development stopped at the end of 2014 and it is now deprecated. All users were urged to upgrade to ASSP V2 which is still actively developed. fail2ban 0.9.5 (and trunk) still have code which only understands ASSP V1 logs. This means the filter ignores brute force attacks against ASSP. This fix adds V2 support. --- config/filter.d/assp.conf | 21 +++++++++++++++------ fail2ban/tests/files/logs/assp | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index e740ad63..e71313e7 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -1,5 +1,5 @@ -# Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP) Version 2.5.1 (or later) -# +# Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP) +# Filter works in theory for both ASSP V1 and V2. Recommended ASSP is V2.5.1 or later. # Homepage: http://sourceforge.net/projects/assp/ # ProjectSite: http://sourceforge.net/projects/assp/?source=directory # @@ -7,14 +7,23 @@ [Definition] -failregex = ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))*: \S+$ - ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)*$ +__assp_actions = (?:dropping|refusing) + +failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ + ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ + ^ Blocking - too much AUTH errors \(\d{,3}\);$ + ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))*: \S+$ + ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)*$ ignoreregex = # DEV Notes: +# V1 Examples matches: +# Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); +# Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded # -# Examples matches: +# V2 Examples matches: # Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org # Jul-30-16 16:59:42 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 # Jul-30-16 00:15:36 m1-52131-09651 [Worker_1] [TLS-out] 0.0.0.0 [SMTP Error] 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6 @@ -28,4 +37,4 @@ ignoreregex = # # Author: Enrico Labedzki (enrico.labedzki@deiwos.de) -# Updated: Robert Hardy (rhardy@webcon.ca) +# V2 Filters: Robert Hardy (rhardy@webcon.ca) diff --git a/fail2ban/tests/files/logs/assp b/fail2ban/tests/files/logs/assp index 6e9c1c35..21b01f9f 100644 --- a/fail2ban/tests/files/logs/assp +++ b/fail2ban/tests/files/logs/assp @@ -1,3 +1,27 @@ +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } +Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:08:36", "match": true , "host": "68.171.223.68" } +Apr-07-13 07:08:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:10:37", "match": true , "host": "68.171.223.68" } +Apr-07-13 07:10:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:12:37", "match": true , "host": "68.171.223.68" } +Apr-07-13 07:12:37 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-07T07:14:36", "match": true , "host": "68.171.223.68" } +Apr-07-13 07:14:36 [SSL-out] 68.171.223.68 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (8); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (9); +# failJSON: { "time": "2013-04-27T02:25:09", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:09 Blocking 217.194.197.97 - too much AUTH errors (10); +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:10", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:10 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; +# failJSON: { "time": "2013-04-27T02:25:11", "match": true , "host": "217.194.197.97" } +Apr-27-13 02:25:11 [SSL-out] 217.194.197.97 max sender authentication errors (5) exceeded -- dropping connection - after reply: 535 5.7.8 Error: authentication failed: UGFzc3dvcmQ6; # failJSON: { "time": "2016-07-29T16:49:52", "match": true , "host": "0.0.0.0" } Jul-29-16 16:49:52 m1-25391-06124 [Worker_1] [TLS-out] [RelayAttempt] 0.0.0.0 to: user@example.org relay attempt blocked for: someone@example.org # failJSON: { "time": "2016-07-30T17:07:25", "match": true , "host": "0.0.0.0" } From 8265e3f0f94e311345e162cd189b6fbbfc033891 Mon Sep 17 00:00:00 2001 From: rhardy613 Date: Fri, 5 Aug 2016 23:25:15 -0400 Subject: [PATCH 09/60] Fix comments For some reasons the comment changes weren't pickup in the last commit. This fixes it. --- config/filter.d/assp.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index e71313e7..896d7514 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -1,11 +1,14 @@ # Fail2Ban filter for Anti-Spam SMTP Proxy Server (ASSP) # Filter works in theory for both ASSP V1 and V2. Recommended ASSP is V2.5.1 or later. +# Support for ASSP V1 ended in 2014 so if you are still running ASSP V1 an immediate upgrade is recommended. +# # Homepage: http://sourceforge.net/projects/assp/ # ProjectSite: http://sourceforge.net/projects/assp/?source=directory # # [Definition] +# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibilty reasons. __assp_actions = (?:dropping|refusing) From 89f8999fe51d52970f9357526a313d174758769e Mon Sep 17 00:00:00 2001 From: rhardy613 Date: Sat, 6 Aug 2016 01:07:04 -0400 Subject: [PATCH 10/60] Add changelog entry for ASSP filter changes Add changelog entry for ASSP filter changes --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 59b6b1af..fa39326a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ new features (e.g. IPv6 support), please consider 0.10 branch and its releases. ### Fixes +* `filter.d/assp.conf` + - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) * `filter.d/monit.conf` - Extended failregex with new monit "access denied" version (gh-1355) - failregex of previous monit version merged as single expression From c52aaa8b78ad9b25ee02ac9fdeedf5e03bddd89c Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 8 Aug 2016 19:06:28 +0200 Subject: [PATCH 11/60] ASSP failregex minor fixes --- config/filter.d/assp.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 896d7514..278e25cb 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -15,8 +15,8 @@ __assp_actions = (?:dropping|refusing) failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ ^ Blocking - too much AUTH errors \(\d{,3}\);$ - ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))*: \S+$ - ^\s*(?:[m0-9\-]+\s+)*(?:\[\S+\]\s+)* \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)*$ + ^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)* (?:\<\S+@\S+\.\S+\> )*(?:to: \S+@\S+\.\S+ )*relay attempt blocked for(?: \(parsing\))?: \S+$ + ^\s*(?:[\w\-]+\s+)*(?:\[\S+\]\s+)* \[SMTP Error\] 535 5\.7\.8 Error: authentication failed:\s+(?:\S+|Connection lost to authentication server|Invalid authentication mechanism|Invalid base64 data in continued response)?$ ignoreregex = From 9ddbd642f725f3d5dfd94de7f08cc7526d83cd43 Mon Sep 17 00:00:00 2001 From: maksyms Date: Mon, 8 Aug 2016 22:07:55 +0100 Subject: [PATCH 12/60] Accept no space after "failed:" (#1501) yoh: Squashed to ease cherry-picking into 0.9 * accept no space after "failed:" fix issue #1497 * accept no space after "failed:" * Update postfix-sasl * Update postfix-sasl * Update postfix-sasl --- config/filter.d/postfix-sasl.conf | 2 +- fail2ban/tests/files/logs/postfix-sasl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index 4a6ceaaa..1a24ca94 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -9,7 +9,7 @@ before = common.conf _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] -failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/:]*={0,2})?\s*$ +failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$ ignoreregex = authentication failed: Connection lost to authentication server$ diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl index 9fcb0f49..cdcb5121 100644 --- a/fail2ban/tests/files/logs/postfix-sasl +++ b/fail2ban/tests/files/logs/postfix-sasl @@ -26,3 +26,7 @@ Jan 29 08:11:45 mail postfix-incoming/smtpd[10752]: warning: unknown[1.1.1.1]: S # failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" } Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: warning: astra4139.startdedicated.de[62.138.2.143]: SASL LOGIN authentication failed: UGFzc3dvcmQ6 + +# failJSON: { "time": "2005-08-03T15:30:49", "match": true , "host": "98.191.84.74" } +Aug 3 15:30:49 ksusha postfix/smtpd[17041]: warning: mail.foldsandwalker.com[98.191.84.74]: SASL Plain authentication failed: + From 123f4ceaee8d1f1b5c1896a6661a7ba066f29325 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 8 Aug 2016 17:11:07 -0400 Subject: [PATCH 13/60] Changelog for postfix-sasl fix --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index cbc7c8d5..76844b6f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ releases. (e.g. test-cases faults on Fedora, see gh-1353) * `filter.d/assp.conf` - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) +* `filter.d/postfix-sasl.conf` + - Allow for having no trailing space after 'failed:' (gh-1497) ### New Features From 9d70c49ea86b41a94cea8f5a2e035db0ff6871a5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 9 Aug 2016 06:49:40 -0400 Subject: [PATCH 14/60] BF: install doc files only under Linuxes and other GNU systems (Closes #1233) (#1503) --- .travis.yml | 2 ++ setup.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50894a94..9ef607da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,8 @@ script: - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . + # Doc files should get installed on Travis under Linux + - test -e /usr/share/doc/fail2ban/FILTERS after_success: - coveralls - codecov diff --git a/setup.py b/setup.py index e3c499d2..2d3f4569 100755 --- a/setup.py +++ b/setup.py @@ -19,9 +19,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. __author__ = "Cyril Jaquier, Steven Hiscocks, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2013 Fail2Ban Contributors" +__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2008-2016 Fail2Ban Contributors" __license__ = "GPL" +import platform + try: import setuptools from setuptools import setup @@ -85,6 +87,18 @@ if os.path.exists('/var/run'): # realpath is used to possibly resolve /var/run -> /run symlink data_files_extra += [(realpath('/var/run/fail2ban'), '')] +# Installing documentation files only under Linux or other GNU/ systems +# (e.g. GNU/kFreeBSD), since others might have protective mechanisms forbidding +# installation there (see e.g. #1233) +platform_system = platform.system().lower() +doc_files = ['README.md', 'DEVELOP', 'FILTERS', 'doc/run-rootless.txt'] +if platform_system in ('solaris', 'sunos'): + doc_files.append('README.Solaris') +if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswith('gnu'): + data_files_extra.append( + ('/usr/share/doc/fail2ban', doc_files) + ) + # Get version number, avoiding importing fail2ban. # This is due to tests not functioning for python3 as 2to3 takes place later exec(open(join("fail2ban", "version.py")).read()) @@ -148,10 +162,6 @@ setup( ('/var/lib/fail2ban', '' ), - ('/usr/share/doc/fail2ban', - ['README.md', 'README.Solaris', 'DEVELOP', 'FILTERS', - 'doc/run-rootless.txt'] - ) ] + data_files_extra, **setup_extra ) From 38d53a72fd2b55fae0c6a57e7ea10ffa3446b650 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Aug 2016 18:34:18 +0200 Subject: [PATCH 15/60] introduces new command "fail2ban-python", as automatically created symlink to python executable, where fail2ban currently installed (resp. its modules are located); fixed pythonic filters and test scripts (running via "fail2ban-python" now); fixed test case "testSetupInstallRoot" not for default python (also using direct call, out of virtualenv); # Conflicts: # config/filter.d/ignorecommands/apache-fakegooglebot # fail2ban/tests/files/config/apache-auth/digest.py # fail2ban/tests/files/ignorecommand.py # fail2ban/tests/misctestcase.py --- bin/fail2ban-testcases | 5 ++- .../ignorecommands/apache-fakegooglebot | 2 +- fail2ban/helpers.py | 18 +++++++++ .../tests/files/config/apache-auth/digest.py | 2 +- fail2ban/tests/files/ignorecommand.py | 2 +- fail2ban/tests/misctestcase.py | 21 ++++++++++ setup.py | 40 ++++++++++++++++++- 7 files changed, 85 insertions(+), 5 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index dd6547a5..d02f482e 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -38,11 +38,14 @@ if os.path.exists("fail2ban/__init__.py"): from fail2ban.version import version from fail2ban.tests.utils import gatherTests -from fail2ban.helpers import FormatterWithTraceBack, getLogger +from fail2ban.helpers import updatePyExec, FormatterWithTraceBack, getLogger from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option +# Update fail2ban-python env to current python version (where f2b-modules located/installed) +updatePyExec(os.path.dirname(__file__)) + def get_opt_parser(): # use module docstring for help output p = OptionParser( diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 19fb5107..9e6f4459 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env fail2ban-python # Inspired by https://isc.sans.edu/forums/diary/When+Google+isnt+Google/15968/ # # Written in Python to reuse built-in Python batteries and not depend on diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index aef39835..8523f21b 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -113,6 +113,24 @@ class FormatterWithTraceBack(logging.Formatter): return logging.Formatter.format(self, record) +def updatePyExec(bindir, executable=None): + """Update fail2ban-python link to current python version (where f2b-modules located/installed) + """ + bindir = os.path.realpath(bindir) + if executable is None: + executable = sys.executable + pypath = os.path.join(bindir, 'fail2ban-python') + # if not exists or point to another version - update link: + isfile = os.path.isfile(pypath) + if not isfile or os.path.realpath(pypath) != os.path.realpath(executable): + if isfile: + os.unlink(pypath) + os.symlink(executable, pypath) + # extend current environment path (e.g. if fail2ban not yet installed): + if bindir not in os.environ["PATH"].split(os.pathsep): + os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir; + + def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py index 875ebffe..03588594 100755 --- a/fail2ban/tests/files/config/apache-auth/digest.py +++ b/fail2ban/tests/files/config/apache-auth/digest.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env fail2ban-python import requests try: diff --git a/fail2ban/tests/files/ignorecommand.py b/fail2ban/tests/files/ignorecommand.py index dd6b5aab..7011b51b 100755 --- a/fail2ban/tests/files/ignorecommand.py +++ b/fail2ban/tests/files/ignorecommand.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env fail2ban-python import sys if sys.argv[1] == "10.0.0.1": exit(0) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 5f7d853d..069ad659 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -70,6 +70,17 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) +def _getSysPythonVersion(): + import subprocess, locale + sysVerCmd = "fail2ban-python -c 'import sys; print(tuple(sys.version_info))'" + if sys.version_info >= (2,7): + sysVer = subprocess.check_output(sysVerCmd, shell=True) + else: + sysVer = subprocess.Popen(sysVerCmd, shell=True, stdout=subprocess.PIPE).stdout.read() + if sys.version_info >= (3,): + sysVer = sysVer.decode(locale.getpreferredencoding(), 'replace') + return str(sysVer).rstrip() + class SetupTest(unittest.TestCase): def setUp(self): @@ -79,6 +90,12 @@ class SetupTest(unittest.TestCase): raise unittest.SkipTest( "Seems to be running not out of source distribution" " -- cannot locate setup.py") + # compare current version of python installed resp. active one: + sysVer = _getSysPythonVersion() + if sysVer != str(tuple(sys.version_info)): + raise unittest.SkipTest( + "Seems to be running with python distribution %s" + " -- install can be tested only with system distribution %s" % (str(tuple(sys.version_info)), sysVer)) def testSetupInstallRoot(self): if not self.setup: @@ -125,6 +142,10 @@ class SetupTest(unittest.TestCase): 'etc/fail2ban/jail.conf'): self.assertTrue(os.path.exists(os.path.join(tmp, f)), msg="Can't find %s" % f) + self.assertEqual( + os.path.realpath(os.path.join(tmp, 'usr/local/bin/fail2ban-python')), + os.path.realpath(sys.executable)) + finally: # clean up shutil.rmtree(tmp) diff --git a/setup.py b/setup.py index 2d3f4569..aa9f7055 100755 --- a/setup.py +++ b/setup.py @@ -40,12 +40,46 @@ except ImportError: # python 2.x from distutils.command.build_py import build_py from distutils.command.build_scripts import build_scripts +# all versions +from distutils.command.install_scripts import install_scripts + import os from os.path import isfile, join, isdir, realpath import sys import warnings from glob import glob + +def updatePyExec(bindir, executable=None): + """Update fail2ban-python link to current python version (where f2b-modules located/installed) + """ + bindir = os.path.realpath(bindir) + if executable is None: + executable = sys.executable + pypath = os.path.join(bindir, 'fail2ban-python') + # if not exists or point to another version - update link: + isfile = os.path.isfile(pypath) + if not isfile or os.path.realpath(pypath) != os.path.realpath(executable): + if isfile: + os.unlink(pypath) + os.symlink(executable, pypath) + + +# Wrapper to install python binding (to current python version): +class install_scripts_f2b(install_scripts): + + def get_outputs(self): + outputs = install_scripts.get_outputs(self) + fn = None + for fn in outputs: + if os.path.basename(fn) == 'fail2ban-server': + break + bindir = os.path.dirname(fn) + print('creating fail2ban-python binding -> %s' % (bindir,)) + updatePyExec(bindir) + return outputs + + if setuptools and "test" in sys.argv: import logging logSys = logging.getLogger("fail2ban") @@ -113,12 +147,16 @@ setup( url = "http://www.fail2ban.org", license = "GPL", platforms = "Posix", - cmdclass = {'build_py': build_py, 'build_scripts': build_scripts}, + cmdclass = { + 'build_py': build_py, 'build_scripts': build_scripts, + 'install_scripts': install_scripts_f2b + }, scripts = [ 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', 'bin/fail2ban-testcases', + # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper ], packages = [ 'fail2ban', From 6cdc1ce68522c137c7242ce8dbdb8bca1bb138e7 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Aug 2016 19:57:40 +0200 Subject: [PATCH 16/60] compatibility fix (virtualenv, running test cases in py3) # Conflicts: # MANIFEST --- MANIFEST | 1 + bin/fail2ban-testcases | 3 ++- fail2ban/helpers.py | 18 --------------- fail2ban/setup.py | 42 ++++++++++++++++++++++++++++++++++ fail2ban/tests/misctestcase.py | 33 ++++++++++++++++---------- setup.py | 18 ++++----------- 6 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 fail2ban/setup.py diff --git a/MANIFEST b/MANIFEST index c27878e8..e0d7398c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -194,6 +194,7 @@ fail2ban/server/server.py fail2ban/server/strptime.py fail2ban/server/ticket.py fail2ban/server/transmitter.py +fail2ban/setup.py fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index d02f482e..e0ff11d2 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -38,7 +38,8 @@ if os.path.exists("fail2ban/__init__.py"): from fail2ban.version import version from fail2ban.tests.utils import gatherTests -from fail2ban.helpers import updatePyExec, FormatterWithTraceBack, getLogger +from fail2ban.helpers import FormatterWithTraceBack, getLogger +from fail2ban.setup import updatePyExec from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 8523f21b..aef39835 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -113,24 +113,6 @@ class FormatterWithTraceBack(logging.Formatter): return logging.Formatter.format(self, record) -def updatePyExec(bindir, executable=None): - """Update fail2ban-python link to current python version (where f2b-modules located/installed) - """ - bindir = os.path.realpath(bindir) - if executable is None: - executable = sys.executable - pypath = os.path.join(bindir, 'fail2ban-python') - # if not exists or point to another version - update link: - isfile = os.path.isfile(pypath) - if not isfile or os.path.realpath(pypath) != os.path.realpath(executable): - if isfile: - os.unlink(pypath) - os.symlink(executable, pypath) - # extend current environment path (e.g. if fail2ban not yet installed): - if bindir not in os.environ["PATH"].split(os.pathsep): - os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir; - - def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ diff --git a/fail2ban/setup.py b/fail2ban/setup.py new file mode 100644 index 00000000..87d50e66 --- /dev/null +++ b/fail2ban/setup.py @@ -0,0 +1,42 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +__author__ = "Serg G. Brester" +__license__ = "GPL" + +import os +import sys + + +def updatePyExec(bindir, executable=None): + """Update fail2ban-python link to current python version (where f2b-modules located/installed) + """ + bindir = os.path.realpath(bindir) + if executable is None: + executable = sys.executable + pypath = os.path.join(bindir, 'fail2ban-python') + # if not exists or point to another version - update link: + isfile = os.path.isfile(pypath) + if not isfile or os.path.realpath(pypath) != os.path.realpath(executable): + if isfile: + os.unlink(pypath) + os.symlink(executable, pypath) + # extend current environment path (e.g. if fail2ban not yet installed): + if bindir not in os.environ["PATH"].split(os.pathsep): + os.environ["PATH"] = os.environ["PATH"] + os.pathsep + bindir; diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 069ad659..e2b4c0b6 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -70,16 +70,21 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) +if sys.version_info >= (2,7): + def _sh_call(cmd): + import subprocess, locale + ret = subprocess.check_output(cmd, shell=True) + if sys.version_info >= (3,): + ret = ret.decode(locale.getpreferredencoding(), 'replace') + return str(ret).rstrip() +else: + def _sh_call(cmd): + import subprocess + ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read() + return str(ret).rstrip() + def _getSysPythonVersion(): - import subprocess, locale - sysVerCmd = "fail2ban-python -c 'import sys; print(tuple(sys.version_info))'" - if sys.version_info >= (2,7): - sysVer = subprocess.check_output(sysVerCmd, shell=True) - else: - sysVer = subprocess.Popen(sysVerCmd, shell=True, stdout=subprocess.PIPE).stdout.read() - if sys.version_info >= (3,): - sysVer = sysVer.decode(locale.getpreferredencoding(), 'replace') - return str(sysVer).rstrip() + return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'") class SetupTest(unittest.TestCase): @@ -142,9 +147,13 @@ class SetupTest(unittest.TestCase): 'etc/fail2ban/jail.conf'): self.assertTrue(os.path.exists(os.path.join(tmp, f)), msg="Can't find %s" % f) - self.assertEqual( - os.path.realpath(os.path.join(tmp, 'usr/local/bin/fail2ban-python')), - os.path.realpath(sys.executable)) + # Because the install (test) path in virtual-env differs from some development-env, + # it is not a `tmp + '/usr/local/bin/'`, so search for it: + installedPath = _sh_call('find ' + tmp+ ' -name fail2ban-python').split('\n') + self.assertTrue(len(installedPath) > 0) + for installedPath in installedPath: + self.assertEqual( + os.path.realpath(installedPath), os.path.realpath(sys.executable)) finally: # clean up diff --git a/setup.py b/setup.py index aa9f7055..a4a3f977 100755 --- a/setup.py +++ b/setup.py @@ -49,20 +49,7 @@ import sys import warnings from glob import glob - -def updatePyExec(bindir, executable=None): - """Update fail2ban-python link to current python version (where f2b-modules located/installed) - """ - bindir = os.path.realpath(bindir) - if executable is None: - executable = sys.executable - pypath = os.path.join(bindir, 'fail2ban-python') - # if not exists or point to another version - update link: - isfile = os.path.isfile(pypath) - if not isfile or os.path.realpath(pypath) != os.path.realpath(executable): - if isfile: - os.unlink(pypath) - os.symlink(executable, pypath) +from fail2ban.setup import updatePyExec # Wrapper to install python binding (to current python version): @@ -80,6 +67,9 @@ class install_scripts_f2b(install_scripts): return outputs +# Update fail2ban-python env to current python version (where f2b-modules located/installed) +updatePyExec(os.path.join(os.path.dirname(__file__), 'bin')) + if setuptools and "test" in sys.argv: import logging logSys = logging.getLogger("fail2ban") From db30b7ce06cd5da3fa0720a34116084026f0083d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Aug 2016 16:41:55 +0200 Subject: [PATCH 17/60] BF: prefer sys.argv[0] by retrieving of root resp. bin path: __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.) --- bin/fail2ban-testcases | 6 +++++- setup.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index e0ff11d2..a711e07c 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -45,7 +45,11 @@ from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option # Update fail2ban-python env to current python version (where f2b-modules located/installed) -updatePyExec(os.path.dirname(__file__)) +bindir = os.path.dirname( + # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): + sys.argv[0] if os.path.basename(sys.argv[0]) == 'fail2ban-testcases' else __file__ +) +updatePyExec(bindir) def get_opt_parser(): # use module docstring for help output diff --git a/setup.py b/setup.py index a4a3f977..cbfc6e07 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,11 @@ class install_scripts_f2b(install_scripts): # Update fail2ban-python env to current python version (where f2b-modules located/installed) -updatePyExec(os.path.join(os.path.dirname(__file__), 'bin')) +rootdir = os.path.realpath(os.path.dirname( + # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): + sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ +)) +updatePyExec(os.path.join(rootdir, 'bin')) if setuptools and "test" in sys.argv: import logging From cb340db2209d9d2f518c8ff3f9bbea03c17be973 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Aug 2016 18:37:46 +0200 Subject: [PATCH 18/60] ChangeLog entry for gh-1508 --- ChangeLog | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ChangeLog b/ChangeLog index 76844b6f..ecb8cc87 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,13 @@ releases. induced a subsequent error: last position of log file will be never retrieved (gh-795) * Fixed a distribution related bug within testReadStockJailConfForceEnabled (e.g. test-cases faults on Fedora, see gh-1353) +* Fixed pythonic filters and test scripts (running via wrong python version, + uses "fail2ban-python" now); +* Fixed test case "testSetupInstallRoot" for not default python version (also + using direct call, out of virtualenv); +* `filter.d/ignorecommands/apache-fakegooglebot` + - Fixed error within apache-fakegooglebot, that will be called + with wrong python version (gh-1506) * `filter.d/assp.conf` - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) * `filter.d/postfix-sasl.conf` @@ -26,6 +33,18 @@ releases. ### New Features ### Enhancements +* Introduces new command "fail2ban-python", as automatically created symlink to + python executable, where fail2ban currently installed (resp. its modules are located): + - allows to use the same version, fail2ban currently running, e.g. in + external scripts just via replace python with fail2ban-python: + ```diff + -#!/usr/bin/env python + +#!/usr/bin/env fail2ban-python + ``` + - always the same pickle protocol + - the same (and also guaranteed available) fail2ban modules + - simplified stand-alone install, resp. stand-alone installation possibility + via setup (like gh-1487) is getting closer * Several test cases rewritten using new methods assertIn, assertNotIn * New forward compatibility method assertRaisesRegexp (normally python >= 2.7). Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged From c49fe12f701807a8d89bfe57c9f7f492375a0a53 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 12:53:40 +0200 Subject: [PATCH 19/60] fix fail2banregextestcase using setUpMyTime/tearDownMyTime: always use correct static time as base-time (using mock up MyTime), correct datetimes inside test --- fail2ban/tests/fail2banregextestcase.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 3321ffd8..1119efdb 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -39,7 +39,7 @@ except ImportError: from ..client import fail2banregex from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output -from .utils import LogCaptureTestCase, logSys +from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase, logSys from .utils import CONFIG_DIR @@ -70,10 +70,12 @@ class Fail2banRegexTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" LogCaptureTestCase.setUp(self) + setUpMyTime() def tearDown(self): """Call after every test case.""" LogCaptureTestCase.tearDown(self) + tearDownMyTime() def testWrongRE(self): (opts, args, fail2banRegex) = _Fail2banRegex( @@ -159,8 +161,8 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') - self.assertLogged('141.3.81.106 Fri Aug 14 11:53:59 2015') - self.assertLogged('141.3.81.106 Fri Aug 14 11:54:59 2015') + self.assertLogged('141.3.81.106 Sun Aug 14 11:53:59 2005') + self.assertLogged('141.3.81.106 Sun Aug 14 11:54:59 2005') def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( @@ -169,9 +171,8 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') - self.assertLogged('Error decoding line'); - self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); - self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); + self.assertLogged('Error decoding line') + self.assertLogged('Continuing to process line ignoring invalid characters:') self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') From 7f55be3fadc1701ab08e0c97936748e3eec98c12 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 15:31:28 +0200 Subject: [PATCH 20/60] amend to b6bb2f88c1dbb111647269590d80d95f72c81c3e: datepattern right word boundary - prevents confusions if end of date-pattern (e.g. optional year part) misleadingly match not date values (see gh-1507) test cases extended to check ambiguous "unbound" patterns in log lines (match/miss resp. positive/negative cases) --- fail2ban/server/datetemplate.py | 8 ++++--- fail2ban/tests/fail2banregextestcase.py | 30 ++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 8210dad4..0a5ec55a 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -64,7 +64,7 @@ class DateTemplate(object): def getRegex(self): return self._regex - def setRegex(self, regex, wordBegin=True): + def setRegex(self, regex, wordBegin=True, wordEnd=True): """Sets regex to use for searching for date in log line. Parameters @@ -82,8 +82,10 @@ class DateTemplate(object): If regular expression fails to compile """ regex = regex.strip() - if (wordBegin and not re.search(r'^\^', regex)): - regex = r'\b' + regex + if wordBegin and not re.search(r'^\^', regex): + regex = r'(?=^|\b|\W)' + regex + if wordEnd and not re.search(r'\$$', regex): + regex += r'(?=\b|\W|$)' self._regex = regex self._cRegex = re.compile(regex, re.UNICODE | re.IGNORECASE) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 1119efdb..fb1bc6e3 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -187,4 +187,32 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') - + def testAmbiguousDatePattern(self): + for (matched, args) in ( + # positive case: + (1, ('Test failure Jan 23 21:59:59 for 192.0.2.1', r'for $')), + # ambiguous "unbound" patterns (missed): + (0, ('Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1', r'for $')), + (0, ('Test failure Jan 23 21:59:59123456789 for 192.0.2.1', r'for $')), + # ambiguous "no optional year" patterns (matched): + (1, ('Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), + (1, ('[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), + # direct specified patterns: + (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), + (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), + ): + logSys.debug('== test: %r', args) + (opts, args, fail2banRegex) = _Fail2banRegex(*args) + self.assertTrue(fail2banRegex.start(opts, args)) + matchedLog = 'Lines: 1 lines, 0 ignored, 1 matched, 0 missed' + missedLog = 'Lines: 1 lines, 0 ignored, 0 matched, 1 missed' + if matched: + self.assertLogged(matchedLog) + self.assertNotLogged(missedLog) + else: + self.assertNotLogged(matchedLog) + self.assertLogged(missedLog) + self.pruneLog() From 8e09be5fc8810a338c323922975410501fa927d7 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 18:53:35 +0200 Subject: [PATCH 21/60] test cases for boundaries for date-pattern extended (negative/positive, left/right) --- fail2ban/tests/fail2banregextestcase.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index fb1bc6e3..59ba60be 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -200,9 +200,18 @@ class Fail2banRegexTest(LogCaptureTestCase): # direct specified patterns: (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), + (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), + (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1', r'^Attempt\s+IP address $')), + (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]', r'^Attempt IP address , date: $')), + # direct specified patterns (begin/end, missed): + (0, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003', '^')), + (0, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1', '$')), + # direct specified patterns (begin/end, matched): + (1, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003', '^')), + (1, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1', '$')), ): logSys.debug('== test: %r', args) (opts, args, fail2banRegex) = _Fail2banRegex(*args) From 0bdee2556f72da7fbf5a14e8abcb402607966811 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 19:35:11 +0200 Subject: [PATCH 22/60] testAmbiguousDatePattern rewritten with DateDetector/DatePatternRegex directly (moved to misctestcase.py) --- fail2ban/tests/fail2banregextestcase.py | 39 +-------------------- fail2ban/tests/misctestcase.py | 45 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 59ba60be..1119efdb 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -187,41 +187,4 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('https://') - def testAmbiguousDatePattern(self): - for (matched, args) in ( - # positive case: - (1, ('Test failure Jan 23 21:59:59 for 192.0.2.1', r'for $')), - # ambiguous "unbound" patterns (missed): - (0, ('Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1', r'for $')), - (0, ('Test failure Jan 23 21:59:59123456789 for 192.0.2.1', r'for $')), - # ambiguous "no optional year" patterns (matched): - (1, ('Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), - (1, ('[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1', r'from $')), - # direct specified patterns: - (1, ('-d', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003', '^')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]', '^$')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), - (1, ('-d', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]', '^$')), - (1, ('-d', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1', '^$')), - (1, ('-d', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1', r'^ Attempt, IP address $')), - (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1', r'^Attempt\s+IP address $')), - (1, ('-d', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]', r'^Attempt IP address , date: $')), - # direct specified patterns (begin/end, missed): - (0, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003', '^')), - (0, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1', '$')), - # direct specified patterns (begin/end, matched): - (1, ('-d', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003', '^')), - (1, ('-d', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1', '$')), - ): - logSys.debug('== test: %r', args) - (opts, args, fail2banRegex) = _Fail2banRegex(*args) - self.assertTrue(fail2banRegex.start(opts, args)) - matchedLog = 'Lines: 1 lines, 0 ignored, 1 matched, 0 missed' - missedLog = 'Lines: 1 lines, 0 ignored, 0 matched, 1 missed' - if matched: - self.assertLogged(matchedLog) - self.assertNotLogged(missedLog) - else: - self.assertNotLogged(matchedLog) - self.assertLogged(missedLog) - self.pruneLog() + diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index e2b4c0b6..4dfff2d4 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -37,6 +37,7 @@ from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger from ..helpers import splitwords +from ..server.datedetector import DateDetector from ..server.datetemplate import DatePatternRegex @@ -340,3 +341,47 @@ class CustomDateFormatsTest(unittest.TestCase): self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) + + def testAmbiguousDatePattern(self): + defDD = DateDetector() + defDD.addDefaultTemplate() + logSys = DefLogSys + for (matched, dp, line) in ( + # positive case: + ('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'), + # ambiguous "unbound" patterns (missed): + (False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'), + (False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'), + # ambiguous "no optional year" patterns (matched): + ('Aug 8 11:25:50', None, 'Aug 8 11:25:50 14430f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 14430f2329b8 Authentication failed from 192.0.2.1'), + ('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 14430f2329b8 Authentication failed from 192.0.2.1'), + # direct specified patterns: + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'), + ('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'), + ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'), + # direct specified patterns (begin/end, missed): + (False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'), + (False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'), + # direct specified patterns (begin/end, matched): + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'), + ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'), + ): + logSys.debug('== test: %r', (matched, dp, line)) + if dp is None: + dd = defDD + else: + dp = DatePatternRegex(dp) + dd = DateDetector() + dd.appendTemplate(dp) + date = dd.getTime(line) + if matched: + self.assertTrue(date) + self.assertEqual(matched, date[1].group()) + else: + self.assertEqual(date, None) From 9935cf19c1b8aa9779e419b3f2c498531b362f1c Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Aug 2016 19:54:11 +0200 Subject: [PATCH 23/60] description provided, ChangeLog entries added --- ChangeLog | 3 +++ fail2ban/server/datetemplate.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ecb8cc87..c4444e2a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,7 @@ releases. uses "fail2ban-python" now); * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); +* Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); * `filter.d/ignorecommands/apache-fakegooglebot` - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) @@ -33,6 +34,8 @@ releases. ### New Features ### Enhancements +* DateTemplate regexp extended with the word-end boundary, additionally to + word-start boundary * Introduces new command "fail2ban-python", as automatically created symlink to python executable, where fail2ban currently installed (resp. its modules are located): - allows to use the same version, fail2ban currently running, e.g. in diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 0a5ec55a..8e602289 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -72,8 +72,12 @@ class DateTemplate(object): regex : str The regex the template will use for searching for a date. wordBegin : bool - Defines whether the regex should be modified to search at - beginning of a word, by adding "\\b" to start of regex. + Defines whether the regex should be modified to search at beginning of a + word, by adding special boundary r'(?=^|\b|\W)' to start of regex. + Default True. + wordEnd : bool + Defines whether the regex should be modified to search at end of a word, + by adding special boundary r'(?=\b|\W|$)' to end of regex. Default True. Raises From a544c5abac03dbc124708afbcd1912cbd9f2686a Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 8 Jul 2016 12:22:38 +0200 Subject: [PATCH 24/60] sshd-filter: recognized "Failed publickey for" now (gh-1477) + improved regexp (not anchored now to recognize all "Failed anything for ... from " ChangeLog entry added --- ChangeLog | 3 +++ config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index c4444e2a..f441b99b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,9 @@ releases. - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) * `filter.d/postfix-sasl.conf` - Allow for having no trailing space after 'failed:' (gh-1497) +* filter.d/sshd.conf + - recognized "Failed publickey for" (gh-1477); + - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index eeb1518e..7003d1b2 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$ + ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(?:: | (?!from )|$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7baf4be7..a6b07222 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -118,6 +118,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } +Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI # failJSON: { "match": false } Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth] @@ -161,4 +163,3 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root - From 2c54f9046970046265f6f6f5d18cdca19e331940 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 18 Aug 2016 21:34:09 +0200 Subject: [PATCH 25/60] sshd-filter: better universal regexp, that matches more complex different injects, using conditional expressions (on username and auth-info section), see new test cases also. --- ChangeLog | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f441b99b..2bc060ee 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,7 @@ releases. * filter.d/sshd.conf - recognized "Failed publickey for" (gh-1477); - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) + - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 7003d1b2..9a3d40f0 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(?:: | (?!from )|$) + ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index a6b07222..c53bbadb 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -117,6 +117,10 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2004-11-11T08:04:52", "match": true , "host": "127.0.0.1", "desc": "More complex injecting on username ssh 'test from 10.10.1.2 port 55555 ssh2'@localhost" } +Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from 10.10.1.2 port 55555 ssh2 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "time": "2004-11-11T08:04:52", "match": true , "host": "127.0.0.1", "desc": "More complex injecting on auth-info ssh test@localhost, auth-info: ' from 10.10.1.2 port 55555 ssh2'" } +Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from 127.0.0.1 port 58946 ssh2: from 10.10.1.2 port 55555 ssh2 # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI From 4a1d7203446a47f0d202aa7f7bac97d86f0e8eab Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Aug 2016 12:55:03 +0200 Subject: [PATCH 26/60] filter.d/asterisk.conf: another part ` chan_sip.c:28468 handle_request_register:` in log prefix --- ChangeLog | 2 ++ config/filter.d/asterisk.conf | 2 +- fail2ban/tests/files/logs/asterisk | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index c4444e2a..7405749c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,8 @@ releases. * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); * Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); +* `filter.d/asterisk.conf` + - Fixed to match different asterisk log prefix (source file: method:) * `filter.d/ignorecommands/apache-fakegooglebot` - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 6af452e2..79bd1ff2 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -16,7 +16,7 @@ __pid_re = (?:\[\d+\]) iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} # All Asterisk log messages begin like this: -log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*( in \w+:)? +log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)? failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index d17d93a1..13808592 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -43,6 +43,8 @@ # failJSON: { "time": "2004-11-04T18:30:40", "match": true , "host": "192.168.200.100" } Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in handle_request_register: Registration from '' failed for '192.168.200.100:36998' - Wrong password +# failJSON: { "time": "2016-08-19T11:11:26", "match": true , "host": "192.0.2.1", "desc": "Another log_prefix used (` in` should be optional)" } +[2016-08-19 11:11:26] NOTICE[12931]: chan_sip.c:28468 handle_request_register: Registration from 'sip:bob@192.0.2.1' failed for '192.0.2.1:42406' - Wrong password # failed authentication attempt on INVITE using PJSIP # failJSON: { "time": "2015-05-24T08:42:16", "match": true, "host": "10.250.251.252" } From 1c4733ef89446c42a01a09c2f781d55d4303b246 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Aug 2016 18:29:12 +0200 Subject: [PATCH 27/60] [systemd] added new constructor parameters like journalpath, journalfiles and journalflags for systemd backup optimized FilterSystemd method `run`: better wait in idle (no busy-loop), better poll handling, the ban will executed anywhere (at least at 100th log-entry), also if we have never ending logging in this jail (e.g. extremely logging or too many failures) systemd test cases extended --- fail2ban/server/filtersystemd.py | 97 +++++++++++++++++++++++--------- fail2ban/tests/filtertestcase.py | 25 +++++--- 2 files changed, 88 insertions(+), 34 deletions(-) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index d0ebec95..9c4c7865 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -33,7 +33,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': from .failmanager import FailManagerEmpty from .filter import JournalFilter from .mytime import MyTime -from ..helpers import getLogger +from ..helpers import getLogger, logging, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) @@ -54,14 +54,45 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # @param jail the jail object def __init__(self, jail, **kwargs): + jrnlargs = FilterSystemd._getJournalArgs(kwargs) JournalFilter.__init__(self, jail, **kwargs) - self.__modified = False + self.__modified = 0 # Initialise systemd-journal connection - self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x}) + self.__journal = journal.Reader(**jrnlargs) self.__matches = [] self.setDatePattern(None) + self.ticks = 0 logSys.debug("Created FilterSystemd") + @staticmethod + def _getJournalArgs(kwargs): + args = {'converters':{'__CURSOR': lambda x: x}} + try: + args['path'] = kwargs.pop('journalpath') + except KeyError: + pass + + try: + args['files'] = kwargs.pop('journalfiles') + except KeyError: + pass + else: + import glob + p = args['files'] + if not isinstance(p, (list, set, tuple)): + p = splitwords(p) + files = [] + for p in p: + files.extend(glob.glob(p)) + args['files'] = list(set(files)) + + try: + args['flags'] = kwargs.pop('journalflags') + except KeyError: + pass + + return args + ## # Add a journal match filters from list structure # @@ -207,6 +238,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover return (('', date.isoformat(), logline), time.mktime(date.timetuple()) + date.microsecond/1.0E6) + def seekToTime(self, date): + if not isinstance(date, datetime.datetime): + date = datetime.datetime.fromtimestamp(date) + self.__journal.seek_realtime(date) + ## # Main loop. # @@ -224,7 +260,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # Seek to now - findtime in journal start_time = datetime.datetime.now() - \ datetime.timedelta(seconds=int(self.getFindTime())) - self.__journal.seek_realtime(start_time) + self.seekToTime(start_time) # Move back one entry to ensure do not end up in dead space # if start time beyond end of journal try: @@ -233,29 +269,38 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover pass # Reading failure, so safe to ignore while self.active: - if not self.idle: - while self.active: - try: - logentry = self.__journal.get_next() - except OSError: - logSys.warning( - "Error reading line from systemd journal") - continue - if logentry: - self.processLineAndAdd( - *self.formatJournalEntry(logentry)) - self.__modified = True - else: - break - if self.__modified: - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False + # wait for records (or for timeout in sleeptime seconds): self.__journal.wait(self.sleeptime) + if self.idle: + # because journal.wait will returns immediatelly if we have records in journal, + # just wait a little bit here for not idle, to prevent hi-load: + time.sleep(self.sleeptime) + continue + self.__modified = 0 + while self.active: + logentry = None + try: + logentry = self.__journal.get_next() + except OSError as e: + logSys.error("Error reading line from systemd journal: %s", + e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) + self.ticks += 1 + if logentry: + self.processLineAndAdd( + *self.formatJournalEntry(logentry)) + self.__modified += 1 + if self.__modified >= 100: # todo: should be configurable + break + else: + break + if self.__modified: + try: + while True: + ticket = self.failManager.toBan() + self.jail.putFailTicket(ticket) + except FailManagerEmpty: + self.failManager.cleanup(MyTime.time()) + logSys.debug((self.jail is not None and self.jail.name or "jailless") +" filter terminated") return True diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 40879b66..6194243b 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -707,11 +707,16 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover """Call before every test case.""" self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log") self.jail = DummyJail() - self.filter = Filter_(self.jail) + self.filter = None # UUID used to ensure that only meeages generated # as part of this test are picked up by the filter self.test_uuid = str(uuid.uuid4()) self.name = "monitorjournalfailures-%s" % self.test_uuid + self.journal_fields = { + 'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid} + + def _initFilter(self, **kwargs): + self.filter = Filter_(self.jail, **kwargs) self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -720,16 +725,16 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=2", "TEST_UUID=%s" % self.test_uuid]) - self.journal_fields = { - 'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid} - self.filter.active = True self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") - self.filter.start() def tearDown(self): - self.filter.stop() - self.filter.join() # wait for the thread to terminate - pass + if self.filter and self.filter.active: + self.filter.stop() + self.filter.join() # wait for the thread to terminate + pass + + def testJournalFlagsArg(self): + self._initFilter(journalflags=0) # journal.RUNTIME_ONLY def __str__(self): return "MonitorJournalFailures%s(%s)" \ @@ -761,6 +766,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.assertEqual(attempts, test_attempts) def test_grow_file(self): + self._initFilter() + self.filter.start() self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file @@ -790,6 +797,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.assert_correct_ban("193.168.0.128", 3) def test_delJournalMatch(self): + self._initFilter() + self.filter.start() # Smoke test for removing of match # basic full test From 7ed6cab1203c33d27a18e43c5809fbdc9e4d224a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Aug 2016 19:09:47 +0200 Subject: [PATCH 28/60] jail configuration extended with new syntax to pass options to the backend (see gh-1408), examples: - `backend = systemd[journalpath=/run/log/journal/machine-1]` - `backend = systemd[journalfiles="/run/log/journal/machine-1/system.journal, /run/log/journal/machine-1/user.journal"]` - `backend = systemd[journalflags=2]` --- ChangeLog | 5 +++++ fail2ban/server/jail.py | 28 +++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7405749c..5f97c996 100644 --- a/ChangeLog +++ b/ChangeLog @@ -54,6 +54,11 @@ releases. * New forward compatibility method assertRaisesRegexp (normally python >= 2.7). Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged are test covered now +* Jail configuration extended with new syntax to pass options to the backend (see gh-1408), + examples: + - `backend = systemd[journalpath=/run/log/journal/machine-1]` + - `backend = systemd[journalfiles="/run/log/journal/machine-1/system.journal, /run/log/journal/machine-1/user.journal"]` + - `backend = systemd[journalflags=2]` ver. 0.9.5 (2016/07/15) - old-not-obsolete diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index a866cb51..951a9891 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -27,6 +27,7 @@ import logging import Queue from .actions import Actions +from ..client.jailreader import JailReader from ..helpers import getLogger # Gets the instance of the logger. @@ -82,6 +83,7 @@ class Jail: return "%s(%r)" % (self.__class__.__name__, self.name) def _setBackend(self, backend): + backend, beArgs = JailReader.extractOptions(backend) backend = backend.lower() # to assure consistent matching backends = self._BACKENDS @@ -98,7 +100,7 @@ class Jail: for b in backends: initmethod = getattr(self, '_init%s' % b.capitalize()) try: - initmethod() + initmethod(**beArgs) if backend != 'auto' and b != backend: logSys.warning("Could only initiated %r backend whenever " "%r was requested" % (b, backend)) @@ -117,28 +119,28 @@ class Jail: raise RuntimeError( "Failed to initialize any backend for Jail %r" % self.name) - def _initPolling(self): + def _initPolling(self, **kwargs): from filterpoll import FilterPoll - logSys.info("Jail '%s' uses poller" % self.name) - self.__filter = FilterPoll(self) + logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs)) + self.__filter = FilterPoll(self, **kwargs) - def _initGamin(self): + def _initGamin(self, **kwargs): # Try to import gamin from filtergamin import FilterGamin - logSys.info("Jail '%s' uses Gamin" % self.name) - self.__filter = FilterGamin(self) + logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs)) + self.__filter = FilterGamin(self, **kwargs) - def _initPyinotify(self): + def _initPyinotify(self, **kwargs): # Try to import pyinotify from filterpyinotify import FilterPyinotify - logSys.info("Jail '%s' uses pyinotify" % self.name) - self.__filter = FilterPyinotify(self) + logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs)) + self.__filter = FilterPyinotify(self, **kwargs) - def _initSystemd(self): # pragma: systemd no cover + def _initSystemd(self, **kwargs): # pragma: systemd no cover # Try to import systemd from filtersystemd import FilterSystemd - logSys.info("Jail '%s' uses systemd" % self.name) - self.__filter = FilterSystemd(self) + logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs)) + self.__filter = FilterSystemd(self, **kwargs) @property def name(self): From 35b5fea038ca76a19b40b40a162affaa9bd6874c Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 1 Sep 2016 15:44:32 +0200 Subject: [PATCH 29/60] backend "systemd" can be used as prefix now - `backend = systemd[...]` --- fail2ban/client/jailreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index c86c3153..b29d12fa 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -192,7 +192,7 @@ class JailReader(ConfigReader): stream = [] for opt in self.__opts: if opt == "logpath" and \ - self.__opts.get('backend', None) != "systemd": + not self.__opts.get('backend', None).startswith("systemd"): found_files = 0 for path in self.__opts[opt].split("\n"): path = path.rsplit(" ", 1) From 5f35b52b9af46edab728742488e8a8afeed850ed Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 1 Sep 2016 15:47:17 +0200 Subject: [PATCH 30/60] test cases extended several test-case functionality cherry picked from 0.10 (SkipTest, with_tmpdir) --- fail2ban/tests/clientreadertestcase.py | 54 +++++++++++++++++++------- fail2ban/tests/databasetestcase.py | 4 +- fail2ban/tests/filtertestcase.py | 2 +- fail2ban/tests/servertestcase.py | 9 +---- fail2ban/tests/utils.py | 41 ++++++++++++++++++- 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ee362e3d..dd98374c 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -36,7 +36,7 @@ from ..client.jailsreader import JailsReader from ..client.actionreader import ActionReader from ..client.configurator import Configurator from ..version import version -from .utils import LogCaptureTestCase +from .utils import LogCaptureTestCase, with_tmpdir TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -94,9 +94,8 @@ option = %s if not os.access(f, os.R_OK): self.assertFalse(self.c.read('d')) # should not be readable BUT present else: - # SkipTest introduced only in 2.7 thus can't yet use generally - # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) - pass + import platform + raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform.platform()) def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet @@ -281,8 +280,8 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) - def testGlob(self): - d = tempfile.mkdtemp(prefix="f2b-temp") + @with_tmpdir + def testGlob(self, d): # Generate few files # regular file f1 = os.path.join(d, 'f1') @@ -297,9 +296,6 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(JailReader._glob(f2), []) self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) - os.remove(f1) - os.remove(f2) - os.rmdir(d) class FilterReaderTest(unittest.TestCase): @@ -433,10 +429,10 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt += 1 return cnt - def testTestJailConfCache(self): + @with_tmpdir + def testTestJailConfCache(self, basedir): saved_ll = configparserinc.logLevel configparserinc.logLevel = logging.DEBUG - basedir = tempfile.mkdtemp("fail2ban_conf") try: shutil.rmtree(basedir) shutil.copytree(CONFIG_DIR, basedir) @@ -468,7 +464,6 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: - shutil.rmtree(basedir) configparserinc.logLevel = saved_ll @@ -718,8 +713,8 @@ class JailsReaderTest(LogCaptureTestCase): self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp') self.assertEqual(configurator.getBaseDir(), CONFIG_DIR) - def testMultipleSameAction(self): - basedir = tempfile.mkdtemp("fail2ban_conf") + @with_tmpdir + def testMultipleSameAction(self, basedir): os.mkdir(os.path.join(basedir, "filter.d")) os.mkdir(os.path.join(basedir, "action.d")) open(os.path.join(basedir, "action.d", "testaction1.conf"), 'w').close() @@ -748,4 +743,33 @@ filter = testfilter1 # Python actions should not be passed `actname` self.assertEqual(add_actions[-1][-1], "{}") - shutil.rmtree(basedir) + def testLogPathFileFilterBackend(self): + self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", + self._testLogPath, backend='polling') + + def testLogPathSystemdBackend(self): + try: # pragma: systemd no cover + from ..server.filtersystemd import FilterSystemd + except Exception, e: # pragma: no cover + raise unittest.SkipTest("systemd python interface not available") + self._testLogPath(backend='systemd') + self._testLogPath(backend='systemd[journalflags=2]') + + @with_tmpdir + def _testLogPath(self, basedir, backend): + jailfd = open(os.path.join(basedir, "jail.conf"), 'w') + jailfd.write(""" +[testjail1] +enabled = true +backend = %s +logpath = %s/not/exist.log + /this/path/should/not/exist.log +action = +filter = +failregex = test +""" % (backend, basedir)) + jailfd.close() + jails = JailsReader(basedir=basedir) + self.assertTrue(jails.read()) + self.assertTrue(jails.getOptions()) + jails.convert() diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index e934ba45..2b599f61 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -48,12 +48,10 @@ class DatabaseTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" super(DatabaseTest, self).setUp() - if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover + if Fail2BanDb is None: # pragma: no cover raise unittest.SkipTest( "Unable to import fail2ban database module as sqlite is not " "available.") - elif Fail2BanDb is None: - return _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") self.db = Fail2BanDb(self.dbFilename) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 6194243b..4f9d8a06 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -734,7 +734,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover pass def testJournalFlagsArg(self): - self._initFilter(journalflags=0) # journal.RUNTIME_ONLY + self._initFilter(journalflags=2) # journal.RUNTIME_ONLY def __str__(self): return "MonitorJournalFailures%s(%s)" \ diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 21e2d784..99502794 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -688,10 +688,7 @@ class Transmitter(TransmitterBase): def testJournalMatch(self): if not filtersystemd: # pragma: no cover - if sys.version_info >= (2, 7): - raise unittest.SkipTest( - "systemd python interface not available") - return + raise unittest.SkipTest("systemd python interface not available") jailName = "TestJail2" self.server.addJail(jailName, "systemd") values = [ @@ -791,10 +788,8 @@ class TransmitterLogging(TransmitterBase): self.setGetTest("logtarget", "STDERR") def testLogTargetSYSLOG(self): - if not os.path.exists("/dev/log") and sys.version_info >= (2, 7): + if not os.path.exists("/dev/log"): raise unittest.SkipTest("'/dev/log' not present") - elif not os.path.exists("/dev/log"): - return self.assertTrue(self.server.getSyslogSocket(), "auto") self.setGetTest("logtarget", "SYSLOG") self.assertTrue(self.server.getSyslogSocket(), "/dev/log") diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e091c935..05d9d5ed 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -26,10 +26,13 @@ import itertools import logging import os import re +import tempfile +import shutil import sys import time import unittest from StringIO import StringIO +from functools import wraps from ..server.mytime import MyTime from ..helpers import getLogger @@ -46,6 +49,40 @@ if not CONFIG_DIR: CONFIG_DIR = '/etc/fail2ban' +def with_tmpdir(f): + """Helper decorator to create a temporary directory + + Directory gets removed after function returns, regardless + if exception was thrown of not + """ + @wraps(f) + def wrapper(self, *args, **kwargs): + tmp = tempfile.mkdtemp(prefix="f2b-temp") + try: + return f(self, tmp, *args, **kwargs) + finally: + # clean up + shutil.rmtree(tmp) + return wrapper + + +# backwards compatibility to python 2.6: +if not hasattr(unittest, 'SkipTest'): # pragma: no cover + class SkipTest(Exception): + pass + unittest.SkipTest = SkipTest + _org_AddError = unittest._TextTestResult.addError + def addError(self, test, err): + if err[0] is SkipTest: + if self.showAll: + self.stream.writeln(str(err[1])) + elif self.dots: + self.stream.write('s') + self.stream.flush() + return + _org_AddError(self, test, err) + unittest._TextTestResult.addError = addError + def mtimesleep(): # no sleep now should be necessary since polling tracks now not only # mtime but also ino and size @@ -218,8 +255,8 @@ if not hasattr(unittest.TestCase, 'assertRaisesRegexp'): try: fun(*args, **kwargs) except exccls as e: - if re.search(regexp, e.message) is None: - self.fail('\"%s\" does not match \"%s\"' % (regexp, e.message)) + if re.search(regexp, str(e)) is None: + self.fail('\"%s\" does not match \"%s\"' % (regexp, e)) else: self.fail('%s not raised' % getattr(exccls, '__name__')) unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp From b875e51cd7b6cf4e257f88ce0a5c01b58a765e15 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 4 Sep 2016 23:25:09 -0400 Subject: [PATCH 31/60] RF: Replace old fashioned "except E , e" with "except E as e" (Closes #1537) --- bin/fail2ban-client | 6 +++--- bin/fail2ban-server | 2 +- fail2ban/client/configparserinc.py | 2 +- fail2ban/client/configreader.py | 2 +- fail2ban/client/fail2banregex.py | 6 +++--- fail2ban/client/jailreader.py | 2 +- fail2ban/server/database.py | 10 +++++----- fail2ban/server/filter.py | 18 +++++++++--------- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 2 +- fail2ban/server/jail.py | 2 +- fail2ban/server/server.py | 10 +++++----- fail2ban/server/transmitter.py | 2 +- fail2ban/tests/clientreadertestcase.py | 2 +- fail2ban/tests/samplestestcase.py | 2 +- fail2ban/tests/utils.py | 6 +++--- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index bc0c0be8..dc2f7d84 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -176,7 +176,7 @@ class Fail2banClient: if showRet: self.__logSocketError() return False - except Exception, e: + except Exception as e: if showRet: logSys.error(e) return False @@ -429,7 +429,7 @@ class Fail2banClient: elif not cmd == "": try: self.__processCommand(shlex.split(cmd)) - except Exception, e: + except Exception as e: logSys.error(e) except (EOFError, KeyboardInterrupt): print @@ -451,7 +451,7 @@ class Fail2banClient: ret = self.__configurator.getOptions(jail) self.__configurator.convertToProtocol() self.__stream = self.__configurator.getConfigStream() - except Exception, e: + except Exception as e: logSys.error("Failed during configuration: %s" % e) ret = False return ret diff --git a/bin/fail2ban-server b/bin/fail2ban-server index f522f418..23eae136 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -127,7 +127,7 @@ class Fail2banServer: self.__conf["pidfile"], self.__conf["force"]) return True - except Exception, e: + except Exception as e: logSys.exception(e) self.__server.quit() return False diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7bbc7886..81f5bfb3 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -168,7 +168,7 @@ after = 1.conf parser, i = self._getSharedSCPWI(resource) if not i: return [] - except UnicodeDecodeError, e: + except UnicodeDecodeError as e: logSys.error("Error decoding config file '%s': %s" % (resource, e)) return [] diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index c6dd1b60..a48a0cc6 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -221,7 +221,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if not pOptions is None and option[1] in pOptions: continue values[option[1]] = v - except NoSectionError, e: + except NoSectionError as e: # No "Definition" section or wrong basedir logSys.error(e) values[option[1]] = option[2] diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index c2b2e894..71f50955 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -329,7 +329,7 @@ class Fail2banRegex(object): if ret is not None: found = True regex = self._ignoreregex[ret].inc() - except RegexException, e: + except RegexException as e: output( e ) return False return found @@ -346,7 +346,7 @@ class Fail2banRegex(object): regex = self._failregex[match[0]] regex.inc() regex.appendIP(match) - except RegexException, e: + except RegexException as e: output( e ) return False except IndexError: @@ -510,7 +510,7 @@ class Fail2banRegex(object): output( "Use log file : %s" % cmd_log ) output( "Use encoding : %s" % self.encoding ) test_lines = self.file_lines_gen(hdlr) - except IOError, e: + except IOError as e: output( e ) return False elif cmd_log == "systemd-journal": # pragma: no cover diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index c86c3153..01765148 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -171,7 +171,7 @@ class JailReader(ConfigReader): self.__actions.append(action) else: raise AttributeError("Unable to read action") - except Exception, e: + except Exception as e: logSys.error("Error in action definition " + act) logSys.debug("Caught exception: %s" % (e,)) return False diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 9f562511..34a8b39a 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -42,7 +42,7 @@ if sys.version_info >= (3,): try: x = json.dumps(x, ensure_ascii=False).encode( locale.getpreferredencoding(), 'replace') - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.error('json dumps failed: %s', e) x = '{}' return x @@ -51,7 +51,7 @@ if sys.version_info >= (3,): try: x = json.loads(x.decode( locale.getpreferredencoding(), 'replace')) - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.error('json loads failed: %s', e) x = {} return x @@ -70,7 +70,7 @@ else: try: x = json.dumps(_normalize(x), ensure_ascii=False).decode( locale.getpreferredencoding(), 'replace') - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.error('json dumps failed: %s', e) x = '{}' return x @@ -79,7 +79,7 @@ else: try: x = _normalize(json.loads(x.decode( locale.getpreferredencoding(), 'replace'))) - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.error('json loads failed: %s', e) x = {} return x @@ -175,7 +175,7 @@ class Fail2BanDb(object): logSys.info( "Connected to fail2ban persistent database '%s'", filename) - except sqlite3.OperationalError, e: + except sqlite3.OperationalError as e: logSys.error( "Error connecting to fail2ban persistent database '%s': %s", filename, e.args[0]) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 45ae7704..98d0bce3 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -105,7 +105,7 @@ class Filter(JailThread): logSys.warning( "Mutliline regex set for jail '%s' " "but maxlines not greater than 1") - except RegexException, e: + except RegexException as e: logSys.error(e) raise e @@ -138,7 +138,7 @@ class Filter(JailThread): try: regex = Regex(value) self.__ignoreRegex.append(regex) - except RegexException, e: + except RegexException as e: logSys.error(e) raise e @@ -536,7 +536,7 @@ class Filter(JailThread): failRegex.getMatchedLines()]) if not checkAllRegex: break - except RegexException, e: # pragma: no cover - unsure if reachable + except RegexException as e: # pragma: no cover - unsure if reachable logSys.error(e) return failList @@ -660,15 +660,15 @@ class FileFilter(Filter): try: has_content = log.open() # see http://python.org/dev/peps/pep-3151/ - except IOError, e: + except IOError as e: logSys.error("Unable to open %s" % filename) logSys.exception(e) return False - except OSError, e: # pragma: no cover - requires race condition to tigger this + except OSError as e: # pragma: no cover - requires race condition to tigger this logSys.error("Error opening %s" % filename) logSys.exception(e) return False - except Exception, e: # pragma: no cover - Requires implemention error in FileContainer to generate + except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate logSys.error("Internal errror in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues") logSys.exception(e) return False @@ -858,11 +858,11 @@ class DNSUtils: # retrieve ip (todo: use AF_INET6 for IPv6) try: return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)]) - except socket.error, e: + except socket.error as e: logSys.warning("Unable to find a corresponding IP address for %s: %s" % (dns, e)) return list() - except socket.error, e: + except socket.error as e: logSys.warning("Socket error raised trying to resolve hostname %s: %s" % (dns, e)) return list() @@ -871,7 +871,7 @@ class DNSUtils: def ipToName(ip): try: return socket.gethostbyaddr(ip)[0] - except socket.error, e: + except socket.error as e: logSys.debug("Unable to find a name for the IP %s: %s" % (ip, e)) return None diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index d0b37775..6d9f84b3 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -138,7 +138,7 @@ class FilterPoll(FileFilter): logSys.debug("%s has been modified", filename) self.__prevStats[filename] = stats return True - except OSError, e: + except OSError as e: logSys.error("Unable to get stat on %s because of: %s" % (filename, e)) self.__file404Cnt[filename] += 1 diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 100ad233..d551e0f4 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -44,7 +44,7 @@ if not hasattr(pyinotify, '__version__') \ try: manager = pyinotify.WatchManager() del manager -except Exception, e: +except Exception as e: raise ImportError("Pyinotify is probably not functional on this system: %s" % str(e)) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index a866cb51..d5a0384f 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -106,7 +106,7 @@ class Jail: logSys.info("Initiated %r backend" % b) self.__actions = Actions(self) return # we are done - except ImportError, e: + except ImportError as e: # Log debug if auto, but error if specific logSys.log( logging.DEBUG if backend == "auto" else logging.ERROR, diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 3e371945..f3d44829 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -108,20 +108,20 @@ class Server: pidFile = open(pidfile, 'w') pidFile.write("%s\n" % os.getpid()) pidFile.close() - except IOError, e: + except IOError as e: logSys.error("Unable to create PID file: %s" % e) # Start the communication logSys.debug("Starting communication") try: self.__asyncServer.start(sock, force) - except AsyncServerException, e: + except AsyncServerException as e: logSys.error("Could not start server: %s", e) # Removes the PID file. try: logSys.debug("Remove PID file %s" % pidfile) os.remove(pidfile) - except OSError, e: + except OSError as e: logSys.error("Unable to remove PID file: %s" % e) logSys.info("Exiting Fail2ban") @@ -543,7 +543,7 @@ class Server: # the child gets a new PID, making it impossible for its PID to equal its # PGID. pid = os.fork() - except OSError, e: + except OSError as e: return((e.errno, e.strerror)) # ERROR (return a tuple) if pid == 0: # The first child. @@ -564,7 +564,7 @@ class Server: # fork guarantees that the child is no longer a session leader, thus # preventing the daemon from ever acquiring a controlling terminal. pid = os.fork() # Fork a second child. - except OSError, e: + except OSError as e: return((e.errno, e.strerror)) # ERROR (return a tuple) if (pid == 0): # The second child. diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 0d9f0fe4..636b90da 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -56,7 +56,7 @@ class Transmitter: try: ret = self.__commandHandler(command) ack = 0, ret - except Exception, e: + except Exception as e: logSys.warning("Command %r has failed. Received %r" % (command, e)) ack = 1, e diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ee362e3d..8b6564a3 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -410,7 +410,7 @@ class FilterReaderTest(unittest.TestCase): # from testcase01 filterReader.get('Definition', 'failregex') filterReader.get('Definition', 'ignoreregex') - except Exception, e: # pragma: no cover - failed if reachable + except Exception as e: # pragma: no cover - failed if reachable self.fail('unexpected options after readexplicit: %s' % (e)) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index a40a11da..327410bc 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -94,7 +94,7 @@ def testSampleRegexsFactory(name, basedir): if jsonREMatch: try: faildata = json.loads(jsonREMatch.group(1)) - except ValueError, e: + except ValueError as e: raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) line = next(logFile) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e091c935..ee641d41 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -184,13 +184,13 @@ def gatherTests(regexps=None, no_network=False): try: from ..server.filtergamin import FilterGamin filters.append(FilterGamin) - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.warning("Skipping gamin backend testing. Got exception '%s'" % e) try: from ..server.filterpyinotify import FilterPyinotify filters.append(FilterPyinotify) - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.warning("I: Skipping pyinotify backend testing. Got exception '%s'" % e) for Filter_ in filters: @@ -199,7 +199,7 @@ def gatherTests(regexps=None, no_network=False): try: # pragma: systemd no cover from ..server.filtersystemd import FilterSystemd tests.addTest(unittest.makeSuite(filtertestcase.get_monitor_failures_journal_testcase(FilterSystemd))) - except Exception, e: # pragma: no cover + except Exception as e: # pragma: no cover logSys.warning("I: Skipping systemd backend testing. Got exception '%s'" % e) # Server test for logging elements which break logging used to support From 3119f8170515bf7539c3e77ffd7cea55f7212c11 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 5 Sep 2016 19:44:32 +0200 Subject: [PATCH 32/60] fixed journal systemd ascii/utf-8 default converting (see gh-1341, gh-1344) --- fail2ban/server/filter.py | 66 ++++++++++++++---- fail2ban/server/filtersystemd.py | 82 +++++++++-------------- fail2ban/tests/files/testcase-journal.log | 8 +-- fail2ban/tests/filtertestcase.py | 50 ++++++++++++-- 4 files changed, 136 insertions(+), 70 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 98d0bce3..8d3f9014 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -24,6 +24,7 @@ __license__ = "GPL" import codecs import fcntl import locale +import logging import os import re import sys @@ -394,11 +395,35 @@ class Filter(JailThread): return False + if sys.version_info >= (3,): + @staticmethod + def uni_decode(x, enc, errors='strict'): + try: + if isinstance(x, bytes): + return x.decode(enc, errors) + return x + except (UnicodeDecodeError, UnicodeEncodeError): + if errors != 'strict': # pragma: no cover - unsure if reachable + raise + return uni_decode(x, enc, 'replace') + else: + @staticmethod + def uni_decode(x, enc, errors='strict'): + try: + if isinstance(x, unicode): + return x.encode(enc, errors) + return x + except (UnicodeDecodeError, UnicodeEncodeError): + if errors != 'strict': # pragma: no cover - unsure if reachable + raise + return uni_decode(x, enc, 'replace') + def processLine(self, line, date=None, returnRawHost=False, - checkAllRegex=False): + checkAllRegex=False, checkFindTime=False): """Split the time portion from log msg and return findFailures on them """ if date: + # be sure each element of tuple line has the same type: tupleLine = line else: l = line.rstrip('\r\n') @@ -414,7 +439,7 @@ class Filter(JailThread): tupleLine = (l, "", "") return "".join(tupleLine[::2]), self.findFailure( - tupleLine, date, returnRawHost, checkAllRegex) + tupleLine, date, returnRawHost, checkAllRegex, checkFindTime) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager @@ -457,7 +482,7 @@ class Filter(JailThread): # @return a dict with IP and timestamp. def findFailure(self, tupleLine, date=None, returnRawHost=False, - checkAllRegex=False): + checkAllRegex=False, checkFindTime=False): failList = list() # Checks if we must ignore this line. @@ -489,6 +514,11 @@ class Filter(JailThread): timeText = self.__lastTimeText or "".join(tupleLine[::2]) date = self.__lastDate + if checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): + logSys.log(5, "Ignore line since time %s < %s - %s", + date, MyTime.time(), self.getFindTime()) + return failList + self.__lineBuffer = ( self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:] logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer) @@ -791,17 +821,25 @@ class FileContainer: @staticmethod def decode_line(filename, enc, line): try: - line = line.decode(enc, 'strict') - except UnicodeDecodeError: - logSys.warning( - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r" % - (filename, enc, line)) + return line.decode(enc, 'strict') + except UnicodeDecodeError as e: # decode with replacing error chars: - line = line.decode(enc, 'replace') - return line + rline = line.decode(enc, 'replace') + except UnicodeEncodeError as e: + # encode with replacing error chars: + rline = line.decode(enc, 'replace') + global _decode_line_warn + lev = logging.DEBUG + if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = logging.WARNING + _decode_line_warn[filename] = MyTime.time() + 24*60*60 + logSys.log(lev, + "Error decoding line from '%s' with '%s'." + " Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r", + filename, enc, line) + return rline def readline(self): if self.__handler is None: @@ -819,6 +857,8 @@ class FileContainer: ## print "D: Closed %s with pos %d" % (handler, self.__pos) ## sys.stdout.flush() +_decode_line_warn = {} + ## # JournalFilter class. diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 9c4c7865..5c5481b9 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -31,7 +31,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': raise ImportError("Fail2Ban requires systemd >= 204") from .failmanager import FailManagerEmpty -from .filter import JournalFilter +from .filter import JournalFilter, Filter, locale from .mytime import MyTime from ..helpers import getLogger, logging, splitwords @@ -170,21 +170,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalMatch(self): return self.__matches - ## - # Join group of log elements which may be a mix of bytes and strings - # - # @param elements list of strings and bytes - # @return elements joined as string - @staticmethod - def _joinStrAndBytes(elements): - strElements = [] - for element in elements: - if isinstance(element, str): - strElements.append(element) - else: - strElements.append(str(element, errors='ignore')) - return " ".join(strElements) + def uni_decode(x): + v = Filter.uni_decode(x, locale.getpreferredencoding()) + return v ## # Format journal log entry into syslog style @@ -194,48 +183,43 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover @classmethod def formatJournalEntry(cls, logentry): - logelements = [""] - if logentry.get('_HOSTNAME'): - logelements.append(logentry['_HOSTNAME']) - if logentry.get('SYSLOG_IDENTIFIER'): - logelements.append(logentry['SYSLOG_IDENTIFIER']) - if logentry.get('SYSLOG_PID'): - logelements[-1] += ("[%i]" % logentry['SYSLOG_PID']) - elif logentry.get('_PID'): - logelements[-1] += ("[%i]" % logentry['_PID']) + # Be sure, all argument of line tuple should have the same type: + uni_decode = FilterSystemd.uni_decode + logelements = [] + v = logentry.get('_HOSTNAME') + if v: + logelements.append(uni_decode(v)) + v = logentry.get('SYSLOG_IDENTIFIER') + if not v: + v = logentry.get('_COMM') + if v: + logelements.append(uni_decode(v)) + v = logentry.get('SYSLOG_PID') + if not v: + v = logentry.get('_PID') + if v: + logelements[-1] += ("[%i]" % v) logelements[-1] += ":" - elif logentry.get('_COMM'): - logelements.append(logentry['_COMM']) - if logentry.get('_PID'): - logelements[-1] += ("[%i]" % logentry['_PID']) - logelements[-1] += ":" - if logelements[-1] == "kernel:": - if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: - monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') - else: - monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] - logelements.append("[%12.6f]" % monotonic.total_seconds()) - if isinstance(logentry.get('MESSAGE',''), list): - logelements.append(" ".join(logentry['MESSAGE'])) + if logelements[-1] == "kernel:": + if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: + monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') + else: + monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] + logelements.append("[%12.6f]" % monotonic.total_seconds()) + msg = logentry.get('MESSAGE','') + if isinstance(msg, list): + logelements.append(" ".join(uni_decode(v) for v in msg)) else: - logelements.append(logentry.get('MESSAGE', '')) + logelements.append(uni_decode(msg)) - try: - logline = u" ".join(logelements) - except UnicodeDecodeError: - # Python 2, so treat as string - logline = " ".join([str(logline) for logline in logelements]) - except TypeError: - # Python 3, one or more elements bytes - logSys.warning("Error decoding log elements from journal: %s" % - repr(logelements)) - logline = cls._joinStrAndBytes(logelements) + logline = " ".join(logelements) date = logentry.get('_SOURCE_REALTIME_TIMESTAMP', logentry.get('__REALTIME_TIMESTAMP')) logSys.debug("Read systemd journal entry: %r" % "".join([date.isoformat(), logline])) - return (('', date.isoformat(), logline), + ## use the same type for 1st argument: + return ((logline[:0], date.isoformat(), logline), time.mktime(date.timetuple()) + date.microsecond/1.0E6) def seekToTime(self, date): diff --git a/fail2ban/tests/files/testcase-journal.log b/fail2ban/tests/files/testcase-journal.log index 720a3130..b6fa427b 100644 --- a/fail2ban/tests/files/testcase-journal.log +++ b/fail2ban/tests/files/testcase-journal.log @@ -13,7 +13,7 @@ error: PAM: Authentication failure for kevin from 193.168.0.128 error: PAM: Authentication failure for kevin from 193.168.0.128 error: PAM: Authentication failure for kevin from 193.168.0.128 error: PAM: Authentication failure for kevin from 193.168.0.128 -error: PAM: Authentication failure for kevin from 87.142.124.10 -error: PAM: Authentication failure for kevin from 87.142.124.10 -error: PAM: Authentication failure for kevin from 87.142.124.10 -error: PAM: Authentication failure for kevin from 87.142.124.10 +error: PAM: Authentication failure for göran from 87.142.124.10 +error: PAM: Authentication failure for göran from 87.142.124.10 +error: PAM: Authentication failure for göran from 87.142.124.10 +error: PAM: Authentication failure for göran from 87.142.124.10 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 4f9d8a06..e9971a06 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -38,7 +38,7 @@ except ImportError: from ..server.jail import Jail from ..server.filterpoll import FilterPoll -from ..server.filter import Filter, FileFilter, DNSUtils +from ..server.filter import Filter, FileFilter, FileContainer, locale, DNSUtils from ..server.failmanager import FailManagerEmpty from ..server.mytime import MyTime from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase @@ -166,6 +166,10 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line return fout +TEST_JOURNAL_FIELDS = { + "SYSLOG_IDENTIFIER": "fail2ban-testcases", + "PRIORITY": "7", +} def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # pragma: systemd no cover """Copy lines from one file to systemd journal @@ -176,9 +180,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p else: fin = in_ # Required for filtering - fields.update({"SYSLOG_IDENTIFIER": "fail2ban-testcases", - "PRIORITY": "7", - }) + fields.update(TEST_JOURNAL_FIELDS) # Skip for i in xrange(skip): fin.readline() @@ -228,6 +230,19 @@ class BasicFilter(unittest.TestCase): 1) ) + def testWrongCharInTupleLine(self): + ## line tuple has different types (ascii after ascii / unicode): + for a1 in ('', u'', b''): + for a2 in ('2016-09-05T20:18:56', u'2016-09-05T20:18:56', b'2016-09-05T20:18:56'): + for a3 in ( + 'Fail for "g\xc3\xb6ran" from 192.0.2.1', + u'Fail for "g\xc3\xb6ran" from 192.0.2.1', + b'Fail for "g\xc3\xb6ran" from 192.0.2.1' + ): + # join should work if all arguments have the same type: + enc = locale.getpreferredencoding() + "".join([Filter.uni_decode(v, enc) for v in (a1, a2, a3)]) + class IgnoreIP(LogCaptureTestCase): @@ -828,6 +843,33 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover # we should detect the failures self.assertTrue(self.isFilled(6)) + def test_WrongChar(self): + self._initFilter() + self.filter.start() + # Now let's feed it with entries from the file + _copy_lines_to_journal( + self.test_file, self.journal_fields, skip=15, n=4) + self.assertTrue(self.isFilled(10)) + self.assert_correct_ban("87.142.124.10", 4) + # Add direct utf, unicode, blob: + for l in ( + "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", + u"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", + b"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1".decode('utf-8', 'replace'), + "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", + u"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", + b"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2".decode('utf-8', 'replace') + ): + fields = self.journal_fields + fields.update(TEST_JOURNAL_FIELDS) + journal.send(MESSAGE=l, **fields) + self.assertTrue(self.isFilled(10)) + endtm = MyTime.time()+10 + while len(self.jail) != 2 and MyTime.time() < endtm: + time.sleep(0.10) + self.assertEqual(sorted([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()]), + ["192.0.2.1", "192.0.2.2"]) + return MonitorJournalFailures From 57458a462ed0d12b50e79ecad5204f69e521b11d Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 6 Sep 2016 13:03:14 +0200 Subject: [PATCH 33/60] allow to set default or preferred encoding for other filters (e.g. to decode bytes from journal) # Conflicts: # fail2ban/server/filter.py --- fail2ban/server/filter.py | 38 ++++++++++++++++++++------------ fail2ban/server/filtersystemd.py | 12 +++++----- fail2ban/server/server.py | 6 ++--- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8d3f9014..75b3d85e 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -83,6 +83,8 @@ class Filter(JailThread): self.__lastDate = None ## External command self.__ignoreCommand = False + ## Default or preferred encoding (to decode bytes from file or journal): + self.__encoding = locale.getpreferredencoding() self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -280,6 +282,27 @@ class Filter(JailThread): def getMaxLines(self): return self.__lineBufferSize + ## + # Set the log file encoding + # + # @param encoding the encoding used with log files + + def setLogEncoding(self, encoding): + if encoding.lower() == "auto": + encoding = locale.getpreferredencoding() + codecs.lookup(encoding) # Raise LookupError if invalid codec + self.__encoding = encoding + logSys.info("Set jail log file encoding to %s" % encoding) + return encoding + + ## + # Get the log file encoding + # + # @return log encoding value + + def getLogEncoding(self): + return self.__encoding + ## # Main loop. # @@ -584,7 +607,6 @@ class FileFilter(Filter): Filter.__init__(self, jail, **kwargs) ## The log file path. self.__logs = dict() - self.setLogEncoding("auto") ## # Add a log file path @@ -655,21 +677,9 @@ class FileFilter(Filter): # @param encoding the encoding used with log files def setLogEncoding(self, encoding): - if encoding.lower() == "auto": - encoding = locale.getpreferredencoding() - codecs.lookup(encoding) # Raise LookupError if invalid codec + encoding = super(FileFilter, self).setLogEncoding(encoding) for log in self.__logs.itervalues(): log.setEncoding(encoding) - self.__encoding = encoding - logSys.info("Set jail log file encoding to %s" % encoding) - - ## - # Get the log file encoding - # - # @return log encoding value - - def getLogEncoding(self): - return self.__encoding def getLog(self, path): return self.__logs.get(path, None) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 5c5481b9..3023155c 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -31,7 +31,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': raise ImportError("Fail2Ban requires systemd >= 204") from .failmanager import FailManagerEmpty -from .filter import JournalFilter, Filter, locale +from .filter import JournalFilter, Filter from .mytime import MyTime from ..helpers import getLogger, logging, splitwords @@ -170,9 +170,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalMatch(self): return self.__matches - @staticmethod - def uni_decode(x): - v = Filter.uni_decode(x, locale.getpreferredencoding()) + def uni_decode(self, x): + v = Filter.uni_decode(x, self.getLogEncoding()) return v ## @@ -181,10 +180,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover # @param entry systemd journal entry dict # @return format log line - @classmethod - def formatJournalEntry(cls, logentry): + def formatJournalEntry(self, logentry): # Be sure, all argument of line tuple should have the same type: - uni_decode = FilterSystemd.uni_decode + uni_decode = self.uni_decode logelements = [] v = logentry.get('_HOSTNAME') if v: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index f3d44829..abaf4dfd 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -237,13 +237,11 @@ class Server: def setLogEncoding(self, name, encoding): filter_ = self.__jails[name].filter - if isinstance(filter_, FileFilter): - filter_.setLogEncoding(encoding) + filter_.setLogEncoding(encoding) def getLogEncoding(self, name): filter_ = self.__jails[name].filter - if isinstance(filter_, FileFilter): - return filter_.getLogEncoding() + return filter_.getLogEncoding() def setFindTime(self, name, value): self.__jails[name].filter.setFindTime(value) From 51fd9a10274c7a09083e4f4bba9d6b67b726e7d9 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 6 Sep 2016 16:09:59 +0200 Subject: [PATCH 34/60] amend to activate performance-fix (respect findtime before search of match) + code coverage --- fail2ban/server/filter.py | 42 ++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 75b3d85e..d057c63d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -425,8 +425,8 @@ class Filter(JailThread): if isinstance(x, bytes): return x.decode(enc, errors) return x - except (UnicodeDecodeError, UnicodeEncodeError): - if errors != 'strict': # pragma: no cover - unsure if reachable + except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable + if errors != 'strict': raise return uni_decode(x, enc, 'replace') else: @@ -436,8 +436,8 @@ class Filter(JailThread): if isinstance(x, unicode): return x.encode(enc, errors) return x - except (UnicodeDecodeError, UnicodeEncodeError): - if errors != 'strict': # pragma: no cover - unsure if reachable + except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable + if errors != 'strict': raise return uni_decode(x, enc, 'replace') @@ -446,7 +446,6 @@ class Filter(JailThread): """Split the time portion from log msg and return findFailures on them """ if date: - # be sure each element of tuple line has the same type: tupleLine = line else: l = line.rstrip('\r\n') @@ -467,7 +466,7 @@ class Filter(JailThread): def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager """ - for element in self.processLine(line, date)[1]: + for element in self.processLine(line, date, checkFindTime=True)[1]: ip = element[1] unixTime = element[2] lines = element[3] @@ -832,24 +831,21 @@ class FileContainer: def decode_line(filename, enc, line): try: return line.decode(enc, 'strict') - except UnicodeDecodeError as e: + except (UnicodeDecodeError, UnicodeEncodeError) as e: + global _decode_line_warn + lev = logging.DEBUG + if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = logging.WARNING + _decode_line_warn[filename] = MyTime.time() + 24*60*60 + logSys.log(lev, + "Error decoding line from '%s' with '%s'." + " Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r", + filename, enc, line) # decode with replacing error chars: - rline = line.decode(enc, 'replace') - except UnicodeEncodeError as e: - # encode with replacing error chars: - rline = line.decode(enc, 'replace') - global _decode_line_warn - lev = logging.DEBUG - if _decode_line_warn.get(filename, 0) <= MyTime.time(): - lev = logging.WARNING - _decode_line_warn[filename] = MyTime.time() + 24*60*60 - logSys.log(lev, - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", - filename, enc, line) - return rline + line = line.decode(enc, 'replace') + return line def readline(self): if self.__handler is None: From 7ac9890bf6ad68b237d72ee13886bfab44e04973 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 6 Sep 2016 16:51:06 +0200 Subject: [PATCH 35/60] forgotten obsolete code removed --- fail2ban/server/filter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d057c63d..b2031096 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -472,10 +472,6 @@ class Filter(JailThread): lines = element[3] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) - if unixTime < MyTime.time() - self.getFindTime(): - logSys.debug("Ignore line since time %s < %s - %s" - % (unixTime, MyTime.time(), self.getFindTime())) - break if self.inIgnoreIPList(ip, log_ignore=True): continue logSys.info("[%s] Found %s" % (self.jail.name, ip)) From 9fb167b5e11530df0383aac858a17fcaf638a6aa Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 9 Sep 2016 09:14:29 +0200 Subject: [PATCH 36/60] filter.d/vsftpd.conf: optional reason message after FAIL LOGIN, closes #1543 --- ChangeLog | 3 +++ config/filter.d/vsftpd.conf | 2 +- fail2ban/tests/files/logs/vsftpd | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 5f97c996..ff68bab8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,9 @@ releases. - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) * `filter.d/postfix-sasl.conf` - Allow for having no trailing space after 'failed:' (gh-1497) +* `filter.d/vsftpd.conf` + - Optional reason part in message after FAIL LOGIN (gh-1543) + ### New Features diff --git a/config/filter.d/vsftpd.conf b/config/filter.d/vsftpd.conf index 930b0d7e..2ecc44d3 100644 --- a/config/filter.d/vsftpd.conf +++ b/config/filter.d/vsftpd.conf @@ -14,7 +14,7 @@ __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:? _daemon = vsftpd failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=(?:\s+user=.*)?\s*$ - ^ \[pid \d+\] \[.+\] FAIL LOGIN: Client ""\s*$ + ^ \[pid \d+\] \[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) ignoreregex = diff --git a/fail2ban/tests/files/logs/vsftpd b/fail2ban/tests/files/logs/vsftpd index bcd7f611..3205fac3 100644 --- a/fail2ban/tests/files/logs/vsftpd +++ b/fail2ban/tests/files/logs/vsftpd @@ -12,3 +12,6 @@ Fri Jan 19 12:20:33 2007 [pid 27202] [anonymous] FAIL LOGIN: Client "64.106.46.9 # failJSON: { "time": "2004-10-23T21:15:42", "match": true , "host": "58.254.172.161" } Oct 23 21:15:42 vps vsftpd: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty=ftp ruser=test rhost=58.254.172.161 + +# failJSON: { "time": "2016-09-08T00:39:49", "match": true , "host": "192.0.2.1" } +Thu Sep 8 00:39:49 2016 [pid 15019] [guest] FAIL LOGIN: Client "::ffff:192.0.2.1", "User is not in the allow user list." From 1071db2256e5a39e5b0cbbe3dc37434e378bbffb Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 20 Sep 2016 00:00:26 +0200 Subject: [PATCH 37/60] filter.py: easy-fix to use sha1 instead of md5 if its usage prohibited by some systems following strict standards (like FIPS) closes gh-1540 --- fail2ban/server/filter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b2031096..459a47d0 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -742,7 +742,12 @@ class FileFilter(Filter): try: import hashlib - md5sum = hashlib.md5 + try: + md5sum = hashlib.md5 + # try to use it (several standards like FIPS forbid it): + md5sum(' ').hexdigest() + except: # pragma: no cover + md5sum = hashlib.sha1 except ImportError: # pragma: no cover # hashlib was introduced in Python 2.5. For compatibility with those # elderly Pythons, import from md5 From 0f1d1a0d4d4943317b8b5d1457b29c93a7a37dc8 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 21 Sep 2016 09:22:18 +0200 Subject: [PATCH 38/60] ChangeLog: FIPS compliant --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ff68bab8..c46e795e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,7 @@ releases. * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); * Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); +* FIPS compliant, use sha1 instead of md5 if it not allowed (see gh-1540) * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) * `filter.d/ignorecommands/apache-fakegooglebot` From a406c6eb3a20621e43e84cb644ff2c19873d629f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Thu, 22 Sep 2016 20:29:26 +0000 Subject: [PATCH 39/60] By the author: > Yes, scripting is not supported in path. https://bitbucket.org/tildeslash/monit/issues/372/webadmin-shows-only-the-first-part-of#comment-27946048 --- files/monit/fail2ban | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/monit/fail2ban b/files/monit/fail2ban index 8e6c9419..7873dbe0 100644 --- a/files/monit/fail2ban +++ b/files/monit/fail2ban @@ -1,7 +1,7 @@ check process fail2ban with pidfile /var/run/fail2ban/fail2ban.pid group services start program = "/etc/init.d/fail2ban force-start" - stop program = "/etc/init.d/fail2ban stop || :" + stop program = "/etc/init.d/fail2ban stop" if failed unixsocket /var/run/fail2ban/fail2ban.sock then restart if 5 restarts within 5 cycles then timeout From 8e3e333d54e4cef7332fdc4a8eea25bc1c2b30a5 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 27 Sep 2016 14:17:45 +0200 Subject: [PATCH 40/60] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index c46e795e..4f4c2863 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,7 @@ releases. using direct call, out of virtualenv); * Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); * FIPS compliant, use sha1 instead of md5 if it not allowed (see gh-1540) +* Monit config: scripting is not supported in path (gh-1556) * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) * `filter.d/ignorecommands/apache-fakegooglebot` From d08db22b92be8b559ad2351c6f739ee5e242fd62 Mon Sep 17 00:00:00 2001 From: Nils Date: Sun, 7 Aug 2016 17:46:04 +0200 Subject: [PATCH 41/60] Create npf.conf for the NPF packet filter This file adds support for the NPF packet filter, available on NetBSD since version 6.0 --- config/action.d/npf.conf | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 config/action.d/npf.conf diff --git a/config/action.d/npf.conf b/config/action.d/npf.conf new file mode 100644 index 00000000..8b00d177 --- /dev/null +++ b/config/action.d/npf.conf @@ -0,0 +1,61 @@ +# Fail2Ban configuration file +# +# NetBSD npf ban/unban +# +# Author: Nils Ratusznik +# Based on pf.conf action file +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +# we don't enable NPF automatically, as it will be enabled elsewhere +actionstart = + + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +# we don't disable NPF automatically either +actionstop = + + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# number of failures +#