From 0a50f2e19e221a19012a5d68811834d16203dfcf Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 4 Apr 2018 19:44:09 +0200 Subject: [PATCH 1/7] next release of 0.10: bump version, update ChangeLog, man's and MANIFEST etc. --- ChangeLog | 6 ++++-- MANIFEST | 2 ++ fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-python.1 | 8 ++------ man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ChangeLog b/ChangeLog index 74973f3e..2d101ce0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,11 +27,11 @@ Incompatibility list (compared to v.0.9): * v.0.10 uses more precise date template handling, that can be theoretically incompatible to some user configurations resp. `datepattern`. -* Since v0.10 fail2ban supports the matching of the IPv6 addresses, but not all ban actions are +* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are IPv6-capable now. -ver. 0.10.3-dev-1 (20??/??/??) - development edition +ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right ----------- ### Fixes @@ -55,6 +55,8 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition * (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054); ### New Features +* several stability and performance optimizations, more effective filter parsing, etc; +* stable runnable within python versions 3.6 (as well as within 3.7-dev); ### Enhancements * `filter.d/apache-auth.conf`: detection of Apache SNI errors resp. misredirect attempts (gh-2017, gh-2097); diff --git a/MANIFEST b/MANIFEST index bfdd77c0..144ae697 100644 --- a/MANIFEST +++ b/MANIFEST @@ -391,6 +391,8 @@ kill-server man/fail2ban.1 man/fail2ban-client.1 man/fail2ban-client.h2m +man/fail2ban-python.1 +man/fail2ban-python.h2m man/fail2ban-regex.1 man/fail2ban-regex.h2m man/fail2ban-server.1 diff --git a/fail2ban/version.py b/fail2ban/version.py index fd16e76c..0f36b315 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.10.3.dev1" +version = "0.10.3" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 58abc9ff..c4b96650 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "January 2018" "fail2ban-client v0.10.3.dev1" "User Commands" +.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.10.3" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.10.3.dev1 reads log file that contains password failure report +Fail2Ban v0.10.3 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-python.1 b/man/fail2ban-python.1 index e67e468a..9880726e 100644 --- a/man/fail2ban-python.1 +++ b/man/fail2ban-python.1 @@ -1,14 +1,10 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.5. -.TH FAIL2BAN-PYTHON "1" "January 2018" "fail2ban-python f2bversion" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-PYTHON "1" "April 2018" "fail2ban-python f2bversion" "User Commands" .SH NAME fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used .SH DESCRIPTION usage: ../bin/fail2ban\-python [option] ... [\-c cmd | \fB\-m\fR mod | file | \fB\-]\fR [arg] ... Options and arguments (and corresponding environment variables): -\fB\-b\fR : issue warnings about comparing bytearray with unicode -.IP -(\fB\-bb\fR: issue errors) -.PP \fB\-B\fR : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x \fB\-c\fR cmd : program passed in as string (terminates option list) \fB\-d\fR : debug output from parser; also PYTHONDEBUG=x diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 74e9c6bc..1d8d7da8 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "January 2018" "fail2ban-regex 0.10.3.dev1" "User Commands" +.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.10.3" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 63481c36..560bedac 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "January 2018" "fail2ban-server v0.10.3.dev1" "User Commands" +.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.10.3" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.10.3.dev1 reads log file that contains password failure report +Fail2Ban v0.10.3 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 19cd55a9..cae79879 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "January 2018" "fail2ban-testcases 0.10.3.dev1" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.10.3" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 187514eda7c0eeb7f625f8150ef88c62074b4cb3 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 4 Apr 2018 20:17:26 +0200 Subject: [PATCH 2/7] bump version (0.10.3 -> 0.10.4.dev1) --- ChangeLog | 10 ++++++++++ fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2d101ce0..1454ae55 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,6 +31,16 @@ Incompatibility list (compared to v.0.9): IPv6-capable now. +ver. 0.10.4-dev-1 (20??/??/??) - development edition +----------- + +### Fixes + +### New Features + +### Enhancements + + ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right ----------- diff --git a/fail2ban/version.py b/fail2ban/version.py index 0f36b315..41a32469 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.10.3" +version = "0.10.4.dev1" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index c4b96650..6ed4ea4c 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.10.3" "User Commands" +.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.10.4.dev1" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.10.3 reads log file that contains password failure report +Fail2Ban v0.10.4.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 1d8d7da8..ebc9fa4c 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.10.3" "User Commands" +.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.10.4.dev1" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 560bedac..075fb378 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.10.3" "User Commands" +.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.10.4.dev1" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.10.3 reads log file that contains password failure report +Fail2Ban v0.10.4.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index cae79879..56abc600 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.10.3" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.10.4.dev1" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 34b586b51ebc2ca914bdc9e0a4a943481866745d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 4 Apr 2018 23:28:44 +0200 Subject: [PATCH 3/7] fix for JSON serialization bug for set object (gh-2103): currently there are only users, so simply serialized as a list. --- fail2ban/server/database.py | 24 ++++++++++++++++-------- fail2ban/tests/databasetestcase.py | 23 ++++++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index d7ba12c1..a06db21c 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -39,9 +39,14 @@ from ..helpers import getLogger, PREFER_ENC logSys = getLogger(__name__) if sys.version_info >= (3,): + def _json_default(x): + if isinstance(x, set): + x = list(x) + return x + def _json_dumps_safe(x): try: - x = json.dumps(x, ensure_ascii=False).encode( + x = json.dumps(x, ensure_ascii=False, default=_json_default).encode( PREFER_ENC, 'replace') except Exception as e: # pragma: no cover logSys.error('json dumps failed: %s', e) @@ -60,7 +65,7 @@ else: def _normalize(x): if isinstance(x, dict): return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems()) - elif isinstance(x, list): + elif isinstance(x, (list, set)): return [_normalize(element) for element in x] elif isinstance(x, unicode): return x.encode(PREFER_ENC) @@ -527,10 +532,13 @@ class Fail2BanDb(object): except KeyError: pass #TODO: Implement data parts once arbitrary match keys completed + data = ticket.getData() + matches = data.get('matches') + if matches and len(matches) > self.maxEntries: + data['matches'] = matches[-self.maxEntries:] cur.execute( "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", - (jail.name, ip, int(round(ticket.getTime())), - ticket.getData())) + (jail.name, ip, int(round(ticket.getTime())), data)) @commitandrollback def delBan(self, cur, jail, *args): @@ -659,11 +667,11 @@ class Fail2BanDb(object): else: matches = m[-maxadd:] + matches failures += data.get('failures', 1) - tickdata.update(data.get('data', {})) + data['failures'] = failures + data['matches'] = matches + tickdata.update(data) prev_timeofban = timeofban - ticket = FailTicket(banip, prev_timeofban, matches) - ticket.setAttempt(failures) - ticket.setData(**tickdata) + ticket = FailTicket(banip, prev_timeofban, data=tickdata) tickets.append(ticket) if cacheKey: diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 1ee523d9..7690525e 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -302,12 +302,18 @@ class DatabaseTest(LogCaptureTestCase): def testGetBansMerged_MaxEntries(self): self.testAddJail() maxEntries = 2 - failures = ["abc\n", "123\n", "ABC\n", "1234\n"] + failures = [ + {"matches": ["abc\n"], "user": set(['test'])}, + {"matches": ["123\n"], "user": set(['test'])}, + {"matches": ["ABC\n"], "user": set(['test', 'root'])}, + {"matches": ["1234\n"], "user": set(['test', 'root'])}, + ] + matches2find = [f["matches"][0] for f in failures] # add failures sequential: i = 80 for f in failures: i -= 10 - ticket = FailTicket("127.0.0.1", MyTime.time() - i, [f]) + ticket = FailTicket("127.0.0.1", MyTime.time() - i, data=f) ticket.setAttempt(1) self.db.addBan(self.jail, ticket) # should retrieve 2 matches only, but count of all attempts: @@ -316,9 +322,10 @@ class DatabaseTest(LogCaptureTestCase): self.assertEqual(ticket.getIP(), "127.0.0.1") self.assertEqual(ticket.getAttempt(), len(failures)) self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) # add more failures at once: - ticket = FailTicket("127.0.0.1", MyTime.time() - 10, failures) + ticket = FailTicket("127.0.0.1", MyTime.time() - 10, matches2find, + data={"user": set(['test', 'root'])}) ticket.setAttempt(len(failures)) self.db.addBan(self.jail, ticket) # should retrieve 2 matches only, but count of all attempts: @@ -326,7 +333,13 @@ class DatabaseTest(LogCaptureTestCase): ticket = self.db.getBansMerged("127.0.0.1") self.assertEqual(ticket.getAttempt(), 2 * len(failures)) self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) + # also using getCurrentBans: + ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100) + self.assertTrue(ticket is not None) + self.assertEqual(ticket.getAttempt(), len(failures)) + self.assertEqual(len(ticket.getMatches()), maxEntries) + self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) def testGetBansMerged(self): self.testAddJail() From c1923f96443bf5834b21af2de556a8f801141d75 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 4 Apr 2018 23:32:22 +0200 Subject: [PATCH 4/7] update ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2d101ce0..c2f306b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,9 @@ Incompatibility list (compared to v.0.9): ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right ----------- +### ver. 0.10.3.1: +* fixed JSON serialization for the set-object within dump into database (gh-2103). + ### Fixes * `filter.d/asterisk.conf`: fixed failregex prefix by log over remote syslog server (gh-2060); * `filter.d/exim.conf`: failregex extended - SMTP call dropped: too many syntax or protocol errors (gh-2048); From cb0f4cbb32657f2faf0aa666cc86e4144dc69bf3 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Apr 2018 00:17:00 +0200 Subject: [PATCH 5/7] test_badips.py: amend to 2ff65f5d3ce1a4bd107cb4dbbd5343f7146a0677: increase timeout in normal mode + catch timeout exceptions to skip the test (avoid sporadic CI errors if badips gets slowly). --- fail2ban/tests/action_d/test_badips.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index c3c417b8..c3cf33a4 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -20,6 +20,8 @@ import os import unittest import sys +from socket import timeout +from ssl import SSLError from ..actiontestcase import CallingMap from ..dummyjail import DummyJail @@ -51,7 +53,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 60) + BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) if not BadIPsActionTest.available[0]: raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) @@ -62,7 +64,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable 'score': 5, 'key': "fail2ban-test-suite", #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 30), + 'timeout': (3 if unittest.F2B.fast else 60), }) self.action = self.jail.actions["badips"] @@ -108,11 +110,16 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable self.action.updateperiod = "900" def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) + try: + self.action.start() + self.assertTrue(len(self.action._bannedips) > 10, + "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) + self.action.stop() + self.assertTrue(len(self.action._bannedips) == 0) + except (SSLError, timeout) as e: # pragma: no cover - timeout only + if not isinstance(e, timeout) and 'timed out' not in str(e): + raise + raise unittest.SkipTest('Skip test because of %s' % e) def testBanIP(self): aInfo = CallingMap({ From ac0d441fd68852ffda7b15c71f16b7f4fde1a7ee Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Apr 2018 00:21:30 +0200 Subject: [PATCH 6/7] 0.10.3.fix1: version bump --- fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/version.py b/fail2ban/version.py index 0f36b315..3c16adea 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.10.3" +version = "0.10.3.fix1" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index c4b96650..53292b29 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.10.3" "User Commands" +.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.10.3.fix1" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.10.3 reads log file that contains password failure report +Fail2Ban v0.10.3.fix1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 1d8d7da8..fb2952e8 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.10.3" "User Commands" +.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.10.3.fix1" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 560bedac..3bbe4eb0 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.10.3" "User Commands" +.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.10.3.fix1" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.10.3 reads log file that contains password failure report +Fail2Ban v0.10.3.fix1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index cae79879..6f8869b6 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.10.3" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.10.3.fix1" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 8069eef50c30f01f0d21ed23027e0ea40446ff0f Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Apr 2018 11:26:43 +0200 Subject: [PATCH 7/7] badips: try to fix sporadic test errors if badips-server timed out resp. not available (502 bad gateway or similar). --- config/action.d/badips.py | 38 ++++++++++++------------ fail2ban/tests/action_d/test_badips.py | 41 +++++++++++++++++++------- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 0df34c12..6a6760b5 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -114,6 +114,15 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable except Exception as e: # pragma: no cover return False, e + def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) + messages = {} + try: + messages = json.loads(response.read().decode('utf-8')) + except: + pass + self._logSys.error( + "%s. badips.com response: '%s'", what, + messages.get('err', 'Unknown')) def getCategories(self, incParents=False): """Get badips.com categories. @@ -133,11 +142,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable try: response = urlopen( self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.error( - "Failed to fetch categories. badips.com response: '%s'", - messages['err']) + except HTTPError as response: # pragma: no cover + self.logError(response, "Failed to fetch categories") raise else: response_json = json.loads(response.read().decode('utf-8')) @@ -188,11 +194,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable url = "&".join([url, urlencode({'key': key})]) self._logSys.debug('badips.com: get list, url: %r', url) response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.error( - "Failed to fetch bad IP list. badips.com response: '%s'", - messages['err']) + except HTTPError as response: # pragma: no cover + self.logError(response, "Failed to fetch bad IP list") raise else: return set(response.read().decode('utf-8').split()) @@ -286,7 +289,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) else: self._bannedips.add(ip) - self._logSys.info( + self._logSys.debug( "Banned IP %s for jail '%s' with action '%s'", ip, self._jail.name, self.banaction) @@ -306,7 +309,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable ip, self._jail.name, self.banaction, e, exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) else: - self._logSys.info( + self._logSys.debug( "Unbanned IP %s for jail '%s' with action '%s'", ip, self._jail.name, self.banaction) finally: @@ -338,7 +341,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable # Add new IPs which are now listed self._banIPs(ips - self._bannedips) - self._logSys.info( + self._logSys.debug( "Updated IPs for jail '%s'. Update again in %i seconds", self._jail.name, self.updateperiod) finally: @@ -374,15 +377,12 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable url = "?".join([url, urlencode({'key': self.key})]) self._logSys.debug('badips.com: ban, url: %r', url) response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.error( - "Response from badips.com report: '%s'", - messages['err']) + except HTTPError as response: # pragma: no cover + self.logError(response, "Failed to ban") raise else: messages = json.loads(response.read().decode('utf-8')) - self._logSys.info( + self._logSys.debug( "Response from badips.com report: '%s'", messages['suc']) diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index c3cf33a4..1cc3b19d 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -20,6 +20,7 @@ import os import unittest import sys +from functools import wraps from socket import timeout from ssl import SSLError @@ -28,6 +29,25 @@ from ..dummyjail import DummyJail from ..servertestcase import IPAddr from ..utils import LogCaptureTestCase, CONFIG_DIR +if sys.version_info >= (3, ): + from urllib.error import HTTPError, URLError +else: + from urllib2 import HTTPError, URLError + +def skip_if_not_available(f): + """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". + """ + @wraps(f) + def wrapper(self, *args): + try: + return f(self, *args) + except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues + if not isinstance(e, timeout) and 'timed out' not in str(e): + if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: + raise + raise unittest.SkipTest('Skip test because of %s' % e) + return wrapper + if sys.version_info >= (2,7): # pragma: no cover - may be unavailable class BadIPsActionTest(LogCaptureTestCase): @@ -75,6 +95,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable self.action._timer.cancel() super(BadIPsActionTest, self).tearDown() + @skip_if_not_available def testCategory(self): categories = self.action.getCategories() self.assertIn("ssh", categories) @@ -90,17 +111,20 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable # but valid for blacklisting. self.action.bancategory = "mail" + @skip_if_not_available def testScore(self): self.assertRaises(ValueError, setattr, self.action, "score", -5) self.action.score = 3 self.action.score = "3" + @skip_if_not_available def testBanaction(self): self.assertRaises( ValueError, setattr, self.action, "banaction", "invalid-action") self.action.banaction = "test" + @skip_if_not_available def testUpdateperiod(self): self.assertRaises( ValueError, setattr, self.action, "updateperiod", -50) @@ -109,18 +133,15 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable self.action.updateperiod = 900 self.action.updateperiod = "900" + @skip_if_not_available def testStartStop(self): - try: - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - except (SSLError, timeout) as e: # pragma: no cover - timeout only - if not isinstance(e, timeout) and 'timed out' not in str(e): - raise - raise unittest.SkipTest('Skip test because of %s' % e) + self.action.start() + self.assertTrue(len(self.action._bannedips) > 10, + "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) + self.action.stop() + self.assertTrue(len(self.action._bannedips) == 0) + @skip_if_not_available def testBanIP(self): aInfo = CallingMap({ 'ip': IPAddr('192.0.2.1')