Merge branch '0.10' into 0.11, version bump

# Conflicts resolved:
#	fail2ban/server/database.py

ChangeLog

@@ -27,7 +27,7 @@ Incompatibility list (compared to v.0.9):
 * v.0.10 uses more precise date template handling, that can be theoretically incompatible to some
   user configurations resp. `datepattern`.
 
-* Since v0.10 fail2ban supports the matching of the IPv6 addresses, but not all ban actions are
+* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are
   IPv6-capable now.
 
 
@@ -57,10 +57,22 @@ ver. 0.11.0-dev-0 (20??/??/??) - development nightly edition
   (or persistent); not affected if ban-time of the jail is unchanged between stop/start.
 
 
+ver. 0.10.4-dev-1 (20??/??/??) - development edition
+-----------
+
+### Fixes
+
+### New Features
 
-ver. 0.10.3-dev-1 (20??/??/??) - development edition
+### Enhancements
+
+
+ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right
 -----------
 
+### ver. 0.10.3.1:
+* fixed JSON serialization for the set-object within dump into database (gh-2103).
+
 ### Fixes
 * `filter.d/asterisk.conf`: fixed failregex prefix by log over remote syslog server (gh-2060);
 * `filter.d/exim.conf`: failregex extended - SMTP call dropped: too many syntax or protocol errors (gh-2048);
@@ -82,6 +94,8 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition
 * (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054);
 
 ### New Features
+* several stability and performance optimizations, more effective filter parsing, etc;
+* stable runnable within python versions 3.6 (as well as within 3.7-dev);
 
 ### Enhancements
 * `filter.d/apache-auth.conf`: detection of Apache SNI errors resp. misredirect attempts (gh-2017, gh-2097);

MANIFEST

@@ -394,6 +394,8 @@ kill-server
 man/fail2ban.1
 man/fail2ban-client.1
 man/fail2ban-client.h2m
+man/fail2ban-python.1
+man/fail2ban-python.h2m
 man/fail2ban-regex.1
 man/fail2ban-regex.h2m
 man/fail2ban-server.1

config/action.d/badips.py

@@ -114,6 +114,15 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		except Exception as e: # pragma: no cover
 			return False, e
 
+	def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc)
+		messages = {}
+		try:
+			messages = json.loads(response.read().decode('utf-8'))
+		except:
+			pass
+		self._logSys.error(
+			"%s. badips.com response: '%s'", what,
+				messages.get('err', 'Unknown'))
 
 	def getCategories(self, incParents=False):
 		"""Get badips.com categories.
@@ -133,11 +142,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		try:
 			response = urlopen(
 				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to fetch categories")
-			self._logSys.error(
-				"Failed to fetch categories. badips.com response: '%s'",
-				messages['err'])
 			raise
 		else:
 			response_json = json.loads(response.read().decode('utf-8'))
@@ -188,11 +194,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				url = "&".join([url, urlencode({'key': key})])
 			self._logSys.debug('badips.com: get list, url: %r', url)
 			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to fetch bad IP list")
-			self._logSys.error(
-				"Failed to fetch bad IP list. badips.com response: '%s'",
-				messages['err'])
 			raise
 		else:
 			return set(response.read().decode('utf-8').split())
@@ -286,7 +289,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
 			else:
 				self._bannedips.add(ip)
-				self._logSys.info(
+				self._logSys.debug(
 					"Banned IP %s for jail '%s' with action '%s'",
 					ip, self._jail.name, self.banaction)
 
@@ -306,7 +309,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 					ip, self._jail.name, self.banaction, e,
 					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
 			else:
-				self._logSys.info(
+				self._logSys.debug(
 					"Unbanned IP %s for jail '%s' with action '%s'",
 					ip, self._jail.name, self.banaction)
 			finally:
@@ -338,7 +341,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				# Add new IPs which are now listed
 				self._banIPs(ips - self._bannedips)
 
-				self._logSys.info(
+				self._logSys.debug(
 					"Updated IPs for jail '%s'. Update again in %i seconds",
 					self._jail.name, self.updateperiod)
 			finally:
@@ -374,15 +377,12 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				url = "?".join([url, urlencode({'key': self.key})])
 			self._logSys.debug('badips.com: ban, url: %r', url)
 			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to ban")
-			self._logSys.error(
-				"Response from badips.com report: '%s'",
-				messages['err'])
 			raise
 		else:
 			messages = json.loads(response.read().decode('utf-8'))
-			self._logSys.info(
+			self._logSys.debug(
 				"Response from badips.com report: '%s'",
 				messages['suc'])
 

fail2ban/server/database.py

@@ -39,9 +39,14 @@ from ..helpers import getLogger, PREFER_ENC
 logSys = getLogger(__name__)
 
 if sys.version_info >= (3,):
+	def _json_default(x):
+		if isinstance(x, set):
+			x = list(x)
+		return x
+
 	def _json_dumps_safe(x):
 		try:
-			x = json.dumps(x, ensure_ascii=False).encode(
+			x = json.dumps(x, ensure_ascii=False, default=_json_default).encode(
 				PREFER_ENC, 'replace')
 		except Exception as e: # pragma: no cover
 			logSys.error('json dumps failed: %s', e)
@@ -60,7 +65,7 @@ else:
 	def _normalize(x):
 		if isinstance(x, dict):
 			return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems())
-		elif isinstance(x, list):
+		elif isinstance(x, (list, set)):
 			return [_normalize(element) for element in x]
 		elif isinstance(x, unicode):
 			return x.encode(PREFER_ENC)
@@ -561,14 +566,18 @@ class Fail2BanDb(object):
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
+		data = ticket.getData()
+		matches = data.get('matches')
+		if matches and len(matches) > self.maxEntries:
+			data['matches'] = matches[-self.maxEntries:]
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
 			(jail.name, ip, int(round(ticket.getTime())), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount(),
-				ticket.getData()))
+				data))
 		cur.execute(
 			"INSERT OR REPLACE INTO bips(ip, jail, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
 			(ip, jail.name, int(round(ticket.getTime())), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount(),
-				ticket.getData()))
+				data))
 
 	@commitandrollback
 	def delBan(self, cur, jail, *args):
@@ -701,11 +710,11 @@ class Fail2BanDb(object):
 						else:
 							matches = m[-maxadd:] + matches
 					failures += data.get('failures', 1)
-					tickdata.update(data.get('data', {}))
+					data['failures'] = failures
+					data['matches'] = matches
+					tickdata.update(data)
 					prev_timeofban = timeofban
-				ticket = FailTicket(banip, prev_timeofban, matches)
+				ticket = FailTicket(banip, prev_timeofban, data=tickdata)
-				ticket.setAttempt(failures)
-				ticket.setData(**tickdata)
 				tickets.append(ticket)
 
 			if cacheKey:

fail2ban/tests/action_d/test_badips.py

@@ -20,12 +20,34 @@
 import os
 import unittest
 import sys
+from functools import wraps
+from socket import timeout
+from ssl import SSLError
 
 from ..actiontestcase import CallingMap
 from ..dummyjail import DummyJail
 from ..servertestcase import IPAddr
 from ..utils import LogCaptureTestCase, CONFIG_DIR
 
+if sys.version_info >= (3, ):
+	from urllib.error import HTTPError, URLError
+else:
+	from urllib2 import HTTPError, URLError
+
+def skip_if_not_available(f):
+	"""Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway".
+	"""
+	@wraps(f)
+	def wrapper(self, *args):
+		try:
+			return f(self, *args)
+		except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues
+			if not isinstance(e, timeout) and 'timed out' not in str(e):
+				if not hasattr(e, 'code') or e.code > 200 and e.code <= 404:
+					raise
+			raise unittest.SkipTest('Skip test because of %s' % e)
+	return wrapper
+
 if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 	class BadIPsActionTest(LogCaptureTestCase):
 
@@ -51,7 +73,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 						BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName)
 					BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action
 					self.jail.actions._load_python_module(pythonModuleName)
-				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 60)
+				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30)
 			if not BadIPsActionTest.available[0]:
 				raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1])
 
@@ -62,7 +84,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 				'score': 5,
 				'key': "fail2ban-test-suite",
 				#'bankey': "fail2ban-test-suite",
-				'timeout': (3 if unittest.F2B.fast else 30),
+				'timeout': (3 if unittest.F2B.fast else 60),
 				})
 			self.action = self.jail.actions["badips"]
 
@@ -73,6 +95,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 				self.action._timer.cancel()
 			super(BadIPsActionTest, self).tearDown()
 
+		@skip_if_not_available
 		def testCategory(self):
 			categories = self.action.getCategories()
 			self.assertIn("ssh", categories)
@@ -88,17 +111,20 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			# but valid for blacklisting.
 			self.action.bancategory = "mail"
 
+		@skip_if_not_available
 		def testScore(self):
 			self.assertRaises(ValueError, setattr, self.action, "score", -5)
 			self.action.score = 3
 			self.action.score = "3"
 
+		@skip_if_not_available
 		def testBanaction(self):
 			self.assertRaises(
 				ValueError, setattr, self.action, "banaction",
 				"invalid-action")
 			self.action.banaction = "test"
 
+		@skip_if_not_available
 		def testUpdateperiod(self):
 			self.assertRaises(
 				ValueError, setattr, self.action, "updateperiod", -50)
@@ -107,6 +133,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			self.action.updateperiod = 900
 			self.action.updateperiod = "900"
 
+		@skip_if_not_available
 		def testStartStop(self):
 			self.action.start()
 			self.assertTrue(len(self.action._bannedips) > 10,
@@ -114,6 +141,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			self.action.stop()
 			self.assertTrue(len(self.action._bannedips) == 0)
 
+		@skip_if_not_available
 		def testBanIP(self):
 			aInfo = CallingMap({
 				'ip': IPAddr('192.0.2.1')

fail2ban/tests/databasetestcase.py

@@ -339,12 +339,18 @@ class DatabaseTest(LogCaptureTestCase):
 	def testGetBansMerged_MaxEntries(self):
 		self.testAddJail()
 		maxEntries = 2
-		failures = ["abc\n", "123\n", "ABC\n", "1234\n"]
+		failures = [
+			{"matches": ["abc\n"], "user": set(['test'])},
+			{"matches": ["123\n"], "user": set(['test'])},
+			{"matches": ["ABC\n"], "user": set(['test', 'root'])},
+			{"matches": ["1234\n"], "user": set(['test', 'root'])},
+		]
+		matches2find = [f["matches"][0] for f in failures]
 		# add failures sequential:
 		i = 80
 		for f in failures:
 			i -= 10
-			ticket = FailTicket("127.0.0.1", MyTime.time() - i, [f])
+			ticket = FailTicket("127.0.0.1", MyTime.time() - i, data=f)
 			ticket.setAttempt(1)
 			self.db.addBan(self.jail, ticket)
 		# should retrieve 2 matches only, but count of all attempts:
@@ -353,9 +359,10 @@ class DatabaseTest(LogCaptureTestCase):
 		self.assertEqual(ticket.getIP(), "127.0.0.1")
 		self.assertEqual(ticket.getAttempt(), len(failures))
 		self.assertEqual(len(ticket.getMatches()), maxEntries)
-		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
     # add more failures at once:
-		ticket = FailTicket("127.0.0.1", MyTime.time() - 10, failures)
+		ticket = FailTicket("127.0.0.1", MyTime.time() - 10, matches2find,
+			data={"user": set(['test', 'root'])})
 		ticket.setAttempt(len(failures))
 		self.db.addBan(self.jail, ticket)
 		# should retrieve 2 matches only, but count of all attempts:
@@ -363,7 +370,13 @@ class DatabaseTest(LogCaptureTestCase):
 		ticket = self.db.getBansMerged("127.0.0.1")
 		self.assertEqual(ticket.getAttempt(), 2 * len(failures))
 		self.assertEqual(len(ticket.getMatches()), maxEntries)
-		self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:])
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
+		# also using getCurrentBans:
+		ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100)
+		self.assertTrue(ticket is not None)
+		self.assertEqual(ticket.getAttempt(), len(failures))
+		self.assertEqual(len(ticket.getMatches()), maxEntries)
+		self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:])
 
 	def testGetBansMerged(self):
 		self.testAddJail()

fail2ban/version.py

@@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.11.0.dev1"
+version = "0.11.0.dev2"

man/fail2ban-client.1

@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "January 2018" "fail2ban-client v0.11.0.dev1" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.11.0.dev2" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.11.0.dev1 reads log file that contains password failure report
+Fail2Ban v0.11.0.dev2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP

man/fail2ban-python.1

@@ -1,14 +1,10 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.5.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-PYTHON "1" "January 2018" "fail2ban-python f2bversion" "User Commands"
+.TH FAIL2BAN-PYTHON "1" "April 2018" "fail2ban-python f2bversion" "User Commands"
 .SH NAME
 fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used
 .SH DESCRIPTION
 usage: ../bin/fail2ban\-python [option] ... [\-c cmd | \fB\-m\fR mod | file | \fB\-]\fR [arg] ...
 Options and arguments (and corresponding environment variables):
-\fB\-b\fR     : issue warnings about comparing bytearray with unicode
-.IP
-(\fB\-bb\fR: issue errors)
-.PP
 \fB\-B\fR     : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x
 \fB\-c\fR cmd : program passed in as string (terminates option list)
 \fB\-d\fR     : debug output from parser; also PYTHONDEBUG=x

man/fail2ban-regex.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "January 2018" "fail2ban-regex 0.11.0.dev1" "User Commands"
+.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.11.0.dev2" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS

man/fail2ban-server.1

@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "January 2018" "fail2ban-server v0.11.0.dev1" "User Commands"
+.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.11.0.dev2" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.11.0.dev1 reads log file that contains password failure report
+Fail2Ban v0.11.0.dev2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP

man/fail2ban-testcases.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "January 2018" "fail2ban-testcases 0.11.0.dev1" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.11.0.dev2" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS