Merge branch '0.10' into 0.11

ChangeLog

@@ -88,6 +88,8 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition
   e. g. date-pattern `^\[{LEPOCH}\]\s+:` will match and cut out `[1516469849551000] :` from begin of the log-line.
 * badips.py now uses https instead of plain http when requesting badips.com (gh-2057);
 * add support for "any" badips.py bancategory, to be able to retrieve IPs from all categories with a desired score (gh-2056);
+* Introduced new parameter `padding` for logging within fail2ban-server (default on, excepting SYSLOG):
+  Usage `logtarget = target[padding=on|off]`
 
 
 ver. 0.10.2 (2018/01/18) - nothing-burns-like-the-cold

fail2ban/helpers.py

@@ -143,7 +143,7 @@ def str2LogLevel(value):
 		raise ValueError("Invalid log level %r" % value)
 	return ll
 
-def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True):
+def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True):
 	"""Custom log format for the verbose runs
 	"""
 	if verbosity > 1: # pragma: no cover
@@ -155,6 +155,13 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True):
 			fmt = ' %(thread)X %(levelname)-5.5s' + fmt
 			if addtime:
 				fmt = ' %(asctime)-15s' + fmt
+	else: # default (not verbose):
+		fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt
+		if addtime:
+			fmt = "%(asctime)s " + fmt
+	# remove padding if not needed:
+	if not padding:
+		fmt = re.sub(r'(?<=\))-?\d+(?:\.\d+)?s', lambda m: 's', fmt)
 	return fmt
 
 

fail2ban/server/server.py

@@ -587,10 +587,12 @@ class Server:
 			if systarget == "INHERITED":
 				self.__logTarget = target
 				return True
+			padding = logOptions.get('padding')
 			# set a format which is simpler for console use
-			fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s %(message)s"
 			if systarget == "SYSLOG":
 				facility = logOptions.get('facility', 'DAEMON').upper()
+				# backwards compatibility - default no padding for syslog handler:
+				if padding is None: padding = '0'
 				try:
 					facility = getattr(logging.handlers.SysLogHandler, 'LOG_' + facility)
 				except AttributeError: # pragma: no cover
@@ -651,15 +653,19 @@ class Server:
 				addtime = addtime in ('1', 'on', 'true', 'yes')
 			else:
 				addtime = systarget not in ("SYSLOG", "SYSOUT")
+			if padding is not None:
+				padding = padding in ('1', 'on', 'true', 'yes') 
+			else:
+				padding = True
 			# If log-format is redefined in options:
 			if logOptions.get('format', '') != '':
 				fmt = logOptions.get('format')
-			# verbose log-format:
-			elif self.__verbose is not None and self.__verbose > 2: # pragma: no cover
-				fmt = getVerbosityFormat(self.__verbose-1,
-					addtime=addtime)
-			elif addtime:
-				fmt = "%(asctime)s " + fmt
+			else:
+				# verbose log-format:
+				verbose = 0
+				if self.__verbose is not None and self.__verbose > 2: # pragma: no cover
+					verbose = self.__verbose-1
+				fmt = getVerbosityFormat(verbose, addtime=addtime, padding=padding)
 			# tell the handler to use this format
 			hdlr.setFormatter(logging.Formatter(fmt))
 			logger.addHandler(hdlr)

fail2ban/tests/filtertestcase.py

@@ -25,6 +25,7 @@ __license__ = "GPL"
 from __builtin__ import open as fopen
 import unittest
 import os
+import re
 import sys
 import time, datetime
 import tempfile
@@ -43,7 +44,7 @@ from ..server.ipdns import DNSUtils, IPAddr
 from ..server.mytime import MyTime
 from ..server.utils import Utils, uni_decode
 from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase, \
-	CONFIG_DIR as STOCK_CONF_DIR
+	logSys as DefLogSys, CONFIG_DIR as STOCK_CONF_DIR
 from .dummyjail import DummyJail
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
@@ -424,19 +425,35 @@ class IgnoreIPDNS(LogCaptureTestCase):
 		self.jail = DummyJail()
 		self.filter = FileFilter(self.jail)
 
-	def testIgnoreIPDNSOK(self):
-		self.filter.addIgnoreIP("www.epfl.ch")
-		self.assertTrue(self.filter.inIgnoreIPList("128.178.222.69"))
-		self.filter.addIgnoreIP("example.com")
-		self.assertTrue(self.filter.inIgnoreIPList("93.184.216.34"))
-		self.assertTrue(self.filter.inIgnoreIPList("2606:2800:220:1:248:1893:25c8:1946"))
-
-	def testIgnoreIPDNSNOK(self):
-		# Test DNS
-		self.filter.addIgnoreIP("www.epfl.ch")
-		self.assertFalse(self.filter.inIgnoreIPList("127.178.222.69"))
-		self.assertFalse(self.filter.inIgnoreIPList("128.178.222.68"))
-		self.assertFalse(self.filter.inIgnoreIPList("128.178.222.70"))
+	def testIgnoreIPDNS(self):
+		for dns in ("www.epfl.ch", "example.com"):
+			self.filter.addIgnoreIP(dns)
+			ips = DNSUtils.dnsToIp(dns)
+			self.assertTrue(len(ips) > 0)
+			# for each ip from dns check ip ignored:
+			for ip in ips:
+				ip = str(ip)
+				DefLogSys.debug('  ++ positive case for %s', ip)
+				self.assertTrue(self.filter.inIgnoreIPList(ip))
+				# check another ips (with increment/decrement of first/last part) not ignored:
+				iparr = []
+				ip2 = re.search(r'^([^.:]+)([.:])(.*?)([.:])([^.:]+)$', ip)
+				if ip2:
+					ip2 = ip2.groups()
+					for o in (0, 4):
+						for i in (1, -1):
+							ipo = list(ip2)
+							if ipo[1] == '.':
+								ipo[o] = str(int(ipo[o])+i)
+							else:
+								ipo[o] = '%x' % (int(ipo[o], 16)+i)
+							ipo = ''.join(ipo)
+							if ipo not in ips:
+								iparr.append(ipo)
+				self.assertTrue(len(iparr) > 0)
+				for ip in iparr:
+					DefLogSys.debug('  -- negative case for %s', ip)
+					self.assertFalse(self.filter.inIgnoreIPList(str(ip)))
 
 	def testIgnoreCmdApacheFakegooglebot(self):
 		unittest.F2B.SkipIfCfgMissing(stock=True)

fail2ban/tests/servertestcase.py

@@ -832,7 +832,7 @@ class TransmitterLogging(TransmitterBase):
 			os.remove(logTarget)
 
 		self.setGetTest("logtarget", 'STDOUT[format="%(message)s"]', 'STDOUT')
-		self.setGetTest("logtarget", 'STDERR[datetime=off]', 'STDERR')
+		self.setGetTest("logtarget", 'STDERR[datetime=off, padding=off]', 'STDERR')
 
 	def testLogTargetSYSLOG(self):
 		if not os.path.exists("/dev/log"):