Merge branch '0.10' into 0.11

config/action.d/abuseipdb.conf

@@ -21,14 +21,13 @@
 #
 # Example, for ssh bruteforce (in section [sshd] of `jail.local`): 
 #   action = %(known/action)s
-#            %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
+#            abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"]
 #
-# See below for catagories.
+# See below for categories.
 #
-# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban
 # Added to fail2ban by Andrew James Collett (ajcollett)
 
-## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call.
+## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call.
 # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"]
 # ID	Title	Description
 # 3	  Fraud Orders

fail2ban/client/csocket.py

@@ -48,7 +48,8 @@ class CSocket:
 	def send(self, msg, nonblocking=False, timeout=None):
 		# Convert every list member to string
 		obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL)
-		self.__csock.send(obj + CSPROTO.END)
+		self.__csock.send(obj)
+		self.__csock.send(CSPROTO.END)
 		return self.receive(self.__csock, nonblocking, timeout)
 
 	def settimeout(self, timeout):
@@ -81,9 +82,12 @@ class CSocket:
 		msg = CSPROTO.EMPTY
 		if nonblocking: sock.setblocking(0)
 		if timeout: sock.settimeout(timeout)
-		while msg.rfind(CSPROTO.END) == -1:
+		bufsize = 1024
-			chunk = sock.recv(512)
+		while msg.rfind(CSPROTO.END, -32) == -1:
-			if chunk in ('', b''): # python 3.x may return b'' instead of ''
+			chunk = sock.recv(bufsize)
-				raise RuntimeError("socket connection broken")
+			if not len(chunk):
+				raise socket.error(104, 'Connection reset by peer')
+			if chunk == CSPROTO.END: break
 			msg = msg + chunk
+			if bufsize < 32768: bufsize <<= 1
 		return loads(msg)

fail2ban/client/fail2bancmdline.py

@@ -27,13 +27,17 @@ import sys
 
 from ..version import version, normVersion
 from ..protocol import printFormatted
-from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
+from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError
 
 # Gets the instance of the logger.
 logSys = getLogger("fail2ban")
 
 def output(s): # pragma: no cover
-	print(s)
+	try:
+		print(s)
+	except (BrokenPipeError, IOError) as e: # pragma: no cover
+		if e.errno != 32: # closed / broken pipe
+			raise
 
 # Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf),
 CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket")
@@ -310,12 +314,16 @@ class Fail2banCmdLine():
 	def _exit(code=0):
 		# implicit flush without to produce broken pipe error (32):
 		sys.stderr.close()
-		sys.stdout.close()
+		try:
-		# exit:
+			sys.stdout.flush()
-		if hasattr(os, '_exit') and os._exit:
+			# exit:
-			os._exit(code)
+			if hasattr(sys, 'exit') and sys.exit:
-		else:
+				sys.exit(code)
-			sys.exit(code)
+			else:
+				os._exit(code)
+		except (BrokenPipeError, IOError) as e: # pragma: no cover
+			if e.errno != 32: # closed / broken pipe
+				raise
 
 	@staticmethod
 	def exit(code=0):

fail2ban/client/fail2banregex.py

@@ -21,7 +21,6 @@ Fail2Ban  reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 
 This tools can test regular expressions for "fail2ban".
-
 """
 
 __author__ = "Fail2Ban Developers"
@@ -109,19 +108,22 @@ class _f2bOptParser(OptionParser):
 	def format_help(self, *args, **kwargs):
 		""" Overwritten format helper with full ussage."""
 		self.usage = ''
-		return "Usage: " + usage() + __doc__ + """
+		return "Usage: " + usage() + "\n" + __doc__ + """
 LOG:
-    string                  a string representing a log line
+  string                a string representing a log line
-    filename                path to a log file (/var/log/auth.log)
+  filename              path to a log file (/var/log/auth.log)
-    "systemd-journal"       search systemd journal (systemd-python required)
+  systemd-journal       search systemd journal (systemd-python required),
+                        optionally with backend parameters, see `man jail.conf`
+                        for usage and examples (systemd-journal[journalflags=1]).
 
 REGEX:
-    string                  a string representing a 'failregex'
+  string                a string representing a 'failregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  filter                name of filter, optionally with options (sshd[mode=aggressive])
+  filename              path to a filter file (filter.d/sshd.conf)
 
 IGNOREREGEX:
-    string                  a string representing an 'ignoreregex'
+  string                a string representing an 'ignoreregex'
-    filename                path to a filter file (filter.d/sshd.conf)
+  filename              path to a filter file (filter.d/sshd.conf)
 \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n
 Report bugs to https://github.com/fail2ban/fail2ban/issues\n
 """ + __copyright__ + "\n"

fail2ban/helpers.py

@@ -224,9 +224,10 @@ def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover
 		sys.exit(0)
 
 try:
-    BrokenPipeError
+	BrokenPipeError = BrokenPipeError
 except NameError: # pragma: 3.x no cover
-    BrokenPipeError = IOError
+	BrokenPipeError = IOError
+
 __origLog = logging.Logger._log
 def __safeLog(self, level, msg, args, **kwargs):
 	"""Safe log inject to avoid possible errors by unsafe log-handlers, 

fail2ban/server/actions.py

@@ -707,13 +707,19 @@ class Actions(JailThread, Mapping):
 		"""Status of current and total ban counts and current banned IP list.
 		"""
 		# TODO: Allow this list to be printed as 'status' output
-		supported_flavors = ["basic", "cymru"]
+		supported_flavors = ["short", "basic", "cymru"]
 		if flavor is None or flavor not in supported_flavors:
 			logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors))
 		# Always print this information (basic)
-		ret = [("Currently banned", self.__banManager.size()),
+		if flavor != "short":
-			   ("Total banned", self.__banManager.getBanTotal()),
+			banned = self.__banManager.getBanList()
-			   ("Banned IP list", self.__banManager.getBanList())]
+			cnt = len(banned)
+		else:
+			cnt = self.__banManager.size()
+		ret = [("Currently banned", cnt),
+			   ("Total banned", self.__banManager.getBanTotal())]
+		if flavor != "short":
+			ret += [("Banned IP list", banned)]
 		if flavor == "cymru":
 			cymru_info = self.__banManager.getBanListExtendedCymruInfo()
 			ret += \

fail2ban/server/banmanager.py

@@ -66,7 +66,6 @@ class BanManager:
 	# @param value the time
 	
 	def setBanTime(self, value):
-		with self.__lock:
 			self.__banTime = int(value)
 	
 	##
@@ -76,7 +75,6 @@ class BanManager:
 	# @return the time
 	
 	def getBanTime(self):
-		with self.__lock:
 			return self.__banTime
 	
 	##
@@ -85,7 +83,6 @@ class BanManager:
 	# @param value total number
 	
 	def setBanTotal(self, value):
-		with self.__lock:
 			self.__banTotal = value
 	
 	##
@@ -94,7 +91,6 @@ class BanManager:
 	# @return the total number
 	
 	def getBanTotal(self):
-		with self.__lock:
 			return self.__banTotal
 
 	##
@@ -103,21 +99,21 @@ class BanManager:
 	# @return IP list
 	
 	def getBanList(self, ordered=False, withTime=False):
+		if not ordered:
+			return list(self.__banList.keys())
 		with self.__lock:
-			if not ordered:
-				return list(self.__banList.keys())
 			lst = []
 			for ticket in self.__banList.itervalues():
 				eob = ticket.getEndOfBanTime(self.__banTime)
 				lst.append((ticket,eob))
-			lst.sort(key=lambda t: t[1])
+		lst.sort(key=lambda t: t[1])
-			t2s = MyTime.time2str
+		t2s = MyTime.time2str
-			if withTime:
+		if withTime:
-				return ['%s \t%s + %d = %s' % (
+			return ['%s \t%s + %d = %s' % (
-						t[0].getID(), 
+					t[0].getID(), 
-						t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1])
+					t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1])
-					) for t in lst]
+				) for t in lst]
-			return [t[0].getID() for t in lst]
+		return [t[0].getID() for t in lst]
 
 	##
 	# Returns a iterator to ban list (used in reload, so idle).
@@ -125,8 +121,7 @@ class BanManager:
 	# @return ban list iterator
 	
 	def __iter__(self):
-		# ensure iterator is safe (traverse over the list in snapshot created within lock):
+		# ensure iterator is safe - traverse over the list in snapshot created within lock (GIL):
-		with self.__lock:
 			return iter(list(self.__banList.values()))
 
 	##

fail2ban/server/failmanager.py

@@ -47,12 +47,10 @@ class FailManager:
 		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):
-		with self.__lock:
+		self.__failTotal = value
-			self.__failTotal = value
 		
 	def getFailTotal(self):
-		with self.__lock:
+		return self.__failTotal
-			return self.__failTotal
 	
 	def getFailCount(self):
 		# may be slow on large list of failures, should be used for test purposes only...
@@ -126,8 +124,7 @@ class FailManager:
 		return attempts
 	
 	def size(self):
-		with self.__lock:
+		return len(self.__failList)
-			return len(self.__failList)
 	
 	def cleanup(self, time):
 		with self.__lock:

fail2ban/tests/actionstestcase.py

@@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase):
 		self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop')
 		self.assertEqual(self.__actions.status(),[("Currently banned", 0 ),
                ("Total banned", 0 ), ("Banned IP list", [] )])
+		self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ),
+               ("Total banned", 0 )])
 
 	def testAddActionPython(self):
 		self.__actions.add(

fail2ban/tests/sockettestcase.py

@@ -153,7 +153,7 @@ class Socket(LogCaptureTestCase):
 		org_handler = RequestHandler.found_terminator
 		try:
 			RequestHandler.found_terminator = lambda self: self.close()
-			self.assertRaisesRegexp(RuntimeError, r"socket connection broken", 
+			self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe",
 				lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10)))
 		finally:
 			RequestHandler.found_terminator = org_handler
@@ -169,7 +169,7 @@ class Socket(LogCaptureTestCase):
 		org_handler = RequestHandler.found_terminator
 		try:
 			RequestHandler.found_terminator = lambda self: TestMsgError()
-			#self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage)
+			#self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage)
 			self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error')
 		finally:
 			RequestHandler.found_terminator = org_handler

man/fail2ban-regex.1

@@ -18,13 +18,18 @@ a string representing a log line
 filename
 path to a log file (\fI\,/var/log/auth.log\/\fP)
 .TP
-"systemd\-journal"
+systemd\-journal
-search systemd journal (systemd\-python required)
+search systemd journal (systemd\-python required),
+optionally with backend parameters, see `man jail.conf`
+for usage and examples (systemd\-journal[journalflags=1]).
 .SS "REGEX:"
 .TP
 string
 a string representing a 'failregex'
 .TP
+filter
+name of filter, optionally with options (sshd[mode=aggressive])
+.TP
 filename
 path to a filter file (filter.d/sshd.conf)
 .SS "IGNOREREGEX:"

man/jail.conf.5

@@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst
 uses a polling algorithm which does not require external libraries.
 .TP
 .B systemd
-uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config.
+uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples:
+.PP
+.RS
+.nf
+backend = systemd[journalpath=/run/log/journal/machine-1]
+backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"]
+backend = systemd[journalflags=1]
+.fi
 
 .SS Actions
 Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: