Merge pull request #2351 from sebres/0.10-multi-ban-unban-in-jail

fail2ban-client: multi ban/unban and attempt for set jail

ChangeLog

@@ -53,6 +53,14 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition
 ### Enhancements
 * jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations
   containing new-line);
+* fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349);
+  Syntax:
+  - `fail2ban-client set <jain> banip <ip1> ... <ipN>`
+  - `fail2ban-client set <jain> unbanip [--report-absent] <ip1> ... <ipN>`
+* fail2ban-client: extended with new feature which allows to inform fail2ban about single or multiple
+  attempts (failure) for IP (resp. failure-ID), see gh-2351;
+  Syntax:
+  - `fail2ban-client set <jail> attempt <ip> [<failure-message1> ... <failure-messageN>]`
 
 
 ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four

fail2ban/client/jailreader.py

@@ -246,8 +246,7 @@ class JailReader(ConfigReader):
 			elif opt == "backend":
 				backend = value
 			elif opt == "ignoreip":
-				for ip in splitwords(value):
-					stream.append(["set", self.__name, "addignoreip", ip])
+				stream.append(["set", self.__name, "addignoreip"] + splitwords(value))
 			elif opt in ("failregex", "ignoreregex"):
 				multi = []
 				for regex in value.split('\n'):

fail2ban/protocol.py

@@ -99,8 +99,9 @@ protocol = [
 ["set <JAIL> bantime <TIME>", "sets the number of seconds <TIME> a host will be banned for <JAIL>"], 
 ["set <JAIL> datepattern <PATTERN>", "sets the <PATTERN> used to match date/times for <JAIL>"],
 ["set <JAIL> usedns <VALUE>", "sets the usedns mode for <JAIL>"],
-["set <JAIL> banip <IP>", "manually Ban <IP> for <JAIL>"], 
-["set <JAIL> unbanip <IP>", "manually Unban <IP> in <JAIL>"], 
+["set <JAIL> attempt <IP> [<failure1> ... <failureN>]", "manually notify about <IP> failure"], 
+["set <JAIL> banip <IP> ... <IP>", "manually Ban <IP> for <JAIL>"], 
+["set <JAIL> unbanip [--report-absent] <IP> ... <IP>", "manually Unban <IP> in <JAIL>"], 
 ["set <JAIL> maxretry <RETRY>", "sets the number of failures <RETRY> before banning the host for <JAIL>"], 
 ["set <JAIL> maxlines <LINES>", "sets the number of <LINES> to buffer for regex search for <JAIL>"], 
 ["set <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]", "adds a new action named <ACT> for <JAIL>. Optionally for a Python based action, a <PYTHONFILE> and <JSONKWARGS> can be specified, else will be a Command Action"], 

fail2ban/server/actions.py

@@ -34,7 +34,8 @@ try:
 except ImportError:
 	OrderedDict = dict
 
-from .banmanager import BanManager
+from .banmanager import BanManager, BanTicket
+from .ipdns import IPAddr
 from .jailthread import JailThread
 from .action import ActionBase, CommandAction, CallingMap
 from .mytime import MyTime
@@ -203,6 +204,19 @@ class Actions(JailThread, Mapping):
 	def getBanTime(self):
 		return self.__banManager.getBanTime()
 
+	def addBannedIP(self, ip):
+		"""Ban an IP or list of IPs."""
+		unixTime = MyTime.time()
+
+		if isinstance(ip, list):
+			# Multiple IPs:
+			tickets = (BanTicket(ip if isinstance(ip, IPAddr) else IPAddr(ip), unixTime) for ip in ip)
+		else:
+			# Single IP:
+			tickets = (BanTicket(ip if isinstance(ip, IPAddr) else IPAddr(ip), unixTime),)
+
+		return self.__checkBan(tickets)
+
 	def removeBannedIP(self, ip=None, db=True, ifexists=False):
 		"""Removes banned IP calling actions' unban method
 
@@ -211,8 +225,8 @@ class Actions(JailThread, Mapping):
 
 		Parameters
 		----------
-		ip : str or IPAddr or None
-			The IP address to unban or all IPs if None
+		ip : list, str, IPAddr or None
+			The IP address (or multiple IPs as list) to unban or all IPs if None
 
 		Raises
 		------
@@ -222,6 +236,19 @@ class Actions(JailThread, Mapping):
 		# Unban all?
 		if ip is None:
 			return self.__flushBan(db)
+		# Multiple IPs:
+		if isinstance(ip, list):
+			missed = []
+			cnt = 0
+			for i in ip:
+				try:
+					cnt += self.removeBannedIP(i, db, ifexists)
+				except ValueError:
+					if not ifexists:
+						missed.append(i)
+			if missed:
+				raise ValueError("not banned: %r" % missed)
+			return cnt
 		# Single IP:
 		# Always delete ip from database (also if currently not banned)
 		if db and self._jail.database is not None:
@@ -232,9 +259,11 @@ class Actions(JailThread, Mapping):
 			# Unban the IP.
 			self.__unBan(ticket)
 		else:
+			msg = "%s is not banned" % ip
+			logSys.log(logging.MSG, msg)
 			if ifexists:
 				return 0
-			raise ValueError("%s is not banned" % ip)
+			raise ValueError(msg)
 		return 1
 
 
@@ -373,11 +402,20 @@ class Actions(JailThread, Mapping):
 		aInfo = Actions.ActionInfo(ticket, self._jail)
 		return aInfo
 
+	def __getFailTickets(self, count=100):
+		"""Generator to get maximal count failure tickets from fail-manager."""
+		cnt = 0
+		while cnt < count:
+			ticket = self._jail.getFailTicket()
+			if not ticket:
+				break
+			yield ticket
+			cnt += 1
 
-	def __checkBan(self):
+	def __checkBan(self, tickets=None):
 		"""Check for IP address to ban.
 
-		Look in the jail queue for FailTicket. If a ticket is available,
+		If tickets are not specified look in the jail queue for FailTicket. If a ticket is available,
 		it executes the "ban" command and adds a ticket to the BanManager.
 
 		Returns
@@ -386,10 +424,9 @@ class Actions(JailThread, Mapping):
 			True if an IP address get banned.
 		"""
 		cnt = 0
-		while cnt < 100:
-			ticket = self._jail.getFailTicket()
-			if not ticket:
-				break
+		if not tickets:
+			tickets = self.__getFailTickets()
+		for ticket in tickets:
 			bTicket = BanManager.createBanTicket(ticket)
 			ip = bTicket.getIP()
 			aInfo = self.__getActionInfo(bTicket)

fail2ban/server/failmanager.py

@@ -159,7 +159,7 @@ class FailManager:
 	
 	def toBan(self, fid=None):
 		with self.__lock:
-			for fid in ([fid] if fid != None and fid in self.__failList else self.__failList):
+			for fid in ([fid] if fid is not None and fid in self.__failList else self.__failList):
 				data = self.__failList[fid]
 				if data.getRetry() >= self.__maxRetry:
 					del self.__failList[fid]

fail2ban/server/filter.py

@@ -427,23 +427,9 @@ class Filter(JailThread):
 			)
 		else:
 			self.__ignoreCache = None
-	##
-	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
-	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
-	#
-	# to enable banip fail2ban-client BAN command
 
-	def addBannedIP(self, ip):
-		if not isinstance(ip, IPAddr):
-			ip = IPAddr(ip)
-
-		unixTime = MyTime.time()
-		ticket = FailTicket(ip, unixTime)
-		if self._inIgnoreIPList(ip, ticket, log_ignore=False):
-			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
-		self.failManager.addFailure(ticket, self.failManager.getMaxRetry())
-
-		# Perform the banning of the IP now.
+	def performBan(self, ip=None):
+		"""Performs a ban for IPs (or given ip) that are reached maxretry of the jail."""
 		try: # pragma: no branch - exception is the only way out
 			while True:
 				ticket = self.failManager.toBan(ip)
@@ -451,7 +437,24 @@ class Filter(JailThread):
 		except FailManagerEmpty:
 			self.failManager.cleanup(MyTime.time())
 
-		return ip
+	def addAttempt(self, ip, *matches):
+		"""Generate a failed attempt for ip"""
+		if not isinstance(ip, IPAddr):
+			ip = IPAddr(ip)
+		matches = list(matches) # tuple to list
+
+		# Generate the failure attempt for the IP:
+		unixTime = MyTime.time()
+		ticket = FailTicket(ip, unixTime, matches=matches)
+		logSys.info(
+			"[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
+		)
+		self.failManager.addFailure(ticket, len(matches) or 1)
+
+		# Perform the ban if this attempt is resulted to:
+		self.performBan(ip)
+
+		return 1
 
 	##
 	# Ignore own IP/DNS.

fail2ban/server/filtergamin.py

@@ -79,12 +79,7 @@ class FilterGamin(FileFilter):
 		this is a common logic and must be shared/provided by FileFilter
 		"""
 		self.getFailures(path)
-		try:
-			while True:
-				ticket = self.failManager.toBan()
-				self.jail.putFailTicket(ticket)
-		except FailManagerEmpty:
-			self.failManager.cleanup(MyTime.time())
+		self.performBan()
 		self.__modified = False
 
 	##

fail2ban/server/filterpoll.py

@@ -117,12 +117,7 @@ class FilterPoll(FileFilter):
 
 				self.ticks += 1
 				if self.__modified:
-					try:
-						while True:
-							ticket = self.failManager.toBan()
-							self.jail.putFailTicket(ticket)
-					except FailManagerEmpty:
-						self.failManager.cleanup(MyTime.time())
+					self.performBan()
 					self.__modified = False
 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...

fail2ban/server/filterpyinotify.py

@@ -140,12 +140,7 @@ class FilterPyinotify(FileFilter):
 		"""
 		if not self.idle:
 			self.getFailures(path)
-			try:
-				while True:
-					ticket = self.failManager.toBan()
-					self.jail.putFailTicket(ticket)
-			except FailManagerEmpty:
-				self.failManager.cleanup(MyTime.time())
+			self.performBan()
 			self.__modified = False
 
 	def _addPending(self, path, reason, isDir=False):

fail2ban/server/filtersystemd.py

@@ -300,12 +300,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 					else:
 						break
 				if self.__modified:
-					try:
-						while True:
-							ticket = self.failManager.toBan()
-							self.jail.putFailTicket(ticket)
-					except FailManagerEmpty:
-						self.failManager.cleanup(MyTime.time())
+					self.performBan()
+					self.__modified = 0
 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...
 					break

fail2ban/server/server.py

@@ -473,20 +473,24 @@ class Server:
 	def setBanTime(self, name, value):
 		self.__jails[name].actions.setBanTime(value)
 	
+	def addAttemptIP(self, name, *args):
+		return self.__jails[name].filter.addAttempt(*args)
+
 	def setBanIP(self, name, value):
-		return self.__jails[name].filter.addBannedIP(value)
-		
-	def setUnbanIP(self, name=None, value=None):
+		return self.__jails[name].actions.addBannedIP(value)
+
+	def setUnbanIP(self, name=None, value=None, ifexists=True):
 		if name is not None:
-			# in all jails:
+			# single jail:
 			jails = [self.__jails[name]]
 		else:
-			# single jail:
+			# in all jails:
 			jails = self.__jails.values()
 		# unban given or all (if value is None):
 		cnt = 0
+		ifexists |= (name is None)
 		for jail in jails:
-			cnt += jail.actions.removeBannedIP(value, ifexists=(name is None))
+			cnt += jail.actions.removeBannedIP(value, ifexists=ifexists)
 		if value and not cnt:
 			logSys.info("%s is not banned", value)
 		return cnt

fail2ban/server/transmitter.py

@@ -109,10 +109,7 @@ class Transmitter:
 			# if all ips:
 			if len(value) == 1 and value[0] == "--all":
 				return self.__server.setUnbanIP()
-			cnt = 0
-			for value in value:
-				cnt += self.__server.setUnbanIP(None, value)
-			return cnt
+			return self.__server.setUnbanIP(None, value)
 		elif command[0] == "echo":
 			return command[1:]
 		elif command[0] == "server-status":
@@ -189,8 +186,8 @@ class Transmitter:
 			self.__server.setIgnoreSelf(name, value)
 			return self.__server.getIgnoreSelf(name)
 		elif command[1] == "addignoreip":
-			value = command[2]
-			self.__server.addIgnoreIP(name, value)
+			for value in command[2:]:
+				self.__server.addIgnoreIP(name, value)
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "delignoreip":
 			value = command[2]
@@ -285,13 +282,20 @@ class Transmitter:
 			value = command[2]
 			self.__server.setBanTime(name, value)
 			return self.__server.getBanTime(name)
+		elif command[1] == "attempt":
+			value = command[2:]
+			return self.__server.addAttemptIP(name, *value)
 		elif command[1] == "banip":
-			value = command[2]
+			value = command[2:]
 			return self.__server.setBanIP(name,value)
 		elif command[1] == "unbanip":
-			value = command[2]
-			self.__server.setUnbanIP(name, value)
-			return value
+			ifexists = True
+			if command[2] != "--report-absent":
+				value = command[2:]
+			else:
+				ifexists = False
+				value = command[3:]
+			return self.__server.setUnbanIP(name, value, ifexists=ifexists)
 		elif command[1] == "addaction":
 			args = [command[2]]
 			if len(command) > 3:

fail2ban/tests/actionstestcase.py

@@ -78,6 +78,16 @@ class ExecuteActions(LogCaptureTestCase):
 		self.assertEqual(self.__actions.getBanTime(),127)
 		self.assertRaises(ValueError, self.__actions.removeBannedIP, '127.0.0.1')
 
+	def testAddBannedIP(self):
+		self.assertEqual(self.__actions.addBannedIP('192.0.2.1'), 1)
+		self.assertLogged('Ban 192.0.2.1')
+		self.pruneLog()
+		self.assertEqual(self.__actions.addBannedIP(['192.0.2.1', '192.0.2.2', '192.0.2.3']), 2)
+		self.assertLogged('192.0.2.1 already banned')
+		self.assertNotLogged('Ban 192.0.2.1')
+		self.assertLogged('Ban 192.0.2.2')
+		self.assertLogged('Ban 192.0.2.3')
+
 	def testActionsOutput(self):
 		self.defaultActions()
 		self.__actions.start()

fail2ban/tests/fail2banclienttestcase.py

@@ -1121,7 +1121,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"--async", "unban", "192.0.2.5", "192.0.2.6")
 		self.assertLogged(
 			"192.0.2.5 is not banned",
-			"[test-jail1] Unban 192.0.2.6", all=True
+			"[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME
 		)
 
 		# reload all (one jail) with unban all:
@@ -1194,6 +1194,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"Jail 'test-jail1' stopped", 
 			"Jail 'test-jail1' started", all=True, wait=MID_WAITTIME)
 
+		# Coverage for pickle of IPAddr (as string):
+		self.pruneLog("[test-phase end-3]")
+		self.execCmd(SUCCESS, startparams,
+			"--async", "set", "test-jail1", "addignoreip", "192.0.2.1/32", "2001:DB8::1/96")
+		self.execCmd(SUCCESS, startparams,
+			"--async", "get", "test-jail1", "ignoreip")
+		self.assertLogged("192.0.2.1/32", "2001:DB8::1/96", all=True)
+
 	# test action.d/nginx-block-map.conf --
 	@unittest.F2B.skip_if_cfg_missing(action="nginx-block-map")
 	@with_foreground_server_thread(startextra={

fail2ban/tests/filtertestcase.py

@@ -394,11 +394,13 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertLogged('Ignore 192.168.1.32')
 		tearDownMyTime()
 
-	def testIgnoreAddBannedIP(self):
-		self.filter.addIgnoreIP('192.168.1.0/25')
-		self.filter.addBannedIP('192.168.1.32')
-		self.assertNotLogged('Ignore 192.168.1.32')
-		self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')
+	def testAddAttempt(self):
+		self.filter.setMaxRetry(3)
+		for i in xrange(1, 1+3):
+			self.filter.addAttempt('192.0.2.1')
+			self.assertLogged('Attempt 192.0.2.1', '192.0.2.1:%d' % i, all=True, wait=True)
+		self.jail.actions._Actions__checkBan()
+		self.assertLogged('Ban 192.0.2.1', wait=True)
 
 	def testIgnoreCommand(self):
 		self.filter.ignoreCommand = sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>")

fail2ban/tests/servertestcase.py

@@ -330,22 +330,49 @@ class Transmitter(TransmitterBase):
 		self.server.startJail(self.jailName) # Jail must be started
 
 		self.assertEqual(
-			self.transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]),
-			(0, "127.0.0.1"))
-		self.assertLogged("Ban 127.0.0.1", wait=True) # Give chance to ban
+			self.transm.proceed(["set", self.jailName, "banip", "192.0.2.1", "192.0.2.1", "192.0.2.2"]),
+			(0, 2))
+		self.assertLogged("Ban 192.0.2.1", "Ban 192.0.2.2", all=True, wait=True) # Give chance to ban
 		self.assertEqual(
 			self.transm.proceed(["set", self.jailName, "banip", "Badger"]),
-			(0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done?
+			(0, 1)) #NOTE: Is IP address validated? Is DNS Lookup done?
 		self.assertLogged("Ban Badger", wait=True) # Give chance to ban
-		# Unban IP
+		# Unban IP (first/last are not banned, so checking unban of both other succeeds):
 		self.assertEqual(
 			self.transm.proceed(
-				["set", self.jailName, "unbanip", "127.0.0.1"]),
-			(0, "127.0.0.1"))
-		# Unban IP which isn't banned
+				["set", self.jailName, "unbanip", "192.0.2.255", "192.0.2.1", "192.0.2.2", "192.0.2.254"]),
+			(0, 2))
+		self.assertLogged("Unban 192.0.2.1", "Unban 192.0.2.2", all=True, wait=True)
+		self.assertLogged("192.0.2.255 is not banned", "192.0.2.254 is not banned", all=True, wait=True)
+		self.pruneLog()
+		# Unban IP which isn't banned (error):
 		self.assertEqual(
 			self.transm.proceed(
-				["set", self.jailName, "unbanip", "192.168.1.1"])[0],1)
+				["set", self.jailName, "unbanip", "--report-absent", "192.0.2.255"])[0],1)
+		# ... (no error, IPs logged only):
+		self.assertEqual(
+			self.transm.proceed(
+				["set", self.jailName, "unbanip", "192.0.2.255", "192.0.2.254"]),(0, 0))
+		self.assertLogged("192.0.2.255 is not banned", "192.0.2.254 is not banned", all=True, wait=True)
+
+	def testJailAttemptIP(self):
+		self.server.startJail(self.jailName) # Jail must be started
+
+		def attempt(ip, matches):
+			return self.transm.proceed(["set", self.jailName, "attempt", ip] + matches)
+
+		self.setGetTest("maxretry", "5", 5, jail=self.jailName)
+		# produce 2 single attempts per IP:
+		for i in (1, 2):
+			for ip in ("192.0.2.1", "192.0.2.2"):
+				self.assertEqual(attempt(ip, ["test failure %d" % i]), (0, 1))
+		self.assertLogged("192.0.2.1:2", "192.0.2.2:2", all=True, wait=True)
+		# this 3 attempts at once should cause a ban:
+		self.assertEqual(attempt(ip, ["test failure %d" % i for i in (3,4,5)]), (0, 1))
+		self.assertLogged("192.0.2.2:5", wait=True)
+		# resulted to ban for "192.0.2.2" but not for "192.0.2.1":
+		self.assertLogged("Ban 192.0.2.2", wait=True)
+		self.assertNotLogged("Ban 192.0.2.1")
 
 	def testJailMaxRetry(self):
 		self.setGetTest("maxretry", "5", 5, jail=self.jailName)

man/fail2ban-client.1

@@ -272,10 +272,13 @@ date/times for <JAIL>
 \fBset <JAIL> usedns <VALUE>\fR
 sets the usedns mode for <JAIL>
 .TP
-\fBset <JAIL> banip <IP>\fR
+\fBset <JAIL> attempt <IP> [<failure1> ... <failureN>]\fR
+manually notify about <IP> failure
+.TP
+\fBset <JAIL> banip <IP> ... <IP>\fR
 manually Ban <IP> for <JAIL>
 .TP
-\fBset <JAIL> unbanip <IP>\fR
+\fBset <JAIL> unbanip [\-\-report\-absent] <IP> ... <IP>\fR
 manually Unban <IP> in <JAIL>
 .TP
 \fBset <JAIL> maxretry <RETRY>\fR