temp commit: partially cherry picked from ban-time-incr branch

fail2ban/protocol.py

@@ -52,9 +52,9 @@ protocol = [
 ["restart [--unban] [--if-exists] <JAIL>", "restarts the jail <JAIL> (alias for 'reload --restart ... <JAIL>')"], 
 ["reload [--restart] [--unban] [--all]", "reloads the configuration without restarting of the server, the option '--restart' activates completely restarting of affected jails, thereby unbans IP addresses (if option '--unban' specified)"],
 ["reload [--restart] [--unban] [--if-exists] <JAIL>", "reloads the jail <JAIL>, or restarts it (if option '--restart' specified)"],
-["unban --all", "unbans all IP addresses (in all jails)"],
-["unban <IP> ... <IP>", "unbans <IP> (in all jails)"],
 ["stop", "stops all jails and terminate the server"], 
+["unban --all", "unbans all IP addresses (in all jails and database)"],
+["unban <IP> ... <IP>", "unbans <IP> (in all jails and database)"],
 ["status", "gets the current status of the server"], 
 ["ping", "tests if the server is alive"],
 ["echo", "for internal usage, returns back and outputs a given string"],

fail2ban/server/actions.py

@@ -320,7 +320,7 @@ class Actions(JailThread, Mapping):
 				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
 				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
 			if self.__banManager.addBanTicket(bTicket):
-				logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"]))
+				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.getRestored() else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
 						action.ban(aInfo.copy())

fail2ban/server/database.py

@@ -576,6 +576,49 @@ class Fail2BanDb(object):
 				self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 			return tickets if ip is None else ticket
 
+	@commitandrollback
+	def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None):
+		if fromtime is None:
+			fromtime = MyTime.time()
+		queryArgs = []
+		if jail is not None:
+			query = "SELECT ip, timeofban, data FROM bans WHERE jail=?"
+			queryArgs.append(jail.name)
+		else:
+			query = "SELECT ip, max(timeofban), data FROM bans WHERE 1"
+		if ip is not None:
+			query += " AND ip=?"
+			queryArgs.append(ip)
+		if forbantime is not None:
+			query += " AND timeofban > ?"
+			queryArgs.append(fromtime - forbantime)
+		if ip is None:
+			query += " GROUP BY ip ORDER BY ip, timeofban DESC"
+		cur = self._db.cursor()
+		return cur.execute(query, queryArgs)
+
+	def getCurrentBans(self, jail = None, ip = None, forbantime=None, fromtime=None):
+		tickets = []
+		ticket = None
+
+		results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime, fromtime=fromtime))
+
+		if results:
+			matches = []
+			failures = 0
+			for banip, timeofban, data in results:
+				#TODO: Implement data parts once arbitrary match keys completed
+				ticket = FailTicket(banip, timeofban, matches)
+				ticket.setAttempt(failures)
+				matches = []
+				failures = 0
+				matches.extend(data['matches'])
+				failures += data['failures']
+				ticket.setAttempt(failures)
+				tickets.append(ticket)
+
+		return tickets if ip is None else ticket
+
 	@commitandrollback
 	def purge(self, cur):
 		"""Purge old bans, jails and log files from database.

fail2ban/server/jail.py

@@ -194,7 +194,7 @@ class Jail(object):
 		Used by filter to add a failure for banning.
 		"""
 		self.__queue.put(ticket)
-		if self.database is not None:
+		if not ticket.getRestored() and self.database is not None:
 			self.database.addBan(self, ticket)
 
 	def getFailTicket(self):
@@ -207,6 +207,21 @@ class Jail(object):
 		except Queue.Empty:
 			return False
 
+	def restoreCurrentBans(self):
+		"""Restore any previous valid bans from the database.
+		"""
+		try:
+			if self.database is not None:
+				forbantime = self.actions.getBanTime()
+				for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime):
+					#logSys.debug('restored ticket: %s', ticket)
+					if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
+						# mark ticked was restored from database - does not put it again into db:
+						ticket.setRestored(True)
+						self.putFailTicket(ticket)
+		except Exception as e: # pragma: no cover
+			logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+
 	def start(self):
 		"""Start the jail, by starting filter and actions threads.
 
@@ -215,12 +230,7 @@ class Jail(object):
 		"""
 		self.filter.start()
 		self.actions.start()
-		# Restore any previous valid bans from the database
+		self.restoreCurrentBans()
-		if self.database is not None:
-			for ticket in self.database.getBansMerged(
-				jail=self, bantime=self.actions.getBanTime()):
-				if not self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True):
-					self.__queue.put(ticket)
 		logSys.info("Jail %r started", self.name)
 
 	def stop(self):

fail2ban/server/ticket.py

@@ -36,6 +36,8 @@ logSys = getLogger(__name__)
 
 class Ticket:
 	
+	RESTORED = 0x01
+
 	def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None):
 		"""Ticket constructor
 
@@ -126,6 +128,15 @@ class Ticket:
 	def getMatches(self):
 		return self._data.get('matches', [])
 
+	def setRestored(self, value):
+		if value:
+			self._flags = Ticket.RESTORED
+		else:
+			self._flags &= ~(Ticket.RESTORED)
+	
+	def getRestored(self):
+		return self._flags & Ticket.RESTORED
+
 	def setData(self, *args, **argv):
 		# if overwrite - set data and filter None values:
 		if len(args) == 1:

fail2ban/tests/fail2banclienttestcase.py

@@ -58,6 +58,7 @@ SERVER = "fail2ban-server"
 BIN = dirname(Fail2banServer.getServerPath())
 
 MAX_WAITTIME = 30 if not unittest.F2B.fast else 5
+MID_WAITTIME = MAX_WAITTIME / 2.5
 
 ##
 # Several wrappers and settings for proper testing:
@@ -663,14 +664,14 @@ class Fail2banServerTest(Fail2banClientServerBase):
 
 	@with_foreground_server_thread(startextra={'db': 'auto'})
 	def testServerReloadTest(self, tmp, startparams):
-		"""Very complicated test-case, that expected running server (foreground in thread).
+		# Very complicated test-case, that expected running server (foreground in thread).
-
+		#
-		In this test-case, each phase is related from previous one, 
+		# In this test-case, each phase is related from previous one, 
-		so it cannot be splitted in multiple test cases.
+		# so it cannot be splitted in multiple test cases.
-
+		# Additionaly many log-messages used as ready-sign (to wait for end of phase).
-		It uses file database (instead of :memory:), to restore bans and log-file positions,
+		#
-		after restart/reload between phases.
+		# Used file database (instead of :memory:), to restore bans and log-file positions,
-		"""
+		# after restart/reload between phases.
 		cfg = pjoin(tmp, "config")
 		test1log = pjoin(tmp, "test1.log")
 		test2log = pjoin(tmp, "test2.log")
@@ -710,7 +711,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("[test-jail1] Ban 192.0.2.1"), MID_WAITTIME))
 		self.assertLogged("Added logfile: %r" % test1log)
 		
 		# enable both jails, 3 logs for jail1, etc...
@@ -722,7 +723,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			_out_file(test1log)
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# test not unbanned / banned again:
 		self.assertNotLogged(
 			"[test-jail1] Unban 192.0.2.1", 
@@ -741,7 +742,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_file(test2log, "w+", *(
 			(str(int(MyTime.time())) + "   error 403 from 192.0.2.2: test 2",) * 3 +
 		  (str(int(MyTime.time())) + "   error 403 from 192.0.2.3: test 2",) * 3 +
-		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3
+		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 +
+		  (str(int(MyTime.time())) + " failure 401 from 192.0.2.8: test 2",) * 3
 		))
 		if DefLogSys.level < logging.DEBUG:  # if HEAVYDEBUG
 			_out_file(test2log)
@@ -751,8 +753,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				self._is_logged("[test-jail1] Ban 192.0.2.2") and 
 				self._is_logged("[test-jail1] Ban 192.0.2.3") and 
 				self._is_logged("[test-jail1] Ban 192.0.2.4") and 
-				self._is_logged("[test-jail2] Ban 192.0.2.4")
+				self._is_logged("[test-jail1] Ban 192.0.2.8") and 
-				, MAX_WAITTIME / 5.0))
+				self._is_logged("[test-jail2] Ban 192.0.2.4") and
+				self._is_logged("[test-jail2] Ban 192.0.2.8")
+				, MID_WAITTIME))
 		# test ips at all not visible for jail2:
 		self.assertNotLogged(
 			"[test-jail2] Found 192.0.2.2", 
@@ -771,14 +775,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertTrue(
 			Utils.wait_for(lambda: \
 				self._is_logged("Jail 'test-jail2' started") and
-				self._is_logged("[test-jail2] Ban 192.0.2.4")
+				self._is_logged("[test-jail2] Restore Ban 192.0.2.4") and
-				, MAX_WAITTIME / 5.0))
+				self._is_logged("[test-jail2] Restore Ban 192.0.2.8")
-		# stop/start and ban/unban:
+				, MID_WAITTIME))
+		# stop/start and unban/restore ban:
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
 			"[test-jail2] Unban 192.0.2.4",
-			"[test-jail2] Ban 192.0.2.4", all=True
+			"[test-jail2] Unban 192.0.2.8",
+			"[test-jail2] Restore Ban 192.0.2.4",
+			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
 
 		# restart jail with unban all:
@@ -787,22 +794,24 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"restart", "--unban", "test-jail2")
 		self.assertTrue(
 			Utils.wait_for(lambda: self._is_logged("Jail 'test-jail2' started"),
-				MAX_WAITTIME / 5.0))
+				MID_WAITTIME))
 		self.assertLogged(
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
-			"[test-jail2] Unban 192.0.2.4", all=True
+			"[test-jail2] Unban 192.0.2.4",
+			"[test-jail2] Unban 192.0.2.8", all=True
 		)
 		# no more ban (unbanned all):
 		self.assertNotLogged(
-			"[test-jail2] Ban 192.0.2.4", all=True
+			"[test-jail2] Ban 192.0.2.4",
+			"[test-jail2] Ban 192.0.2.8", all=True
 		)
 
 		# reload jail1 without restart (without ban/unban):
 		self.pruneLog("[test-phase 3]")
 		self.execSuccess(startparams, "reload", "test-jail1")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		self.assertLogged(
 			"Reload jail 'test-jail1'",
 			"Jail 'test-jail1' reloaded", all=True)
@@ -817,7 +826,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_jail_cfg(enabled=[1])
 		self.execSuccess(startparams, "reload")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# test both jails should be reloaded:
 		self.assertLogged(
 			"Reload jail 'test-jail1'")
@@ -844,7 +853,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			Utils.wait_for(lambda: \
 				self._is_logged("[test-jail1] 192.0.2.1 already banned") and
 				self._is_logged("[test-jail1] Ban 192.0.2.6")
-				, MAX_WAITTIME / 5.0))
+				, MID_WAITTIME))
 		self.assertLogged(
 			"[test-jail1] Found 192.0.2.1",
 			"[test-jail1] Found 192.0.2.6", all=True
@@ -866,7 +875,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.execSuccess(startparams,
 			"reload", "--unban")
 		self.assertTrue(
-			Utils.wait_for(lambda: self._is_logged("Reload finished."), MAX_WAITTIME / 5.0))
+			Utils.wait_for(lambda: self._is_logged("Reload finished."), MID_WAITTIME))
 		# reloads unbanned all:
 		self.assertLogged(
 			"Jail 'test-jail1' reloaded",
@@ -885,7 +894,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail1] Ban 192.0.2.4", all=True
 		)
 
-		# several small cases:
+		# several small cases (cover several parts):
 		self.pruneLog("[test-phase end-1]")
 		# wrong jail (not-started):
 		self.execFailed(startparams,