"magic" formula for auto increasing of retry count for known (bad) ip, corresponding banCount of it

(one try will count than 2, 3, 5, 9 ...)

fail2ban/server/actions.py

@@ -324,9 +324,13 @@ class Actions(JailThread, Mapping):
 					if banCount > 0:
 						banTime = be['evformula'](self.BanTimeIncr(banTime, banCount))
 					bTicket.setBanTime(banTime);
-					logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, 
+					# check current ticket time to prevent increasing for twice read tickets (restored from log file besides database after restart)
-						datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), 
+					if bTicket.getTime() > timeOfBan:
-						datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime))));
+						logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, 
+							datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), 
+							datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime))));
+					else:
+						bTicket.setRestored(True)
 					break
 		except Exception as e:
 			logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
@@ -372,23 +376,27 @@ class Actions(JailThread, Mapping):
 					self._jail.database.getBansMerged(
 						ip=ip, jail=self._jail).getAttempt())
 				try:
-					# if ban time was not set:
+					# if not permanent, not restored and ban time was not set:
-					if not ticket.getRestored() and bTicket.getBanTime() is None:
+					if btime != -1 and not ticket.getRestored() and bTicket.getBanTime() is None:
 						btime = self.incrBanTime(bTicket)
-					bTicket.setBanTime(btime);
+						bTicket.setBanTime(btime);
 				except Exception as e:
 					logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 					#logSys.error('%s', e, exc_info=True)
 
+			if btime != -1:
+				logtime = (datetime.timedelta(seconds=int(btime)),
+					datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S"))
+			else:
+				logtime = ('permanent', 'infinite')
 			if self.__banManager.addBanTicket(bTicket):
 				if self._jail.database is not None:
 					# add to database always only after ban time was calculated an not yet already banned:
 					# if ticked was not restored from database - put it into database:
-					if not ticket.getRestored():
+					if not ticket.getRestored() and not bTicket.getRestored():
 						self._jail.database.addBan(self._jail, bTicket)
-				logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % (self._jail.name, ('Resore ' if ticket.getRestored() else ''),
+				logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % ((self._jail.name, ('Resore ' if ticket.getRestored() else ''),
-					aInfo["ip"], bTicket.getBanCount(), datetime.timedelta(seconds=int(btime)),
+					aInfo["ip"], bTicket.getBanCount()) + logtime))
-					datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S")))
 				for name, action in self._actions.iteritems():
 					try:
 						action.ban(aInfo)
@@ -399,8 +407,8 @@ class Actions(JailThread, Mapping):
 							exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 				return True
 			else:
-				logSys.notice("[%s] %s already banned" % (self._jail.name,
+				logSys.notice("[%s] %s already banned (%d # %s -> %s)" % ((self._jail.name,
-														aInfo["ip"]))
+					aInfo["ip"], bTicket.getBanCount()) + logtime))
 		return False
 
 	def __checkUnBan(self):

fail2ban/server/banmanager.py

@@ -130,10 +130,11 @@ class BanManager:
 	def createBanTicket(ticket):
 		ip = ticket.getIP()
 		# if ticked was restored from database - set time of original restored ticket:
-		if ticket.getRestored():
+		# we should always use correct time to calculate correct end time (ban time is variable now, 
-			lastTime = ticket.getTime()
+		# + possible double banning by restore from database and from log file)
-		else:
+		lastTime = ticket.getTime()
-			lastTime = MyTime.time()
+		# if not ticket.getRestored():
+		# 	lastTime = MyTime.time()
 		banTicket = BanTicket(ip, lastTime, ticket.getMatches())
 		banTicket.setAttempt(ticket.getAttempt())
 		return banTicket
@@ -149,11 +150,25 @@ class BanManager:
 	def addBanTicket(self, ticket):
 		try:
 			self.__lock.acquire()
-			if not self._inBanList(ticket):
+			# check already banned
-				self.__banList.append(ticket)
+			for i in self.__banList:
-				self.__banTotal += 1
+				if ticket.getIP() == i.getIP():
-				return True
+					# if already permanent
-			return False
+					btorg, torg = i.getBanTime(self.__banTime), i.getTime()
+					if btorg == -1:
+						return False
+					# if given time is less than already banned time
+					btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime()
+					if btnew != -1 and tnew + btnew <= torg + btorg:
+						return False
+					# we have longest ban - set new (increment) ban time
+					i.setTime(tnew)
+					i.setBanTime(btnew)
+					return False
+			# not yet banned - add new
+			self.__banList.append(ticket)
+			self.__banTotal += 1
+			return True
 		finally:
 			self.__lock.release()
 	
@@ -199,8 +214,7 @@ class BanManager:
 				return list()
 
 			# Gets the list of ticket to remove.
-			unBanList = [ticket for ticket in self.__banList
+			unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)]
-						 if ticket.getTime() < time - ticket.getBanTime(self.__banTime)]
 			
 			# Removes tickets.
 			self.__banList = [ticket for ticket in self.__banList

fail2ban/server/database.py

@@ -87,7 +87,7 @@ class Fail2BanDb(object):
 	filename
 	purgeage
 	"""
-	__version__ = 3
+	__version__ = 4
 	# Note all _TABLE_* strings must end in ';' for py26 compatibility
 	_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
 	_TABLE_jails = "CREATE TABLE jails(" \
@@ -123,10 +123,20 @@ class Fail2BanDb(object):
 			"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
 			"CREATE INDEX bans_ip ON bans(ip);" \
 
-  # todo: for performance reasons create a table with currently banned unique jails-ips only (with last ban of time and bantime).
+	_TABLE_bips = "CREATE TABLE bips(" \
-  #       check possible view performance instead of new table;
+			"ip TEXT NOT NULL, " \
+			"jail TEXT NOT NULL, " \
+			"timeofban INTEGER NOT NULL, " \
+			"bantime INTEGER NOT NULL, " \
+			"bancount INTEGER NOT NULL default 1, " \
+			"data JSON, " \
+			"PRIMARY KEY(ip, jail), " \
+			"FOREIGN KEY(jail) REFERENCES jails(name) " \
+			");" \
+			"CREATE INDEX bips_timeofban ON bips(timeofban);" \
+			"CREATE INDEX bips_ip ON bips(ip);" \
 
-	def __init__(self, filename, purgeAge=24*60*60):
+	def __init__(self, filename, purgeAge=24*60*60, outDatedFactor=3):
 		try:
 			self._lock = Lock()
 			self._db = sqlite3.connect(
@@ -134,6 +144,7 @@ class Fail2BanDb(object):
 				detect_types=sqlite3.PARSE_DECLTYPES)
 			self._dbFilename = filename
 			self._purgeAge = purgeAge
+			self._outDatedFactor = outDatedFactor;
 
 			self._bansMergedCache = {}
 
@@ -198,6 +209,8 @@ class Fail2BanDb(object):
 		cur.executescript(Fail2BanDb._TABLE_logs)
 		# Bans
 		cur.executescript(Fail2BanDb._TABLE_bans)
+		# BIPs (bad ips)
+		cur.executescript(Fail2BanDb._TABLE_bips)
 
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
@@ -232,8 +245,12 @@ class Fail2BanDb(object):
 						"%s;"
 						"INSERT INTO bans SELECT * from bans_temp;"
 						"DROP TABLE bans_temp;"
-						"UPDATE fail2banDb SET version = 3;"
 						"COMMIT;" % Fail2BanDb._TABLE_bans)
+		if version < 4:
+			cur.executescript("BEGIN TRANSACTION;"
+						"%s;"
+						"UPDATE fail2banDb SET version = 4;"
+						"COMMIT;" % Fail2BanDb._TABLE_bips)
 
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
@@ -386,6 +403,11 @@ class Fail2BanDb(object):
 			(jail.name, ticket.getIP(), ticket.getTime(), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount() + 1,
 				{"matches": ticket.getMatches(),
 					"failures": ticket.getAttempt()}))
+		cur.execute(
+			"INSERT OR REPLACE INTO bips(ip, jail, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
+			(ticket.getIP(), jail.name, ticket.getTime(), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount() + 1,
+				{"matches": ticket.getMatches(),
+					"failures": ticket.getAttempt()}))
 
 	@commitandrollback
 	def _getBans(self, cur, jail=None, bantime=None, ip=None):
@@ -487,42 +509,47 @@ class Fail2BanDb(object):
 			self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 		return tickets if ip is None else ticket
 
-	def getBan(self, ip, jail=None, forbantime=None, overalljails=None):
+	def getBan(self, ip, jail=None, forbantime=None, overalljails=None, fromtime=None):
-		#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
+		if not overalljails:
-		if overalljails is None or not overalljails:
+			query = "SELECT bancount, timeofban, bantime FROM bips"
-			query = "SELECT bancount, max(timeofban), bantime FROM bans"
 		else:
-			query = "SELECT max(bancount), max(timeofban), bantime FROM bans"
+			query = "SELECT max(bancount), max(timeofban), max(bantime) FROM bips"
 		query += " WHERE ip = ?"
 		queryArgs = [ip]
-		if (overalljails is None or not overalljails) and jail is not None:
+		if not overalljails and jail is not None:
 			query += " AND jail=?"
 			queryArgs.append(jail.name)
 		if forbantime is not None:
 			query += " AND timeofban > ?"
 			queryArgs.append(MyTime.time() - forbantime)
-		query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1"
+		if fromtime is not None:
+			query += " AND timeofban > ?"
+			queryArgs.append(fromtime)
+		if overalljails or jail is None:
+			query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1"
 		cur = self._db.cursor()
+		#logSys.debug((query, queryArgs));
 		return cur.execute(query, queryArgs)
 
 	def _getCurrentBans(self, jail = None, ip = None, forbantime=None, fromtime=None):
 		if fromtime is None:
 			fromtime = MyTime.time()
-		#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
-		query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bans WHERE 1"
 		queryArgs = []
 		if jail is not None:
-			query += " AND jail=?"
+			query = "SELECT ip, timeofban, bantime, bancount, data FROM bips WHERE jail=?"
 			queryArgs.append(jail.name)
+		else:
+			query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bips WHERE 1"
 		if ip is not None:
 			query += " AND ip=?"
 			queryArgs.append(ip)
-		query += " AND timeofban + bantime > ?"
+		query += " AND (timeofban + bantime > ? OR bantime = -1)"
 		queryArgs.append(fromtime)
 		if forbantime is not None:
 			query += " AND timeofban > ?"
 			queryArgs.append(fromtime - forbantime)
-		query += " GROUP BY ip ORDER BY ip, timeofban DESC"
+		if ip is None:
+			query += " GROUP BY ip ORDER BY ip, timeofban DESC"
 		cur = self._db.cursor()
 		#logSys.debug((query, queryArgs));
 		return cur.execute(query, queryArgs)
@@ -558,15 +585,35 @@ class Fail2BanDb(object):
 			self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
 		return tickets if ip is None else ticket
 
-	@commitandrollback
+	def _cleanjails(self, cur):
-	def purge(self, cur):
+		"""Remove empty jails jails and log files from database.
+		"""
+		cur.execute(
+			"DELETE FROM jails WHERE enabled = 0 "
+				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name) "
+				"AND NOT EXISTS(SELECT * FROM bips WHERE jail = jails.name)")
+
+	def _purge_bips(self, cur):
+		"""Purge old bad ips (jails and log files from database).
+		Currently it is timed out IP, whose time since last ban is several times out-dated (outDatedFactor is default 3).
+		Permanent banned ips will be never removed.
+		"""
+		cur.execute(
+			"DELETE FROM bips WHERE timeofban < ? and bantime != -1 and (timeofban + (bantime * ?)) < ?",
+			(int(MyTime.time()) - self._purgeAge, self._outDatedFactor, int(MyTime.time()) - self._purgeAge))
+
+	#@commitandrollback
+	def purge(self):
 		"""Purge old bans, jails and log files from database.
 		"""
+		cur = self._db.cursor()
 		self._bansMergedCache = {}
 		cur.execute(
 			"DELETE FROM bans WHERE timeofban < ?",
 			(MyTime.time() - self._purgeAge, ))
-		cur.execute(
+		affected = cur.rowcount
-			"DELETE FROM jails WHERE enabled = 0 "
+		self._purge_bips(cur)
-				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name)")
+		affected += cur.rowcount
+		if affected:
+			self._cleanjails(cur)
 

fail2ban/server/faildata.py

@@ -52,8 +52,8 @@ class FailData:
 	def getMatches(self):
 		return self.__matches
 
-	def inc(self, matches=None):
+	def inc(self, matches=None, count=1):
-		self.__retry += 1
+		self.__retry += count
 		self.__matches += matches or []
 
 	def setLastTime(self, value):

fail2ban/server/failmanager.py

@@ -84,7 +84,7 @@ class FailManager:
 		finally:
 			self.__lock.release()
 
-	def addFailure(self, ticket):
+	def addFailure(self, ticket, count=1):
 		try:
 			self.__lock.acquire()
 			ip = ticket.getIP()
@@ -95,11 +95,11 @@ class FailManager:
 				if fData.getLastReset() < unixTime - self.__maxTime:
 					fData.setLastReset(unixTime)
 					fData.setRetry(0)
-				fData.inc(matches)
+				fData.inc(matches, count)
 				fData.setLastTime(unixTime)
 			else:
 				fData = FailData()
-				fData.inc(matches)
+				fData.inc(matches, count)
 				fData.setLastReset(unixTime)
 				fData.setLastTime(unixTime)
 				self.__failList[ip] = fData

fail2ban/server/filter.py

@@ -420,9 +420,26 @@ class Filter(JailThread):
 			if self.inIgnoreIPList(ip):
 				logSys.info("[%s] Ignore %s" % (self.jail.name, ip))
 				continue
-			logSys.info("[%s] Found %s" % (self.jail.name, ip))
+			# increase retry count for known (bad) ip, corresponding banCount of it (one try will count than 2, 3, 5, 9 ...)  :
-			## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),)
+			banCount = 0
-			self.failManager.addFailure(FailTicket(ip, unixTime, lines))
+			retryCount = 1
+			db = self.jail.database
+			if db is not None:
+				try:
+					for banCount, timeOfBan, lastBanTime in db.getBan(ip, self.jail):
+						retryCount = ((1 << (banCount if banCount < 20 else 20))/2 + 1)
+						# if lastBanTime == -1 or timeOfBan + lastBanTime * 2 > MyTime.time():
+						# 	retryCount = self.failManager.getMaxRetry()
+						break
+					retryCount = min(retryCount, self.failManager.getMaxRetry())
+				except Exception as e:
+					#logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+					logSys.error('%s', e, exc_info=True)
+			if banCount == 1 and retryCount == 1:
+				logSys.info("[%s] Found %s" % (self.jail.name, ip))
+			else:
+				logSys.info("[%s] Found %s, %s # -> %s" % (self.jail.name, ip, banCount, retryCount))
+			self.failManager.addFailure(FailTicket(ip, unixTime, lines), retryCount)
 
 	##
 	# Returns true if the line should be ignored.

fail2ban/server/ticket.py

@@ -91,6 +91,14 @@ class Ticket:
 	def getBanCount(self):
 		return self.__banCount;
 
+	def isTimedOut(self, time, defaultBT = None):
+		bantime = (self.__banTime if not self.__banTime is None else defaultBT);
+		# permanent
+		if bantime == -1:
+			return False
+		# timed out
+		return (time > self.__time + bantime)
+
 	def setAttempt(self, value):
 		self.__attempt = value
 	

fail2ban/tests/databasetestcase.py

@@ -407,3 +407,95 @@ class BanTimeIncr(unittest.TestCase):
 			str(restored_tickets[2]),
 			'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'2', stime-24*60*60, 12*60*60)
 		)
+		# should be still banned
+		self.assertFalse(restored_tickets[1].isTimedOut(stime))
+		self.assertFalse(restored_tickets[1].isTimedOut(stime))
+		# the last should be timed out now
+		self.assertTrue(restored_tickets[2].isTimedOut(stime))
+		self.assertFalse(restored_tickets[2].isTimedOut(stime-18*60*60))
+
+		# test permanent, create timed out:
+		ticket=FailTicket(ip+'3', stime-36*60*60, [])
+		self.assertTrue(ticket.isTimedOut(stime, 600))
+		# not timed out - permanent jail:
+		self.assertFalse(ticket.isTimedOut(stime, -1))
+		# not timed out - permanent ticket:
+		ticket.setBanTime(-1)
+		self.assertFalse(ticket.isTimedOut(stime, 600))
+		self.assertFalse(ticket.isTimedOut(stime, -1))
+		# timed out - permanent jail but ticket time (not really used behavior)
+		ticket.setBanTime(600)
+		self.assertTrue(ticket.isTimedOut(stime, -1))
+
+		# get currently banned pis with permanent one:
+		ticket.setBanTime(-1)
+		self.db.addBan(jail, ticket)
+		restored_tickets = self.db.getCurrentBans(fromtime=stime)
+		self.assertEqual(len(restored_tickets), 3)
+		self.assertEqual(
+			str(restored_tickets[2]),
+			'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'3', stime-36*60*60, -1)
+		)
+		# purge (nothing should be changed):
+		self.db.purge()
+		restored_tickets = self.db.getCurrentBans(fromtime=stime)
+		self.assertEqual(len(restored_tickets), 3)
+		# set short time and purge again:
+		ticket.setBanTime(600)
+		self.db.addBan(jail, ticket)
+		self.db.purge()
+		# this old ticket should be removed now:
+		restored_tickets = self.db.getCurrentBans(fromtime=stime)
+		self.assertEqual(len(restored_tickets), 2)
+		self.assertEqual(restored_tickets[0].getIP(), ip)
+
+    # purge remove 1st ip
+		self.db._purgeAge = -48*60*60
+		self.db.purge()
+		restored_tickets = self.db.getCurrentBans(fromtime=stime)
+		self.assertEqual(len(restored_tickets), 1)
+		self.assertEqual(restored_tickets[0].getIP(), ip+'1')
+
+    # this should purge all bans, bips and logs - nothing should be found now
+		self.db._purgeAge = -240*60*60
+		self.db.purge()
+		restored_tickets = self.db.getCurrentBans(fromtime=stime)
+		self.assertEqual(restored_tickets, [])
+
+    # two separate jails :
+		jail1 = DummyJail()
+		jail1.database = self.db
+		self.db.addJail(jail1)
+		jail2 = DummyJail()
+		jail2.database = self.db
+		self.db.addJail(jail2)
+		ticket1 = FailTicket(ip, stime, [])
+		ticket1.setBanTime(6000)
+		self.db.addBan(jail1, ticket1)
+		ticket2 = FailTicket(ip, stime-6000, [])
+		ticket2.setBanTime(12000)
+		ticket2.setBanCount(1)
+		self.db.addBan(jail2, ticket2)
+		restored_tickets = self.db.getCurrentBans(jail=jail1, fromtime=stime)
+		self.assertEqual(len(restored_tickets), 1)
+		self.assertEqual(
+			str(restored_tickets[0]),
+			'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip, stime, 6000)
+		)
+		restored_tickets = self.db.getCurrentBans(jail=jail2, fromtime=stime)
+		self.assertEqual(len(restored_tickets), 1)
+		self.assertEqual(
+			str(restored_tickets[0]),
+			'FailTicket: ip=%s time=%s bantime=%s bancount=2 #attempts=0 matches=[]' % (ip, stime-6000, 12000)
+		)
+    # get last ban values for this ip separately for each jail:
+		for row in self.db.getBan(ip, jail1):
+			self.assertEqual(row, (1, stime, 6000))
+			break
+		for row in self.db.getBan(ip, jail2):
+			self.assertEqual(row, (2, stime-6000, 12000))
+			break
+    # get max values for this ip (over all jails):
+		for row in self.db.getBan(ip, overalljails=True):
+			self.assertEqual(row, (2, stime, 12000))
+			break