diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 9a2dfa13..0cc15bc6 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -324,9 +324,13 @@ class Actions(JailThread, Mapping): if banCount > 0: banTime = be['evformula'](self.BanTimeIncr(banTime, banCount)) bTicket.setBanTime(banTime); - logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, - datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), - datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + # check current ticket time to prevent increasing for twice read tickets (restored from log file besides database after restart) + if bTicket.getTime() > timeOfBan: + logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, + datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), + datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + else: + bTicket.setRestored(True) break except Exception as e: logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) @@ -372,23 +376,27 @@ class Actions(JailThread, Mapping): self._jail.database.getBansMerged( ip=ip, jail=self._jail).getAttempt()) try: - # if ban time was not set: - if not ticket.getRestored() and bTicket.getBanTime() is None: + # if not permanent, not restored and ban time was not set: + if btime != -1 and not ticket.getRestored() and bTicket.getBanTime() is None: btime = self.incrBanTime(bTicket) - bTicket.setBanTime(btime); + bTicket.setBanTime(btime); except Exception as e: logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) #logSys.error('%s', e, exc_info=True) + if btime != -1: + logtime = (datetime.timedelta(seconds=int(btime)), + datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S")) + else: + logtime = ('permanent', 'infinite') if self.__banManager.addBanTicket(bTicket): if self._jail.database is not None: # add to database always only after ban time was calculated an not yet already banned: # if ticked was not restored from database - put it into database: - if not ticket.getRestored(): + if not ticket.getRestored() and not bTicket.getRestored(): self._jail.database.addBan(self._jail, bTicket) - logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % (self._jail.name, ('Resore ' if ticket.getRestored() else ''), - aInfo["ip"], bTicket.getBanCount(), datetime.timedelta(seconds=int(btime)), - datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S"))) + logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % ((self._jail.name, ('Resore ' if ticket.getRestored() else ''), + aInfo["ip"], bTicket.getBanCount()) + logtime)) for name, action in self._actions.iteritems(): try: action.ban(aInfo) @@ -399,8 +407,8 @@ class Actions(JailThread, Mapping): exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) return True else: - logSys.notice("[%s] %s already banned" % (self._jail.name, - aInfo["ip"])) + logSys.notice("[%s] %s already banned (%d # %s -> %s)" % ((self._jail.name, + aInfo["ip"], bTicket.getBanCount()) + logtime)) return False def __checkUnBan(self): diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 5d614cf5..8d0ae191 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -130,10 +130,11 @@ class BanManager: def createBanTicket(ticket): ip = ticket.getIP() # if ticked was restored from database - set time of original restored ticket: - if ticket.getRestored(): - lastTime = ticket.getTime() - else: - lastTime = MyTime.time() + # we should always use correct time to calculate correct end time (ban time is variable now, + # + possible double banning by restore from database and from log file) + lastTime = ticket.getTime() + # if not ticket.getRestored(): + # lastTime = MyTime.time() banTicket = BanTicket(ip, lastTime, ticket.getMatches()) banTicket.setAttempt(ticket.getAttempt()) return banTicket @@ -149,11 +150,25 @@ class BanManager: def addBanTicket(self, ticket): try: self.__lock.acquire() - if not self._inBanList(ticket): - self.__banList.append(ticket) - self.__banTotal += 1 - return True - return False + # check already banned + for i in self.__banList: + if ticket.getIP() == i.getIP(): + # if already permanent + btorg, torg = i.getBanTime(self.__banTime), i.getTime() + if btorg == -1: + return False + # if given time is less than already banned time + btnew, tnew = ticket.getBanTime(self.__banTime), ticket.getTime() + if btnew != -1 and tnew + btnew <= torg + btorg: + return False + # we have longest ban - set new (increment) ban time + i.setTime(tnew) + i.setBanTime(btnew) + return False + # not yet banned - add new + self.__banList.append(ticket) + self.__banTotal += 1 + return True finally: self.__lock.release() @@ -199,8 +214,7 @@ class BanManager: return list() # Gets the list of ticket to remove. - unBanList = [ticket for ticket in self.__banList - if ticket.getTime() < time - ticket.getBanTime(self.__banTime)] + unBanList = [ticket for ticket in self.__banList if ticket.isTimedOut(time, self.__banTime)] # Removes tickets. self.__banList = [ticket for ticket in self.__banList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index b75501cc..85b6160b 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -87,7 +87,7 @@ class Fail2BanDb(object): filename purgeage """ - __version__ = 3 + __version__ = 4 # Note all _TABLE_* strings must end in ';' for py26 compatibility _TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);" _TABLE_jails = "CREATE TABLE jails(" \ @@ -123,10 +123,20 @@ class Fail2BanDb(object): "CREATE INDEX bans_jail_ip ON bans(jail, ip);" \ "CREATE INDEX bans_ip ON bans(ip);" \ - # todo: for performance reasons create a table with currently banned unique jails-ips only (with last ban of time and bantime). - # check possible view performance instead of new table; + _TABLE_bips = "CREATE TABLE bips(" \ + "ip TEXT NOT NULL, " \ + "jail TEXT NOT NULL, " \ + "timeofban INTEGER NOT NULL, " \ + "bantime INTEGER NOT NULL, " \ + "bancount INTEGER NOT NULL default 1, " \ + "data JSON, " \ + "PRIMARY KEY(ip, jail), " \ + "FOREIGN KEY(jail) REFERENCES jails(name) " \ + ");" \ + "CREATE INDEX bips_timeofban ON bips(timeofban);" \ + "CREATE INDEX bips_ip ON bips(ip);" \ - def __init__(self, filename, purgeAge=24*60*60): + def __init__(self, filename, purgeAge=24*60*60, outDatedFactor=3): try: self._lock = Lock() self._db = sqlite3.connect( @@ -134,6 +144,7 @@ class Fail2BanDb(object): detect_types=sqlite3.PARSE_DECLTYPES) self._dbFilename = filename self._purgeAge = purgeAge + self._outDatedFactor = outDatedFactor; self._bansMergedCache = {} @@ -198,6 +209,8 @@ class Fail2BanDb(object): cur.executescript(Fail2BanDb._TABLE_logs) # Bans cur.executescript(Fail2BanDb._TABLE_bans) + # BIPs (bad ips) + cur.executescript(Fail2BanDb._TABLE_bips) cur.execute("SELECT version FROM fail2banDb LIMIT 1") return cur.fetchone()[0] @@ -232,8 +245,12 @@ class Fail2BanDb(object): "%s;" "INSERT INTO bans SELECT * from bans_temp;" "DROP TABLE bans_temp;" - "UPDATE fail2banDb SET version = 3;" "COMMIT;" % Fail2BanDb._TABLE_bans) + if version < 4: + cur.executescript("BEGIN TRANSACTION;" + "%s;" + "UPDATE fail2banDb SET version = 4;" + "COMMIT;" % Fail2BanDb._TABLE_bips) cur.execute("SELECT version FROM fail2banDb LIMIT 1") return cur.fetchone()[0] @@ -386,6 +403,11 @@ class Fail2BanDb(object): (jail.name, ticket.getIP(), ticket.getTime(), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount() + 1, {"matches": ticket.getMatches(), "failures": ticket.getAttempt()})) + cur.execute( + "INSERT OR REPLACE INTO bips(ip, jail, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)", + (ticket.getIP(), jail.name, ticket.getTime(), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount() + 1, + {"matches": ticket.getMatches(), + "failures": ticket.getAttempt()})) @commitandrollback def _getBans(self, cur, jail=None, bantime=None, ip=None): @@ -487,42 +509,47 @@ class Fail2BanDb(object): self._bansMergedCache[cacheKey] = tickets if ip is None else ticket return tickets if ip is None else ticket - def getBan(self, ip, jail=None, forbantime=None, overalljails=None): - #query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?" - if overalljails is None or not overalljails: - query = "SELECT bancount, max(timeofban), bantime FROM bans" + def getBan(self, ip, jail=None, forbantime=None, overalljails=None, fromtime=None): + if not overalljails: + query = "SELECT bancount, timeofban, bantime FROM bips" else: - query = "SELECT max(bancount), max(timeofban), bantime FROM bans" + query = "SELECT max(bancount), max(timeofban), max(bantime) FROM bips" query += " WHERE ip = ?" queryArgs = [ip] - if (overalljails is None or not overalljails) and jail is not None: + if not overalljails and jail is not None: query += " AND jail=?" queryArgs.append(jail.name) if forbantime is not None: query += " AND timeofban > ?" queryArgs.append(MyTime.time() - forbantime) - query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" + if fromtime is not None: + query += " AND timeofban > ?" + queryArgs.append(fromtime) + if overalljails or jail is None: + query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" cur = self._db.cursor() + #logSys.debug((query, queryArgs)); return cur.execute(query, queryArgs) def _getCurrentBans(self, jail = None, ip = None, forbantime=None, fromtime=None): if fromtime is None: fromtime = MyTime.time() - #query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?" - query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bans WHERE 1" queryArgs = [] if jail is not None: - query += " AND jail=?" + query = "SELECT ip, timeofban, bantime, bancount, data FROM bips WHERE jail=?" queryArgs.append(jail.name) + else: + query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bips WHERE 1" if ip is not None: query += " AND ip=?" queryArgs.append(ip) - query += " AND timeofban + bantime > ?" + query += " AND (timeofban + bantime > ? OR bantime = -1)" queryArgs.append(fromtime) if forbantime is not None: query += " AND timeofban > ?" queryArgs.append(fromtime - forbantime) - query += " GROUP BY ip ORDER BY ip, timeofban DESC" + if ip is None: + query += " GROUP BY ip ORDER BY ip, timeofban DESC" cur = self._db.cursor() #logSys.debug((query, queryArgs)); return cur.execute(query, queryArgs) @@ -558,15 +585,35 @@ class Fail2BanDb(object): self._bansMergedCache[cacheKey] = tickets if ip is None else ticket return tickets if ip is None else ticket - @commitandrollback - def purge(self, cur): + def _cleanjails(self, cur): + """Remove empty jails jails and log files from database. + """ + cur.execute( + "DELETE FROM jails WHERE enabled = 0 " + "AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name) " + "AND NOT EXISTS(SELECT * FROM bips WHERE jail = jails.name)") + + def _purge_bips(self, cur): + """Purge old bad ips (jails and log files from database). + Currently it is timed out IP, whose time since last ban is several times out-dated (outDatedFactor is default 3). + Permanent banned ips will be never removed. + """ + cur.execute( + "DELETE FROM bips WHERE timeofban < ? and bantime != -1 and (timeofban + (bantime * ?)) < ?", + (int(MyTime.time()) - self._purgeAge, self._outDatedFactor, int(MyTime.time()) - self._purgeAge)) + + #@commitandrollback + def purge(self): """Purge old bans, jails and log files from database. """ + cur = self._db.cursor() self._bansMergedCache = {} cur.execute( "DELETE FROM bans WHERE timeofban < ?", (MyTime.time() - self._purgeAge, )) - cur.execute( - "DELETE FROM jails WHERE enabled = 0 " - "AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name)") + affected = cur.rowcount + self._purge_bips(cur) + affected += cur.rowcount + if affected: + self._cleanjails(cur) diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py index 232a492d..937c1cfa 100644 --- a/fail2ban/server/faildata.py +++ b/fail2ban/server/faildata.py @@ -52,8 +52,8 @@ class FailData: def getMatches(self): return self.__matches - def inc(self, matches=None): - self.__retry += 1 + def inc(self, matches=None, count=1): + self.__retry += count self.__matches += matches or [] def setLastTime(self, value): diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 548e6adb..e4a18f8c 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -84,7 +84,7 @@ class FailManager: finally: self.__lock.release() - def addFailure(self, ticket): + def addFailure(self, ticket, count=1): try: self.__lock.acquire() ip = ticket.getIP() @@ -95,11 +95,11 @@ class FailManager: if fData.getLastReset() < unixTime - self.__maxTime: fData.setLastReset(unixTime) fData.setRetry(0) - fData.inc(matches) + fData.inc(matches, count) fData.setLastTime(unixTime) else: fData = FailData() - fData.inc(matches) + fData.inc(matches, count) fData.setLastReset(unixTime) fData.setLastTime(unixTime) self.__failList[ip] = fData diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index fb5aeb3d..521e4fbc 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -420,9 +420,26 @@ class Filter(JailThread): if self.inIgnoreIPList(ip): logSys.info("[%s] Ignore %s" % (self.jail.name, ip)) continue - logSys.info("[%s] Found %s" % (self.jail.name, ip)) - ## print "D: Adding a ticket for %s" % ((ip, unixTime, [line]),) - self.failManager.addFailure(FailTicket(ip, unixTime, lines)) + # increase retry count for known (bad) ip, corresponding banCount of it (one try will count than 2, 3, 5, 9 ...) : + banCount = 0 + retryCount = 1 + db = self.jail.database + if db is not None: + try: + for banCount, timeOfBan, lastBanTime in db.getBan(ip, self.jail): + retryCount = ((1 << (banCount if banCount < 20 else 20))/2 + 1) + # if lastBanTime == -1 or timeOfBan + lastBanTime * 2 > MyTime.time(): + # retryCount = self.failManager.getMaxRetry() + break + retryCount = min(retryCount, self.failManager.getMaxRetry()) + except Exception as e: + #logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + logSys.error('%s', e, exc_info=True) + if banCount == 1 and retryCount == 1: + logSys.info("[%s] Found %s" % (self.jail.name, ip)) + else: + logSys.info("[%s] Found %s, %s # -> %s" % (self.jail.name, ip, banCount, retryCount)) + self.failManager.addFailure(FailTicket(ip, unixTime, lines), retryCount) ## # Returns true if the line should be ignored. diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 6c7c2f2c..d339dbb3 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -91,6 +91,14 @@ class Ticket: def getBanCount(self): return self.__banCount; + def isTimedOut(self, time, defaultBT = None): + bantime = (self.__banTime if not self.__banTime is None else defaultBT); + # permanent + if bantime == -1: + return False + # timed out + return (time > self.__time + bantime) + def setAttempt(self, value): self.__attempt = value diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 05bd8f9f..3e32683a 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -407,3 +407,95 @@ class BanTimeIncr(unittest.TestCase): str(restored_tickets[2]), 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'2', stime-24*60*60, 12*60*60) ) + # should be still banned + self.assertFalse(restored_tickets[1].isTimedOut(stime)) + self.assertFalse(restored_tickets[1].isTimedOut(stime)) + # the last should be timed out now + self.assertTrue(restored_tickets[2].isTimedOut(stime)) + self.assertFalse(restored_tickets[2].isTimedOut(stime-18*60*60)) + + # test permanent, create timed out: + ticket=FailTicket(ip+'3', stime-36*60*60, []) + self.assertTrue(ticket.isTimedOut(stime, 600)) + # not timed out - permanent jail: + self.assertFalse(ticket.isTimedOut(stime, -1)) + # not timed out - permanent ticket: + ticket.setBanTime(-1) + self.assertFalse(ticket.isTimedOut(stime, 600)) + self.assertFalse(ticket.isTimedOut(stime, -1)) + # timed out - permanent jail but ticket time (not really used behavior) + ticket.setBanTime(600) + self.assertTrue(ticket.isTimedOut(stime, -1)) + + # get currently banned pis with permanent one: + ticket.setBanTime(-1) + self.db.addBan(jail, ticket) + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 3) + self.assertEqual( + str(restored_tickets[2]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'3', stime-36*60*60, -1) + ) + # purge (nothing should be changed): + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 3) + # set short time and purge again: + ticket.setBanTime(600) + self.db.addBan(jail, ticket) + self.db.purge() + # this old ticket should be removed now: + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 2) + self.assertEqual(restored_tickets[0].getIP(), ip) + + # purge remove 1st ip + self.db._purgeAge = -48*60*60 + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual(restored_tickets[0].getIP(), ip+'1') + + # this should purge all bans, bips and logs - nothing should be found now + self.db._purgeAge = -240*60*60 + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(restored_tickets, []) + + # two separate jails : + jail1 = DummyJail() + jail1.database = self.db + self.db.addJail(jail1) + jail2 = DummyJail() + jail2.database = self.db + self.db.addJail(jail2) + ticket1 = FailTicket(ip, stime, []) + ticket1.setBanTime(6000) + self.db.addBan(jail1, ticket1) + ticket2 = FailTicket(ip, stime-6000, []) + ticket2.setBanTime(12000) + ticket2.setBanCount(1) + self.db.addBan(jail2, ticket2) + restored_tickets = self.db.getCurrentBans(jail=jail1, fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual( + str(restored_tickets[0]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip, stime, 6000) + ) + restored_tickets = self.db.getCurrentBans(jail=jail2, fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual( + str(restored_tickets[0]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=2 #attempts=0 matches=[]' % (ip, stime-6000, 12000) + ) + # get last ban values for this ip separately for each jail: + for row in self.db.getBan(ip, jail1): + self.assertEqual(row, (1, stime, 6000)) + break + for row in self.db.getBan(ip, jail2): + self.assertEqual(row, (2, stime-6000, 12000)) + break + # get max values for this ip (over all jails): + for row in self.db.getBan(ip, overalljails=True): + self.assertEqual(row, (2, stime, 12000)) + break