diff --git a/config/jail.conf b/config/jail.conf index 96b3096f..05aa96c4 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -44,6 +44,30 @@ before = paths-debian.conf # MISCELLANEOUS OPTIONS # +# "bantimeextra.enabled" allows to use database for searching of previously banned ip's to increase a +# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32... +bantimeextra.enabled = true +# "bantimeextra.findtime" is the max number of seconds that we search in the database, +# if it is not specified - whole database will be used for ban searching +# (please observe current "dbpurgeage" value of fail2ban.conf). +bantimeextra.findtime = 24*60*60 +# "bantimeextra.rndtime" is the max number of seconds using for mixing with random time +# to prevent "clever" botnets calculate exact time IP can be unbanned again: +bantimeextra.rndtime = 5*60 +# "bantimeextra.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +bantimeextra.maxtime = 24*60*60 +# "bantimeextra.factor" is a coefficient to calculate exponent growing of the formula, +# by default value of factor "2.0 / 2.885385" and default value of formula, the ban time +# grows by 1, 2, 4, 8, 16 ... +#bantimeextra.factor = 2.0 / 2.885385 +# "bantimeextra.formula" used to calculate next value of ban time; +#bantimeextra.formula = banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor) +# "bantimeextra.overalljails" (if true) specifies the search of IP in the database will be executed +# cross over all jails, if false (dafault), only current jail of the ban IP will be searched +#bantimeextra.overalljails = false + +# -------------------- + # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not # ban a host which matches an address in this list. Several addresses can be # defined using space separator. diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 5735b021..85b136d9 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -93,6 +93,13 @@ class JailReader(ConfigReader): ["int", "maxretry", None], ["int", "findtime", None], ["int", "bantime", None], + ["bool", "bantimeextra.enabled", False], + ["string", "bantimeextra.findtime", None], + ["string", "bantimeextra.factor", None], + ["string", "bantimeextra.formula", None], + ["string", "bantimeextra.maxtime", None], + ["string", "bantimeextra.rndtime", None], + ["bool", "bantimeextra.overalljails", None], ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], @@ -198,6 +205,8 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "findtime", self.__opts[opt]]) elif opt == "bantime": stream.append(["set", self.__name, "bantime", self.__opts[opt]]) + elif opt.startswith("bantimeextra."): + stream.append(["set", self.__name, opt, self.__opts[opt]]) elif opt == "usedns": stream.append(["set", self.__name, "usedns", self.__opts[opt]]) elif opt == "failregex": diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index d5799ca1..73848b55 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import time, logging -import os +import os, datetime, math, json, random import sys if sys.version_info >= (3, 3): import importlib.machinery @@ -83,6 +83,8 @@ class Actions(JailThread, Mapping): self._actions = dict() ## The ban manager. self.__banManager = BanManager() + ## Extra parameters for increase ban time + self._banExtra = {'maxtime': 24*60*60}; def add(self, name, pythonModule=None, initOpts=None): """Adds a new action. @@ -240,6 +242,71 @@ class Actions(JailThread, Mapping): logSys.debug(self._jail.name + ": action terminated") return True + def setBanTimeExtra(self, opt, value): + # merge previous extra with new option: + be = self._banExtra; + be[opt] = value; + logSys.info('Set banTimeExtra.%s = %s', opt, value) + if opt == 'enabled': + be[opt] = bool(value) + if bool(value) and self._jail.database is None: + logSys.warning("banTimeExtra is not available as long jail database is not set") + if opt in ['findtime', 'maxtime', 'rndtime']: + if not value is None: + be[opt] = eval(value) + if opt == 'factor' or be.get('factor', None) is None: + be['factor'] = eval(be.get('factor', "2.0 / 2.885385")); + # prepare formula : + if opt in ['formula', 'maxtime', 'rndtime'] or be.get('evformula', None) is None: + be['formula'] = be.get('formula', 'banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor)') + evformula = be['formula']; + evformula = ('max(banTime, %s)' % evformula) + if not be.get('maxtime', None) is None: + evformula = ('min(%s, %s)' % (evformula, be['maxtime'])) + # mix with random time (to prevent botnet calc exact time IP can be unbanned): + if not be.get('rndtime', None) is None: + evformula = ('(%s + random.random() * %s)' % (evformula, be['rndtime'])) + # set to extra dict: + be['evformula'] = evformula + #logSys.info('banTimeExtra : %s' % json.dumps(be)) + + def getBanTimeExtra(self, opt): + return self._banExtra.get(opt, None) + + def incrBanTime(self, bTicket, ip): + """Check for IP address to increment ban time (if was already banned). + + Returns + ------- + float + new ban time. + """ + orgBanTime = self.__banManager.getBanTime() + banTime = orgBanTime + # check ip was already banned (increment time of ban): + try: + be = self._banExtra; + if banTime > 0 and be.get('enabled', False): + banFactor = be['factor']; + # search IP in database and increase time if found: + for banCount, timeOfBan, lastBanTime in \ + self._jail.database.getBan(ip, self._jail, be.get('findtime', None), be.get('overalljails', False) \ + ): + #logSys.debug('IP %s was already banned: %s #, %s' % (ip, banCount, timeOfBan)); + bTicket.setBanCount(banCount); + # calculate new ban time + banTime = eval(be['evformula']) + bTicket.setBanTime(banTime); + logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, + datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), + datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + break + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + + return banTime + def __checkBan(self): """Check for IP address to ban. @@ -255,25 +322,46 @@ class Actions(JailThread, Mapping): if ticket != False: aInfo = CallingMap() bTicket = BanManager.createBanTicket(ticket) - aInfo["ip"] = bTicket.getIP() + if ticket.getBanTime() is not None: + bTicket.setBanTime(ticket.getBanTime()) + bTicket.setBanCount(ticket.getBanCount()) + ip = bTicket.getIP() + aInfo["ip"] = ip aInfo["failures"] = bTicket.getAttempt() aInfo["time"] = bTicket.getTime() aInfo["matches"] = "\n".join(bTicket.getMatches()) + btime = bTicket.getBanTime(self.__banManager.getBanTime()); if self._jail.database is not None: aInfo["ipmatches"] = lambda: "\n".join( self._jail.database.getBansMerged( - ip=bTicket.getIP()).getMatches()) + ip=ip).getMatches()) aInfo["ipjailmatches"] = lambda: "\n".join( self._jail.database.getBansMerged( - ip=bTicket.getIP(), jail=self._jail).getMatches()) + ip=ip, jail=self._jail).getMatches()) aInfo["ipfailures"] = lambda: "\n".join( self._jail.database.getBansMerged( - ip=bTicket.getIP()).getAttempt()) + ip=ip).getAttempt()) aInfo["ipjailfailures"] = lambda: "\n".join( self._jail.database.getBansMerged( - ip=bTicket.getIP(), jail=self._jail).getAttempt()) + ip=ip, jail=self._jail).getAttempt()) + try: + # if ban time was not set: + if bTicket.getBanTime() is None: + btime = self.incrBanTime(bTicket, ip) + bTicket.setBanTime(btime); + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + if self.__banManager.addBanTicket(bTicket): - logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) + if self._jail.database is not None: + # add to database always only after ban time was calculated an not yet already banned: + # if ticked was not restored from database - put it into database: + if not ticket.getRestored(): + self._jail.database.addBan(self._jail, bTicket) + logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % (self._jail.name, ('Resore ' if ticket.getRestored() else ''), + aInfo["ip"], bTicket.getBanCount(), datetime.timedelta(seconds=int(btime)), + datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S"))) for name, action in self._actions.iteritems(): try: action.ban(aInfo) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index fc9eb948..5d614cf5 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -129,8 +129,11 @@ class BanManager: #@staticmethod def createBanTicket(ticket): ip = ticket.getIP() - #lastTime = ticket.getTime() - lastTime = MyTime.time() + # if ticked was restored from database - set time of original restored ticket: + if ticket.getRestored(): + lastTime = ticket.getTime() + else: + lastTime = MyTime.time() banTicket = BanTicket(ip, lastTime, ticket.getMatches()) banTicket.setAttempt(ticket.getAttempt()) return banTicket @@ -197,7 +200,7 @@ class BanManager: # Gets the list of ticket to remove. unBanList = [ticket for ticket in self.__banList - if ticket.getTime() < time - self.__banTime] + if ticket.getTime() < time - ticket.getBanTime(self.__banTime)] # Removes tickets. self.__banList = [ticket for ticket in self.__banList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 93186222..136f3e7a 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -87,7 +87,7 @@ class Fail2BanDb(object): filename purgeage """ - __version__ = 2 + __version__ = 3 # Note all _TABLE_* strings must end in ';' for py26 compatibility _TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);" _TABLE_jails = "CREATE TABLE jails(" \ @@ -114,6 +114,8 @@ class Fail2BanDb(object): "jail TEXT NOT NULL, " \ "ip TEXT, " \ "timeofban INTEGER NOT NULL, " \ + "bantime INTEGER NOT NULL, " \ + "bancount INTEGER NOT NULL, " \ "data JSON, " \ "FOREIGN KEY(jail) REFERENCES jails(name) " \ ");" \ @@ -121,6 +123,9 @@ class Fail2BanDb(object): "CREATE INDEX bans_jail_ip ON bans(jail, ip);" \ "CREATE INDEX bans_ip ON bans(ip);" \ + # todo: for performance reasons create a table with currently banned unique jails-ips only (with last ban of time and bantime). + # check possible view performance instead of new table; + def __init__(self, filename, purgeAge=24*60*60): try: self._lock = Lock() @@ -220,6 +225,16 @@ class Fail2BanDb(object): "UPDATE fail2banDb SET version = 2;" "COMMIT;" % Fail2BanDb._TABLE_logs) + if version < 3: + cur.executescript("BEGIN TRANSACTION;" + "CREATE TEMPORARY TABLE bans_temp AS SELECT jail, ip, timeofban, 600 as bantime, 1 as bancount, data FROM bans;" + "DROP TABLE bans;" + "%s;" + "INSERT INTO bans SELECT * from bans_temp;" + "DROP TABLE bans_temp;" + "UPDATE fail2banDb SET version = 3;" + "COMMIT;" % Fail2BanDb._TABLE_bans) + cur.execute("SELECT version FROM fail2banDb LIMIT 1") return cur.fetchone()[0] @@ -367,8 +382,8 @@ class Fail2BanDb(object): pass #TODO: Implement data parts once arbitrary match keys completed cur.execute( - "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", - (jail.name, ticket.getIP(), ticket.getTime(), + "INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)", + (jail.name, ticket.getIP(), ticket.getTime(), ticket.getBanTime(), ticket.getBanCount() + 1, {"matches": ticket.getMatches(), "failures": ticket.getAttempt()})) @@ -472,6 +487,75 @@ class Fail2BanDb(object): self._bansMergedCache[cacheKey] = tickets if ip is None else ticket return tickets if ip is None else ticket + def getBan(self, ip, jail=None, forbantime=None, overalljails=None): + #query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?" + if overalljails is None or not overalljails: + query = "SELECT bancount, max(timeofban), bantime FROM bans" + else: + query = "SELECT max(bancount), max(timeofban), bantime FROM bans" + query += " WHERE ip = ?" + queryArgs = [ip] + if (overalljails is None or not overalljails) and jail is not None: + query += " AND jail=?" + queryArgs.append(jail.name) + if forbantime is not None: + query += " AND timeofban > ?" + queryArgs.append(MyTime.time() - forbantime) + query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" + cur = self._db.cursor() + return cur.execute(query, queryArgs) + + def _getCurrentBans(self, jail = None, ip = None, forbantime=None): + #query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?" + query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bans WHERE 1" + queryArgs = [] + if jail is not None: + query += " AND jail=?" + queryArgs.append(jail.name) + if ip is not None: + query += " AND ip=?" + queryArgs.append(ip) + query += " AND timeofban + bantime > ?" + queryArgs.append(MyTime.time()) + if forbantime is not None: + query += " AND timeofban > ?" + queryArgs.append(MyTime.time() - forbantime) + query += " GROUP BY ip ORDER BY ip, timeofban DESC" + cur = self._db.cursor() + #logSys.debug((query, queryArgs)); + return cur.execute(query, queryArgs) + + def getCurrentBans(self, jail = None, ip = None, forbantime=None): + if forbantime is None: + cacheKey = (ip, jail) + if cacheKey in self._bansMergedCache: + return self._bansMergedCache[cacheKey] + + tickets = [] + ticket = None + + results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime)) + + if results: + matches = [] + failures = 0 + for banip, timeofban, bantime, bancount, data in results: + #TODO: Implement data parts once arbitrary match keys completed + ticket = FailTicket(banip, timeofban, matches) + ticket.setAttempt(failures) + ticket.setBanTime(bantime) + ticket.setBanCount(bancount) + matches = [] + failures = 0 + matches.extend(data['matches']) + failures += data['failures'] + ticket.setAttempt(failures) + tickets.append(ticket) + + if forbantime is None: + self._bansMergedCache[cacheKey] = tickets if ip is None else ticket + return tickets if ip is None else ticket + @commitandrollback def purge(self, cur): """Purge old bans, jails and log files from database. diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index a7c174ae..8e99e780 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -188,8 +188,8 @@ class Jail: Used by filter to add a failure for banning. """ self.__queue.put(ticket) - if self.database is not None: - self.database.addBan(self, ticket) + # add ban to database moved to actions (should previously check not already banned + # and increase ticket time if "bantimeextra.enabled" set) def getFailTicket(self): """Get a fail ticket from the jail. @@ -210,11 +210,23 @@ class Jail: self.filter.start() self.actions.start() # Restore any previous valid bans from the database - if self.database is not None: - for ticket in self.database.getBansMerged( - jail=self, bantime=self.actions.getBanTime()): - if not self.filter.inIgnoreIPList(ticket.getIP()): - self.__queue.put(ticket) + try: + if self.database is not None: + forbantime = None; + if self.actions.getBanTimeExtra('enabled'): + forbantime = self.actions.getBanTimeExtra('findtime') + if forbantime is None: + forbantime = self.actions.getBanTime() + for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime): + #logSys.debug('restored ticket: %s', ticket) + if not self.filter.inIgnoreIPList(ticket.getIP()): + # mark ticked was restored from database - does not put it again into db: + ticket.setRestored(True) + self.__queue.put(ticket) + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + logSys.info("Jail '%s' started" % self.name) def stop(self): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 1bf8dcbb..a10d5ab3 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -302,6 +302,12 @@ class Server: def getBanTime(self, name): return self.__jails[name].actions.getBanTime() + + def setBanTimeExtra(self, name, opt, value): + self.__jails[name].actions.setBanTimeExtra(opt, value) + + def getBanTimeExtra(self, name, opt): + return self.__jails[name].actions.getBanTimeExtra(opt) # Status def status(self): diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 8d036b2d..6c7c2f2c 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -40,14 +40,17 @@ class Ticket: """ self.setIP(ip) + self.__restored = False; + self.__banCount = 0; + self.__banTime = None; self.__time = time self.__attempt = 0 self.__file = None self.__matches = matches or [] def __str__(self): - return "%s: ip=%s time=%s #attempts=%d matches=%r" % \ - (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches) + return "%s: ip=%s time=%s bantime=%s bancount=%s #attempts=%d matches=%r" % \ + (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__banTime, self.__banCount, self.__attempt, self.__matches) def __repr__(self): return str(self) @@ -75,7 +78,19 @@ class Ticket: def getTime(self): return self.__time - + + def setBanTime(self, value): + self.__banTime = value; + + def getBanTime(self, defaultBT = None): + return (self.__banTime if not self.__banTime is None else defaultBT); + + def setBanCount(self, value): + self.__banCount = value; + + def getBanCount(self): + return self.__banCount; + def setAttempt(self, value): self.__attempt = value @@ -85,6 +100,12 @@ class Ticket: def getMatches(self): return self.__matches + def setRestored(self, value): + self.__restored = value + + def getRestored(self): + return self.__restored + class FailTicket(Ticket): pass diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2baf00a7..471cbea4 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -222,6 +222,11 @@ class Transmitter: value = command[2] self.__server.setBanTime(name, int(value)) return self.__server.getBanTime(name) + elif command[1].startswith("bantimeextra."): + value = command[2] + opt = command[1][len("bantimeextra."):] + self.__server.setBanTimeExtra(name, opt, value) + return self.__server.getBanTimeExtra(name, opt) elif command[1] == "banip": value = command[2] return self.__server.setBanIP(name,value) @@ -300,6 +305,9 @@ class Transmitter: # Action elif command[1] == "bantime": return self.__server.getBanTime(name) + elif command[1].startswith("bantimeextra."): + opt = command[1][len("bantimeextra."):] + return self.__server.getBanTimeExtra(name, opt) elif command[1] == "actions": return self.__server.getActions(name).keys() elif command[1] == "action":