introduced new feature "ban time exponential increasing":

"bantimeextra.enabled" in jail.conf allows to use database for searching of previously banned ip's to increase a default ban time using special formula,
   by default, each next ban it will be original banTime * 1, 2, 4, 8, 16, 32...
see "jail.conf" for some other options of "bantimeextra";
additional we can configure a little randomization of ban time, to prevent "clever" botnets calculate exact time IP can be unbanned.
WARNING: by first start the server upgrades sqlite database (table "bans" will recreated with another schema);
pull/716/head
sebres 2014-04-29 18:01:57 +02:00
parent 7cc64a14e0
commit 6f7c9b7d0f
9 changed files with 278 additions and 23 deletions

View File

@ -44,6 +44,30 @@ before = paths-debian.conf
# MISCELLANEOUS OPTIONS # MISCELLANEOUS OPTIONS
# #
# "bantimeextra.enabled" allows to use database for searching of previously banned ip's to increase a
# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32...
bantimeextra.enabled = true
# "bantimeextra.findtime" is the max number of seconds that we search in the database,
# if it is not specified - whole database will be used for ban searching
# (please observe current "dbpurgeage" value of fail2ban.conf).
bantimeextra.findtime = 24*60*60
# "bantimeextra.rndtime" is the max number of seconds using for mixing with random time
# to prevent "clever" botnets calculate exact time IP can be unbanned again:
bantimeextra.rndtime = 5*60
# "bantimeextra.maxtime" is the max number of seconds using the ban time can reach (don't grows further)
bantimeextra.maxtime = 24*60*60
# "bantimeextra.factor" is a coefficient to calculate exponent growing of the formula,
# by default value of factor "2.0 / 2.885385" and default value of formula, the ban time
# grows by 1, 2, 4, 8, 16 ...
#bantimeextra.factor = 2.0 / 2.885385
# "bantimeextra.formula" used to calculate next value of ban time;
#bantimeextra.formula = banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor)
# "bantimeextra.overalljails" (if true) specifies the search of IP in the database will be executed
# cross over all jails, if false (dafault), only current jail of the ban IP will be searched
#bantimeextra.overalljails = false
# --------------------
# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
# ban a host which matches an address in this list. Several addresses can be # ban a host which matches an address in this list. Several addresses can be
# defined using space separator. # defined using space separator.

View File

@ -93,6 +93,13 @@ class JailReader(ConfigReader):
["int", "maxretry", None], ["int", "maxretry", None],
["int", "findtime", None], ["int", "findtime", None],
["int", "bantime", None], ["int", "bantime", None],
["bool", "bantimeextra.enabled", False],
["string", "bantimeextra.findtime", None],
["string", "bantimeextra.factor", None],
["string", "bantimeextra.formula", None],
["string", "bantimeextra.maxtime", None],
["string", "bantimeextra.rndtime", None],
["bool", "bantimeextra.overalljails", None],
["string", "usedns", None], ["string", "usedns", None],
["string", "failregex", None], ["string", "failregex", None],
["string", "ignoreregex", None], ["string", "ignoreregex", None],
@ -198,6 +205,8 @@ class JailReader(ConfigReader):
stream.append(["set", self.__name, "findtime", self.__opts[opt]]) stream.append(["set", self.__name, "findtime", self.__opts[opt]])
elif opt == "bantime": elif opt == "bantime":
stream.append(["set", self.__name, "bantime", self.__opts[opt]]) stream.append(["set", self.__name, "bantime", self.__opts[opt]])
elif opt.startswith("bantimeextra."):
stream.append(["set", self.__name, opt, self.__opts[opt]])
elif opt == "usedns": elif opt == "usedns":
stream.append(["set", self.__name, "usedns", self.__opts[opt]]) stream.append(["set", self.__name, "usedns", self.__opts[opt]])
elif opt == "failregex": elif opt == "failregex":

View File

@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL" __license__ = "GPL"
import time, logging import time, logging
import os import os, datetime, math, json, random
import sys import sys
if sys.version_info >= (3, 3): if sys.version_info >= (3, 3):
import importlib.machinery import importlib.machinery
@ -83,6 +83,8 @@ class Actions(JailThread, Mapping):
self._actions = dict() self._actions = dict()
## The ban manager. ## The ban manager.
self.__banManager = BanManager() self.__banManager = BanManager()
## Extra parameters for increase ban time
self._banExtra = {'maxtime': 24*60*60};
def add(self, name, pythonModule=None, initOpts=None): def add(self, name, pythonModule=None, initOpts=None):
"""Adds a new action. """Adds a new action.
@ -240,6 +242,71 @@ class Actions(JailThread, Mapping):
logSys.debug(self._jail.name + ": action terminated") logSys.debug(self._jail.name + ": action terminated")
return True return True
def setBanTimeExtra(self, opt, value):
# merge previous extra with new option:
be = self._banExtra;
be[opt] = value;
logSys.info('Set banTimeExtra.%s = %s', opt, value)
if opt == 'enabled':
be[opt] = bool(value)
if bool(value) and self._jail.database is None:
logSys.warning("banTimeExtra is not available as long jail database is not set")
if opt in ['findtime', 'maxtime', 'rndtime']:
if not value is None:
be[opt] = eval(value)
if opt == 'factor' or be.get('factor', None) is None:
be['factor'] = eval(be.get('factor', "2.0 / 2.885385"));
# prepare formula :
if opt in ['formula', 'maxtime', 'rndtime'] or be.get('evformula', None) is None:
be['formula'] = be.get('formula', 'banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor)')
evformula = be['formula'];
evformula = ('max(banTime, %s)' % evformula)
if not be.get('maxtime', None) is None:
evformula = ('min(%s, %s)' % (evformula, be['maxtime']))
# mix with random time (to prevent botnet calc exact time IP can be unbanned):
if not be.get('rndtime', None) is None:
evformula = ('(%s + random.random() * %s)' % (evformula, be['rndtime']))
# set to extra dict:
be['evformula'] = evformula
#logSys.info('banTimeExtra : %s' % json.dumps(be))
def getBanTimeExtra(self, opt):
return self._banExtra.get(opt, None)
def incrBanTime(self, bTicket, ip):
"""Check for IP address to increment ban time (if was already banned).
Returns
-------
float
new ban time.
"""
orgBanTime = self.__banManager.getBanTime()
banTime = orgBanTime
# check ip was already banned (increment time of ban):
try:
be = self._banExtra;
if banTime > 0 and be.get('enabled', False):
banFactor = be['factor'];
# search IP in database and increase time if found:
for banCount, timeOfBan, lastBanTime in \
self._jail.database.getBan(ip, self._jail, be.get('findtime', None), be.get('overalljails', False) \
):
#logSys.debug('IP %s was already banned: %s #, %s' % (ip, banCount, timeOfBan));
bTicket.setBanCount(banCount);
# calculate new ban time
banTime = eval(be['evformula'])
bTicket.setBanTime(banTime);
logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount,
datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"),
datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime))));
break
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
return banTime
def __checkBan(self): def __checkBan(self):
"""Check for IP address to ban. """Check for IP address to ban.
@ -255,25 +322,46 @@ class Actions(JailThread, Mapping):
if ticket != False: if ticket != False:
aInfo = CallingMap() aInfo = CallingMap()
bTicket = BanManager.createBanTicket(ticket) bTicket = BanManager.createBanTicket(ticket)
aInfo["ip"] = bTicket.getIP() if ticket.getBanTime() is not None:
bTicket.setBanTime(ticket.getBanTime())
bTicket.setBanCount(ticket.getBanCount())
ip = bTicket.getIP()
aInfo["ip"] = ip
aInfo["failures"] = bTicket.getAttempt() aInfo["failures"] = bTicket.getAttempt()
aInfo["time"] = bTicket.getTime() aInfo["time"] = bTicket.getTime()
aInfo["matches"] = "\n".join(bTicket.getMatches()) aInfo["matches"] = "\n".join(bTicket.getMatches())
btime = bTicket.getBanTime(self.__banManager.getBanTime());
if self._jail.database is not None: if self._jail.database is not None:
aInfo["ipmatches"] = lambda: "\n".join( aInfo["ipmatches"] = lambda: "\n".join(
self._jail.database.getBansMerged( self._jail.database.getBansMerged(
ip=bTicket.getIP()).getMatches()) ip=ip).getMatches())
aInfo["ipjailmatches"] = lambda: "\n".join( aInfo["ipjailmatches"] = lambda: "\n".join(
self._jail.database.getBansMerged( self._jail.database.getBansMerged(
ip=bTicket.getIP(), jail=self._jail).getMatches()) ip=ip, jail=self._jail).getMatches())
aInfo["ipfailures"] = lambda: "\n".join( aInfo["ipfailures"] = lambda: "\n".join(
self._jail.database.getBansMerged( self._jail.database.getBansMerged(
ip=bTicket.getIP()).getAttempt()) ip=ip).getAttempt())
aInfo["ipjailfailures"] = lambda: "\n".join( aInfo["ipjailfailures"] = lambda: "\n".join(
self._jail.database.getBansMerged( self._jail.database.getBansMerged(
ip=bTicket.getIP(), jail=self._jail).getAttempt()) ip=ip, jail=self._jail).getAttempt())
try:
# if ban time was not set:
if bTicket.getBanTime() is None:
btime = self.incrBanTime(bTicket, ip)
bTicket.setBanTime(btime);
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
if self.__banManager.addBanTicket(bTicket): if self.__banManager.addBanTicket(bTicket):
logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) if self._jail.database is not None:
# add to database always only after ban time was calculated an not yet already banned:
# if ticked was not restored from database - put it into database:
if not ticket.getRestored():
self._jail.database.addBan(self._jail, bTicket)
logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % (self._jail.name, ('Resore ' if ticket.getRestored() else ''),
aInfo["ip"], bTicket.getBanCount(), datetime.timedelta(seconds=int(btime)),
datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S")))
for name, action in self._actions.iteritems(): for name, action in self._actions.iteritems():
try: try:
action.ban(aInfo) action.ban(aInfo)

View File

@ -129,8 +129,11 @@ class BanManager:
#@staticmethod #@staticmethod
def createBanTicket(ticket): def createBanTicket(ticket):
ip = ticket.getIP() ip = ticket.getIP()
#lastTime = ticket.getTime() # if ticked was restored from database - set time of original restored ticket:
lastTime = MyTime.time() if ticket.getRestored():
lastTime = ticket.getTime()
else:
lastTime = MyTime.time()
banTicket = BanTicket(ip, lastTime, ticket.getMatches()) banTicket = BanTicket(ip, lastTime, ticket.getMatches())
banTicket.setAttempt(ticket.getAttempt()) banTicket.setAttempt(ticket.getAttempt())
return banTicket return banTicket
@ -197,7 +200,7 @@ class BanManager:
# Gets the list of ticket to remove. # Gets the list of ticket to remove.
unBanList = [ticket for ticket in self.__banList unBanList = [ticket for ticket in self.__banList
if ticket.getTime() < time - self.__banTime] if ticket.getTime() < time - ticket.getBanTime(self.__banTime)]
# Removes tickets. # Removes tickets.
self.__banList = [ticket for ticket in self.__banList self.__banList = [ticket for ticket in self.__banList

View File

@ -87,7 +87,7 @@ class Fail2BanDb(object):
filename filename
purgeage purgeage
""" """
__version__ = 2 __version__ = 3
# Note all _TABLE_* strings must end in ';' for py26 compatibility # Note all _TABLE_* strings must end in ';' for py26 compatibility
_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);" _TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
_TABLE_jails = "CREATE TABLE jails(" \ _TABLE_jails = "CREATE TABLE jails(" \
@ -114,6 +114,8 @@ class Fail2BanDb(object):
"jail TEXT NOT NULL, " \ "jail TEXT NOT NULL, " \
"ip TEXT, " \ "ip TEXT, " \
"timeofban INTEGER NOT NULL, " \ "timeofban INTEGER NOT NULL, " \
"bantime INTEGER NOT NULL, " \
"bancount INTEGER NOT NULL, " \
"data JSON, " \ "data JSON, " \
"FOREIGN KEY(jail) REFERENCES jails(name) " \ "FOREIGN KEY(jail) REFERENCES jails(name) " \
");" \ ");" \
@ -121,6 +123,9 @@ class Fail2BanDb(object):
"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \ "CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
"CREATE INDEX bans_ip ON bans(ip);" \ "CREATE INDEX bans_ip ON bans(ip);" \
# todo: for performance reasons create a table with currently banned unique jails-ips only (with last ban of time and bantime).
# check possible view performance instead of new table;
def __init__(self, filename, purgeAge=24*60*60): def __init__(self, filename, purgeAge=24*60*60):
try: try:
self._lock = Lock() self._lock = Lock()
@ -220,6 +225,16 @@ class Fail2BanDb(object):
"UPDATE fail2banDb SET version = 2;" "UPDATE fail2banDb SET version = 2;"
"COMMIT;" % Fail2BanDb._TABLE_logs) "COMMIT;" % Fail2BanDb._TABLE_logs)
if version < 3:
cur.executescript("BEGIN TRANSACTION;"
"CREATE TEMPORARY TABLE bans_temp AS SELECT jail, ip, timeofban, 600 as bantime, 1 as bancount, data FROM bans;"
"DROP TABLE bans;"
"%s;"
"INSERT INTO bans SELECT * from bans_temp;"
"DROP TABLE bans_temp;"
"UPDATE fail2banDb SET version = 3;"
"COMMIT;" % Fail2BanDb._TABLE_bans)
cur.execute("SELECT version FROM fail2banDb LIMIT 1") cur.execute("SELECT version FROM fail2banDb LIMIT 1")
return cur.fetchone()[0] return cur.fetchone()[0]
@ -367,8 +382,8 @@ class Fail2BanDb(object):
pass pass
#TODO: Implement data parts once arbitrary match keys completed #TODO: Implement data parts once arbitrary match keys completed
cur.execute( cur.execute(
"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", "INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
(jail.name, ticket.getIP(), ticket.getTime(), (jail.name, ticket.getIP(), ticket.getTime(), ticket.getBanTime(), ticket.getBanCount() + 1,
{"matches": ticket.getMatches(), {"matches": ticket.getMatches(),
"failures": ticket.getAttempt()})) "failures": ticket.getAttempt()}))
@ -472,6 +487,75 @@ class Fail2BanDb(object):
self._bansMergedCache[cacheKey] = tickets if ip is None else ticket self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
return tickets if ip is None else ticket return tickets if ip is None else ticket
def getBan(self, ip, jail=None, forbantime=None, overalljails=None):
#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
if overalljails is None or not overalljails:
query = "SELECT bancount, max(timeofban), bantime FROM bans"
else:
query = "SELECT max(bancount), max(timeofban), bantime FROM bans"
query += " WHERE ip = ?"
queryArgs = [ip]
if (overalljails is None or not overalljails) and jail is not None:
query += " AND jail=?"
queryArgs.append(jail.name)
if forbantime is not None:
query += " AND timeofban > ?"
queryArgs.append(MyTime.time() - forbantime)
query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1"
cur = self._db.cursor()
return cur.execute(query, queryArgs)
def _getCurrentBans(self, jail = None, ip = None, forbantime=None):
#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bans WHERE 1"
queryArgs = []
if jail is not None:
query += " AND jail=?"
queryArgs.append(jail.name)
if ip is not None:
query += " AND ip=?"
queryArgs.append(ip)
query += " AND timeofban + bantime > ?"
queryArgs.append(MyTime.time())
if forbantime is not None:
query += " AND timeofban > ?"
queryArgs.append(MyTime.time() - forbantime)
query += " GROUP BY ip ORDER BY ip, timeofban DESC"
cur = self._db.cursor()
#logSys.debug((query, queryArgs));
return cur.execute(query, queryArgs)
def getCurrentBans(self, jail = None, ip = None, forbantime=None):
if forbantime is None:
cacheKey = (ip, jail)
if cacheKey in self._bansMergedCache:
return self._bansMergedCache[cacheKey]
tickets = []
ticket = None
results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime))
if results:
matches = []
failures = 0
for banip, timeofban, bantime, bancount, data in results:
#TODO: Implement data parts once arbitrary match keys completed
ticket = FailTicket(banip, timeofban, matches)
ticket.setAttempt(failures)
ticket.setBanTime(bantime)
ticket.setBanCount(bancount)
matches = []
failures = 0
matches.extend(data['matches'])
failures += data['failures']
ticket.setAttempt(failures)
tickets.append(ticket)
if forbantime is None:
self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
return tickets if ip is None else ticket
@commitandrollback @commitandrollback
def purge(self, cur): def purge(self, cur):
"""Purge old bans, jails and log files from database. """Purge old bans, jails and log files from database.

View File

@ -188,8 +188,8 @@ class Jail:
Used by filter to add a failure for banning. Used by filter to add a failure for banning.
""" """
self.__queue.put(ticket) self.__queue.put(ticket)
if self.database is not None: # add ban to database moved to actions (should previously check not already banned
self.database.addBan(self, ticket) # and increase ticket time if "bantimeextra.enabled" set)
def getFailTicket(self): def getFailTicket(self):
"""Get a fail ticket from the jail. """Get a fail ticket from the jail.
@ -210,11 +210,23 @@ class Jail:
self.filter.start() self.filter.start()
self.actions.start() self.actions.start()
# Restore any previous valid bans from the database # Restore any previous valid bans from the database
if self.database is not None: try:
for ticket in self.database.getBansMerged( if self.database is not None:
jail=self, bantime=self.actions.getBanTime()): forbantime = None;
if not self.filter.inIgnoreIPList(ticket.getIP()): if self.actions.getBanTimeExtra('enabled'):
self.__queue.put(ticket) forbantime = self.actions.getBanTimeExtra('findtime')
if forbantime is None:
forbantime = self.actions.getBanTime()
for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime):
#logSys.debug('restored ticket: %s', ticket)
if not self.filter.inIgnoreIPList(ticket.getIP()):
# mark ticked was restored from database - does not put it again into db:
ticket.setRestored(True)
self.__queue.put(ticket)
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
logSys.info("Jail '%s' started" % self.name) logSys.info("Jail '%s' started" % self.name)
def stop(self): def stop(self):

View File

@ -302,6 +302,12 @@ class Server:
def getBanTime(self, name): def getBanTime(self, name):
return self.__jails[name].actions.getBanTime() return self.__jails[name].actions.getBanTime()
def setBanTimeExtra(self, name, opt, value):
self.__jails[name].actions.setBanTimeExtra(opt, value)
def getBanTimeExtra(self, name, opt):
return self.__jails[name].actions.getBanTimeExtra(opt)
# Status # Status
def status(self): def status(self):

View File

@ -40,14 +40,17 @@ class Ticket:
""" """
self.setIP(ip) self.setIP(ip)
self.__restored = False;
self.__banCount = 0;
self.__banTime = None;
self.__time = time self.__time = time
self.__attempt = 0 self.__attempt = 0
self.__file = None self.__file = None
self.__matches = matches or [] self.__matches = matches or []
def __str__(self): def __str__(self):
return "%s: ip=%s time=%s #attempts=%d matches=%r" % \ return "%s: ip=%s time=%s bantime=%s bancount=%s #attempts=%d matches=%r" % \
(self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches) (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__banTime, self.__banCount, self.__attempt, self.__matches)
def __repr__(self): def __repr__(self):
return str(self) return str(self)
@ -75,7 +78,19 @@ class Ticket:
def getTime(self): def getTime(self):
return self.__time return self.__time
def setBanTime(self, value):
self.__banTime = value;
def getBanTime(self, defaultBT = None):
return (self.__banTime if not self.__banTime is None else defaultBT);
def setBanCount(self, value):
self.__banCount = value;
def getBanCount(self):
return self.__banCount;
def setAttempt(self, value): def setAttempt(self, value):
self.__attempt = value self.__attempt = value
@ -85,6 +100,12 @@ class Ticket:
def getMatches(self): def getMatches(self):
return self.__matches return self.__matches
def setRestored(self, value):
self.__restored = value
def getRestored(self):
return self.__restored
class FailTicket(Ticket): class FailTicket(Ticket):
pass pass

View File

@ -222,6 +222,11 @@ class Transmitter:
value = command[2] value = command[2]
self.__server.setBanTime(name, int(value)) self.__server.setBanTime(name, int(value))
return self.__server.getBanTime(name) return self.__server.getBanTime(name)
elif command[1].startswith("bantimeextra."):
value = command[2]
opt = command[1][len("bantimeextra."):]
self.__server.setBanTimeExtra(name, opt, value)
return self.__server.getBanTimeExtra(name, opt)
elif command[1] == "banip": elif command[1] == "banip":
value = command[2] value = command[2]
return self.__server.setBanIP(name,value) return self.__server.setBanIP(name,value)
@ -300,6 +305,9 @@ class Transmitter:
# Action # Action
elif command[1] == "bantime": elif command[1] == "bantime":
return self.__server.getBanTime(name) return self.__server.getBanTime(name)
elif command[1].startswith("bantimeextra."):
opt = command[1][len("bantimeextra."):]
return self.__server.getBanTimeExtra(name, opt)
elif command[1] == "actions": elif command[1] == "actions":
return self.__server.getActions(name).keys() return self.__server.getActions(name).keys()
elif command[1] == "action": elif command[1] == "action":