introduced new feature "ban time exponential increasing":

"bantimeextra.enabled" in jail.conf allows to use database for searching of previously banned ip's to increase a default ban time using special formula,
   by default, each next ban it will be original banTime * 1, 2, 4, 8, 16, 32...
see "jail.conf" for some other options of "bantimeextra";
additional we can configure a little randomization of ban time, to prevent "clever" botnets calculate exact time IP can be unbanned.
WARNING: by first start the server upgrades sqlite database (table "bans" will recreated with another schema);
pull/716/head
sebres 2014-04-29 18:01:57 +02:00
parent 7cc64a14e0
commit 6f7c9b7d0f
9 changed files with 278 additions and 23 deletions

View File

@ -44,6 +44,30 @@ before = paths-debian.conf
# MISCELLANEOUS OPTIONS
#
# "bantimeextra.enabled" allows to use database for searching of previously banned ip's to increase a
# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32...
bantimeextra.enabled = true
# "bantimeextra.findtime" is the max number of seconds that we search in the database,
# if it is not specified - whole database will be used for ban searching
# (please observe current "dbpurgeage" value of fail2ban.conf).
bantimeextra.findtime = 24*60*60
# "bantimeextra.rndtime" is the max number of seconds using for mixing with random time
# to prevent "clever" botnets calculate exact time IP can be unbanned again:
bantimeextra.rndtime = 5*60
# "bantimeextra.maxtime" is the max number of seconds using the ban time can reach (don't grows further)
bantimeextra.maxtime = 24*60*60
# "bantimeextra.factor" is a coefficient to calculate exponent growing of the formula,
# by default value of factor "2.0 / 2.885385" and default value of formula, the ban time
# grows by 1, 2, 4, 8, 16 ...
#bantimeextra.factor = 2.0 / 2.885385
# "bantimeextra.formula" used to calculate next value of ban time;
#bantimeextra.formula = banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor)
# "bantimeextra.overalljails" (if true) specifies the search of IP in the database will be executed
# cross over all jails, if false (dafault), only current jail of the ban IP will be searched
#bantimeextra.overalljails = false
# --------------------
# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not
# ban a host which matches an address in this list. Several addresses can be
# defined using space separator.

View File

@ -93,6 +93,13 @@ class JailReader(ConfigReader):
["int", "maxretry", None],
["int", "findtime", None],
["int", "bantime", None],
["bool", "bantimeextra.enabled", False],
["string", "bantimeextra.findtime", None],
["string", "bantimeextra.factor", None],
["string", "bantimeextra.formula", None],
["string", "bantimeextra.maxtime", None],
["string", "bantimeextra.rndtime", None],
["bool", "bantimeextra.overalljails", None],
["string", "usedns", None],
["string", "failregex", None],
["string", "ignoreregex", None],
@ -198,6 +205,8 @@ class JailReader(ConfigReader):
stream.append(["set", self.__name, "findtime", self.__opts[opt]])
elif opt == "bantime":
stream.append(["set", self.__name, "bantime", self.__opts[opt]])
elif opt.startswith("bantimeextra."):
stream.append(["set", self.__name, opt, self.__opts[opt]])
elif opt == "usedns":
stream.append(["set", self.__name, "usedns", self.__opts[opt]])
elif opt == "failregex":

View File

@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import time, logging
import os
import os, datetime, math, json, random
import sys
if sys.version_info >= (3, 3):
import importlib.machinery
@ -83,6 +83,8 @@ class Actions(JailThread, Mapping):
self._actions = dict()
## The ban manager.
self.__banManager = BanManager()
## Extra parameters for increase ban time
self._banExtra = {'maxtime': 24*60*60};
def add(self, name, pythonModule=None, initOpts=None):
"""Adds a new action.
@ -240,6 +242,71 @@ class Actions(JailThread, Mapping):
logSys.debug(self._jail.name + ": action terminated")
return True
def setBanTimeExtra(self, opt, value):
# merge previous extra with new option:
be = self._banExtra;
be[opt] = value;
logSys.info('Set banTimeExtra.%s = %s', opt, value)
if opt == 'enabled':
be[opt] = bool(value)
if bool(value) and self._jail.database is None:
logSys.warning("banTimeExtra is not available as long jail database is not set")
if opt in ['findtime', 'maxtime', 'rndtime']:
if not value is None:
be[opt] = eval(value)
if opt == 'factor' or be.get('factor', None) is None:
be['factor'] = eval(be.get('factor', "2.0 / 2.885385"));
# prepare formula :
if opt in ['formula', 'maxtime', 'rndtime'] or be.get('evformula', None) is None:
be['formula'] = be.get('formula', 'banTime * math.exp(float(banCount)*banFactor)/math.exp(1*banFactor)')
evformula = be['formula'];
evformula = ('max(banTime, %s)' % evformula)
if not be.get('maxtime', None) is None:
evformula = ('min(%s, %s)' % (evformula, be['maxtime']))
# mix with random time (to prevent botnet calc exact time IP can be unbanned):
if not be.get('rndtime', None) is None:
evformula = ('(%s + random.random() * %s)' % (evformula, be['rndtime']))
# set to extra dict:
be['evformula'] = evformula
#logSys.info('banTimeExtra : %s' % json.dumps(be))
def getBanTimeExtra(self, opt):
return self._banExtra.get(opt, None)
def incrBanTime(self, bTicket, ip):
"""Check for IP address to increment ban time (if was already banned).
Returns
-------
float
new ban time.
"""
orgBanTime = self.__banManager.getBanTime()
banTime = orgBanTime
# check ip was already banned (increment time of ban):
try:
be = self._banExtra;
if banTime > 0 and be.get('enabled', False):
banFactor = be['factor'];
# search IP in database and increase time if found:
for banCount, timeOfBan, lastBanTime in \
self._jail.database.getBan(ip, self._jail, be.get('findtime', None), be.get('overalljails', False) \
):
#logSys.debug('IP %s was already banned: %s #, %s' % (ip, banCount, timeOfBan));
bTicket.setBanCount(banCount);
# calculate new ban time
banTime = eval(be['evformula'])
bTicket.setBanTime(banTime);
logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount,
datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"),
datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime))));
break
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
return banTime
def __checkBan(self):
"""Check for IP address to ban.
@ -255,25 +322,46 @@ class Actions(JailThread, Mapping):
if ticket != False:
aInfo = CallingMap()
bTicket = BanManager.createBanTicket(ticket)
aInfo["ip"] = bTicket.getIP()
if ticket.getBanTime() is not None:
bTicket.setBanTime(ticket.getBanTime())
bTicket.setBanCount(ticket.getBanCount())
ip = bTicket.getIP()
aInfo["ip"] = ip
aInfo["failures"] = bTicket.getAttempt()
aInfo["time"] = bTicket.getTime()
aInfo["matches"] = "\n".join(bTicket.getMatches())
btime = bTicket.getBanTime(self.__banManager.getBanTime());
if self._jail.database is not None:
aInfo["ipmatches"] = lambda: "\n".join(
self._jail.database.getBansMerged(
ip=bTicket.getIP()).getMatches())
ip=ip).getMatches())
aInfo["ipjailmatches"] = lambda: "\n".join(
self._jail.database.getBansMerged(
ip=bTicket.getIP(), jail=self._jail).getMatches())
ip=ip, jail=self._jail).getMatches())
aInfo["ipfailures"] = lambda: "\n".join(
self._jail.database.getBansMerged(
ip=bTicket.getIP()).getAttempt())
ip=ip).getAttempt())
aInfo["ipjailfailures"] = lambda: "\n".join(
self._jail.database.getBansMerged(
ip=bTicket.getIP(), jail=self._jail).getAttempt())
ip=ip, jail=self._jail).getAttempt())
try:
# if ban time was not set:
if bTicket.getBanTime() is None:
btime = self.incrBanTime(bTicket, ip)
bTicket.setBanTime(btime);
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
if self.__banManager.addBanTicket(bTicket):
logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"]))
if self._jail.database is not None:
# add to database always only after ban time was calculated an not yet already banned:
# if ticked was not restored from database - put it into database:
if not ticket.getRestored():
self._jail.database.addBan(self._jail, bTicket)
logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % (self._jail.name, ('Resore ' if ticket.getRestored() else ''),
aInfo["ip"], bTicket.getBanCount(), datetime.timedelta(seconds=int(btime)),
datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S")))
for name, action in self._actions.iteritems():
try:
action.ban(aInfo)

View File

@ -129,8 +129,11 @@ class BanManager:
#@staticmethod
def createBanTicket(ticket):
ip = ticket.getIP()
#lastTime = ticket.getTime()
lastTime = MyTime.time()
# if ticked was restored from database - set time of original restored ticket:
if ticket.getRestored():
lastTime = ticket.getTime()
else:
lastTime = MyTime.time()
banTicket = BanTicket(ip, lastTime, ticket.getMatches())
banTicket.setAttempt(ticket.getAttempt())
return banTicket
@ -197,7 +200,7 @@ class BanManager:
# Gets the list of ticket to remove.
unBanList = [ticket for ticket in self.__banList
if ticket.getTime() < time - self.__banTime]
if ticket.getTime() < time - ticket.getBanTime(self.__banTime)]
# Removes tickets.
self.__banList = [ticket for ticket in self.__banList

View File

@ -87,7 +87,7 @@ class Fail2BanDb(object):
filename
purgeage
"""
__version__ = 2
__version__ = 3
# Note all _TABLE_* strings must end in ';' for py26 compatibility
_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
_TABLE_jails = "CREATE TABLE jails(" \
@ -114,6 +114,8 @@ class Fail2BanDb(object):
"jail TEXT NOT NULL, " \
"ip TEXT, " \
"timeofban INTEGER NOT NULL, " \
"bantime INTEGER NOT NULL, " \
"bancount INTEGER NOT NULL, " \
"data JSON, " \
"FOREIGN KEY(jail) REFERENCES jails(name) " \
");" \
@ -121,6 +123,9 @@ class Fail2BanDb(object):
"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
"CREATE INDEX bans_ip ON bans(ip);" \
# todo: for performance reasons create a table with currently banned unique jails-ips only (with last ban of time and bantime).
# check possible view performance instead of new table;
def __init__(self, filename, purgeAge=24*60*60):
try:
self._lock = Lock()
@ -220,6 +225,16 @@ class Fail2BanDb(object):
"UPDATE fail2banDb SET version = 2;"
"COMMIT;" % Fail2BanDb._TABLE_logs)
if version < 3:
cur.executescript("BEGIN TRANSACTION;"
"CREATE TEMPORARY TABLE bans_temp AS SELECT jail, ip, timeofban, 600 as bantime, 1 as bancount, data FROM bans;"
"DROP TABLE bans;"
"%s;"
"INSERT INTO bans SELECT * from bans_temp;"
"DROP TABLE bans_temp;"
"UPDATE fail2banDb SET version = 3;"
"COMMIT;" % Fail2BanDb._TABLE_bans)
cur.execute("SELECT version FROM fail2banDb LIMIT 1")
return cur.fetchone()[0]
@ -367,8 +382,8 @@ class Fail2BanDb(object):
pass
#TODO: Implement data parts once arbitrary match keys completed
cur.execute(
"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
(jail.name, ticket.getIP(), ticket.getTime(),
"INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)",
(jail.name, ticket.getIP(), ticket.getTime(), ticket.getBanTime(), ticket.getBanCount() + 1,
{"matches": ticket.getMatches(),
"failures": ticket.getAttempt()}))
@ -472,6 +487,75 @@ class Fail2BanDb(object):
self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
return tickets if ip is None else ticket
def getBan(self, ip, jail=None, forbantime=None, overalljails=None):
#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
if overalljails is None or not overalljails:
query = "SELECT bancount, max(timeofban), bantime FROM bans"
else:
query = "SELECT max(bancount), max(timeofban), bantime FROM bans"
query += " WHERE ip = ?"
queryArgs = [ip]
if (overalljails is None or not overalljails) and jail is not None:
query += " AND jail=?"
queryArgs.append(jail.name)
if forbantime is not None:
query += " AND timeofban > ?"
queryArgs.append(MyTime.time() - forbantime)
query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1"
cur = self._db.cursor()
return cur.execute(query, queryArgs)
def _getCurrentBans(self, jail = None, ip = None, forbantime=None):
#query = "SELECT count(ip), max(timeofban) FROM bans WHERE ip = ?"
query = "SELECT ip, max(timeofban), bantime, bancount, data FROM bans WHERE 1"
queryArgs = []
if jail is not None:
query += " AND jail=?"
queryArgs.append(jail.name)
if ip is not None:
query += " AND ip=?"
queryArgs.append(ip)
query += " AND timeofban + bantime > ?"
queryArgs.append(MyTime.time())
if forbantime is not None:
query += " AND timeofban > ?"
queryArgs.append(MyTime.time() - forbantime)
query += " GROUP BY ip ORDER BY ip, timeofban DESC"
cur = self._db.cursor()
#logSys.debug((query, queryArgs));
return cur.execute(query, queryArgs)
def getCurrentBans(self, jail = None, ip = None, forbantime=None):
if forbantime is None:
cacheKey = (ip, jail)
if cacheKey in self._bansMergedCache:
return self._bansMergedCache[cacheKey]
tickets = []
ticket = None
results = list(self._getCurrentBans(jail=jail, ip=ip, forbantime=forbantime))
if results:
matches = []
failures = 0
for banip, timeofban, bantime, bancount, data in results:
#TODO: Implement data parts once arbitrary match keys completed
ticket = FailTicket(banip, timeofban, matches)
ticket.setAttempt(failures)
ticket.setBanTime(bantime)
ticket.setBanCount(bancount)
matches = []
failures = 0
matches.extend(data['matches'])
failures += data['failures']
ticket.setAttempt(failures)
tickets.append(ticket)
if forbantime is None:
self._bansMergedCache[cacheKey] = tickets if ip is None else ticket
return tickets if ip is None else ticket
@commitandrollback
def purge(self, cur):
"""Purge old bans, jails and log files from database.

View File

@ -188,8 +188,8 @@ class Jail:
Used by filter to add a failure for banning.
"""
self.__queue.put(ticket)
if self.database is not None:
self.database.addBan(self, ticket)
# add ban to database moved to actions (should previously check not already banned
# and increase ticket time if "bantimeextra.enabled" set)
def getFailTicket(self):
"""Get a fail ticket from the jail.
@ -210,11 +210,23 @@ class Jail:
self.filter.start()
self.actions.start()
# Restore any previous valid bans from the database
if self.database is not None:
for ticket in self.database.getBansMerged(
jail=self, bantime=self.actions.getBanTime()):
if not self.filter.inIgnoreIPList(ticket.getIP()):
self.__queue.put(ticket)
try:
if self.database is not None:
forbantime = None;
if self.actions.getBanTimeExtra('enabled'):
forbantime = self.actions.getBanTimeExtra('findtime')
if forbantime is None:
forbantime = self.actions.getBanTime()
for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime):
#logSys.debug('restored ticket: %s', ticket)
if not self.filter.inIgnoreIPList(ticket.getIP()):
# mark ticked was restored from database - does not put it again into db:
ticket.setRestored(True)
self.__queue.put(ticket)
except Exception as e:
logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
#logSys.error('%s', e, exc_info=True)
logSys.info("Jail '%s' started" % self.name)
def stop(self):

View File

@ -302,6 +302,12 @@ class Server:
def getBanTime(self, name):
return self.__jails[name].actions.getBanTime()
def setBanTimeExtra(self, name, opt, value):
self.__jails[name].actions.setBanTimeExtra(opt, value)
def getBanTimeExtra(self, name, opt):
return self.__jails[name].actions.getBanTimeExtra(opt)
# Status
def status(self):

View File

@ -40,14 +40,17 @@ class Ticket:
"""
self.setIP(ip)
self.__restored = False;
self.__banCount = 0;
self.__banTime = None;
self.__time = time
self.__attempt = 0
self.__file = None
self.__matches = matches or []
def __str__(self):
return "%s: ip=%s time=%s #attempts=%d matches=%r" % \
(self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches)
return "%s: ip=%s time=%s bantime=%s bancount=%s #attempts=%d matches=%r" % \
(self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__banTime, self.__banCount, self.__attempt, self.__matches)
def __repr__(self):
return str(self)
@ -75,7 +78,19 @@ class Ticket:
def getTime(self):
return self.__time
def setBanTime(self, value):
self.__banTime = value;
def getBanTime(self, defaultBT = None):
return (self.__banTime if not self.__banTime is None else defaultBT);
def setBanCount(self, value):
self.__banCount = value;
def getBanCount(self):
return self.__banCount;
def setAttempt(self, value):
self.__attempt = value
@ -85,6 +100,12 @@ class Ticket:
def getMatches(self):
return self.__matches
def setRestored(self, value):
self.__restored = value
def getRestored(self):
return self.__restored
class FailTicket(Ticket):
pass

View File

@ -222,6 +222,11 @@ class Transmitter:
value = command[2]
self.__server.setBanTime(name, int(value))
return self.__server.getBanTime(name)
elif command[1].startswith("bantimeextra."):
value = command[2]
opt = command[1][len("bantimeextra."):]
self.__server.setBanTimeExtra(name, opt, value)
return self.__server.getBanTimeExtra(name, opt)
elif command[1] == "banip":
value = command[2]
return self.__server.setBanIP(name,value)
@ -300,6 +305,9 @@ class Transmitter:
# Action
elif command[1] == "bantime":
return self.__server.getBanTime(name)
elif command[1].startswith("bantimeextra."):
opt = command[1][len("bantimeextra."):]
return self.__server.getBanTimeExtra(name, opt)
elif command[1] == "actions":
return self.__server.getActions(name).keys()
elif command[1] == "action":