diff --git a/config/jail.conf b/config/jail.conf index 275d72c6..0862b800 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -44,46 +44,41 @@ before = paths-debian.conf # MISCELLANEOUS OPTIONS # -# "bantimeextra.enabled" allows to use database for searching of previously banned ip's to increase a +# "bantime.increment" allows to use database for searching of previously banned ip's to increase a # default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32... -#bantimeextra.enabled = true +#bantime.increment = true -# "bantimeextra.findtime" is the max number of seconds that we search in the database, -# if it is not specified - whole database will be used for ban searching -# (please observe current "dbpurgeage" value of fail2ban.conf). -#bantimeextra.findtime = 24*60*60 - -# "bantimeextra.rndtime" is the max number of seconds using for mixing with random time +# "bantime.rndtime" is the max number of seconds using for mixing with random time # to prevent "clever" botnets calculate exact time IP can be unbanned again: -#bantimeextra.rndtime = 5*60 +#bantime.rndtime = 5*60 -# "bantimeextra.maxtime" is the max number of seconds using the ban time can reach (don't grows further) -#bantimeextra.maxtime = 24*60*60 +# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +#bantime.maxtime = -# "bantimeextra.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, +# "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, # default value of factor is 1 and with default value of formula, the ban time # grows by 1, 2, 4, 8, 16 ... -#bantimeextra.factor = 1 +#bantime.factor = 1 -# "bantimeextra.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value bellow, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... -#bantimeextra.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor +#bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # # more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : -#bantimeextra.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) +#bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) -# "bantimeextra.multipliers" used to calculate next value of ban time instead of formula, coresponding -# previously ban count and given "bantimeextra.factor" (for multipliers default is 1); +# "bantime.multipliers" used to calculate next value of ban time instead of formula, coresponding +# previously ban count and given "bantime.factor" (for multipliers default is 1); # following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, # always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours -#bantimeextra.multipliers = 1 2 4 8 16 32 64 +#bantime.multipliers = 1 2 4 8 16 32 64 # following example can be used for small initial ban time (bantime=60) - it grows more aggressive at begin, # for bantime=60 the multipliers are minutes and equal: 1 min, 5 min, 30 min, 1 hour, 5 hour, 12 hour, 1 day, 2 day -#bantimeextra.multipliers = 1 5 30 60 300 720 1440 2880 +#bantime.multipliers = 1 5 30 60 300 720 1440 2880 -# "bantimeextra.overalljails" (if true) specifies the search of IP in the database will be executed +# "bantime.overalljails" (if true) specifies the search of IP in the database will be executed # cross over all jails, if false (dafault), only current jail of the ban IP will be searched -#bantimeextra.overalljails = false +#bantime.overalljails = false # -------------------- diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 06f70277..1d368c62 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -90,17 +90,17 @@ class JailReader(ConfigReader): ["string", "logpath", None], ["string", "logencoding", None], ["string", "backend", "auto"], - ["int", "maxretry", None], - ["int", "findtime", None], - ["int", "bantime", None], - ["bool", "bantimeextra.enabled", None], - ["string", "bantimeextra.findtime", None], - ["string", "bantimeextra.factor", None], - ["string", "bantimeextra.formula", None], - ["string", "bantimeextra.multipliers", None], - ["string", "bantimeextra.maxtime", None], - ["string", "bantimeextra.rndtime", None], - ["bool", "bantimeextra.overalljails", None], + ["int", "maxretry", None], + ["string", "findtime", None], + ["string", "bantime", None], + ["bool", "bantime.increment", None], + ["string", "bantime.findtime", None], + ["string", "bantime.factor", None], + ["string", "bantime.formula", None], + ["string", "bantime.multipliers", None], + ["string", "bantime.maxtime", None], + ["string", "bantime.rndtime", None], + ["bool", "bantime.overalljails", None], ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], @@ -206,7 +206,7 @@ class JailReader(ConfigReader): stream.append(["set", self.__name, "findtime", self.__opts[opt]]) elif opt == "bantime": stream.append(["set", self.__name, "bantime", self.__opts[opt]]) - elif opt.startswith("bantimeextra."): + elif opt.startswith("bantime."): stream.append(["set", self.__name, opt, self.__opts[opt]]) elif opt == "usedns": stream.append(["set", self.__name, "usedns", self.__opts[opt]]) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 9fa2eb4b..47482440 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import time, logging -import os, datetime, math, json, random +import os, datetime import sys if sys.version_info >= (3, 3): import importlib.machinery @@ -38,6 +38,7 @@ except ImportError: OrderedDict = None from .banmanager import BanManager +from .observer import Observers from .jailthread import JailThread from .action import ActionBase, CommandAction, CallingMap from .mytime import MyTime @@ -83,8 +84,6 @@ class Actions(JailThread, Mapping): self._actions = dict() ## The ban manager. self.__banManager = BanManager() - ## Extra parameters for increase ban time - self._banExtra = {'maxtime': 24*60*60}; def add(self, name, pythonModule=None, initOpts=None): """Adds a new action. @@ -166,6 +165,7 @@ class Actions(JailThread, Mapping): # @param value the time def setBanTime(self, value): + value = MyTime.str2seconds(value) self.__banManager.setBanTime(value) logSys.info("Set banTime = %s" % value) @@ -242,102 +242,6 @@ class Actions(JailThread, Mapping): logSys.debug(self._jail.name + ": action terminated") return True - class BanTimeIncr: - def __init__(self, banTime, banCount): - self.Time = banTime - self.Count = banCount - - def setBanTimeExtra(self, opt, value): - # merge previous extra with new option: - be = self._banExtra; - if value == '': - value = None - if value is not None: - be[opt] = value; - elif opt in be: - del be[opt] - logSys.info('Set banTimeExtra.%s = %s', opt, value) - if opt == 'enabled': - if isinstance(value, str): - be[opt] = value.lower() in ("yes", "true", "ok", "1") - if be[opt] and self._jail.database is None: - logSys.warning("banTimeExtra is not available as long jail database is not set") - if opt in ['findtime', 'maxtime', 'rndtime']: - if not value is None: - be[opt] = MyTime.str2seconds(value) - # prepare formula lambda: - if opt in ['formula', 'factor', 'maxtime', 'rndtime', 'multipliers'] or be.get('evformula', None) is None: - # split multifiers to an array begins with 0 (or empty if not set): - if opt == 'multipliers': - be['evmultipliers'] = [int(i) for i in (value.split(' ') if value is not None and value != '' else [])] - # if we have multifiers - use it in lambda, otherwise compile and use formula within lambda - multipliers = be.get('evmultipliers', []) - banFactor = eval(be.get('factor', "1")) - if len(multipliers): - evformula = lambda ban, banFactor=banFactor: ( - ban.Time * banFactor * multipliers[ban.Count if ban.Count < len(multipliers) else -1] - ) - else: - formula = be.get('formula', 'ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor') - formula = compile(formula, '~inline-conf-expr~', 'eval') - evformula = lambda ban, banFactor=banFactor, formula=formula: max(ban.Time, eval(formula)) - # extend lambda with max time : - if not be.get('maxtime', None) is None: - maxtime = be['maxtime'] - evformula = lambda ban, evformula=evformula: min(evformula(ban), maxtime) - # mix lambda with random time (to prevent bot-nets to calculate exact time IP can be unbanned): - if not be.get('rndtime', None) is None: - rndtime = be['rndtime'] - evformula = lambda ban, evformula=evformula: (evformula(ban) + random.random() * rndtime) - # set to extra dict: - be['evformula'] = evformula - #logSys.info('banTimeExtra : %s' % json.dumps(be)) - - def getBanTimeExtra(self, opt): - return self._banExtra.get(opt, None) - - def calcBanTime(self, banTime, banCount): - return self._banExtra['evformula'](self.BanTimeIncr(banTime, banCount)) - - def incrBanTime(self, bTicket): - """Check for IP address to increment ban time (if was already banned). - - Returns - ------- - float - new ban time. - """ - ip = bTicket.getIP() - orgBanTime = self.__banManager.getBanTime() - banTime = orgBanTime - # check ip was already banned (increment time of ban): - try: - be = self._banExtra; - if banTime > 0 and be.get('enabled', False): - # search IP in database and increase time if found: - for banCount, timeOfBan, lastBanTime in \ - self._jail.database.getBan(ip, self._jail, be.get('findtime', None), be.get('overalljails', False) \ - ): - #logSys.debug('IP %s was already banned: %s #, %s' % (ip, banCount, timeOfBan)); - bTicket.setBanCount(banCount); - # calculate new ban time - if banCount > 0: - banTime = be['evformula'](self.BanTimeIncr(banTime, banCount)) - bTicket.setBanTime(banTime); - # check current ticket time to prevent increasing for twice read tickets (restored from log file besides database after restart) - if bTicket.getTime() > timeOfBan: - logSys.info('[%s] %s was already banned: %s # at last %s - increase time %s to %s' % (self._jail.name, ip, banCount, - datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), - datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); - else: - bTicket.setRestored(True) - break - except Exception as e: - logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - #logSys.error('%s', e, exc_info=True) - - return banTime - def __checkBan(self): """Check for IP address to ban. @@ -356,12 +260,15 @@ class Actions(JailThread, Mapping): if ticket.getBanTime() is not None: bTicket.setBanTime(ticket.getBanTime()) bTicket.setBanCount(ticket.getBanCount()) + if ticket.getRestored(): + bTicket.setRestored(True) ip = bTicket.getIP() aInfo["ip"] = ip aInfo["failures"] = bTicket.getAttempt() aInfo["time"] = bTicket.getTime() aInfo["matches"] = "\n".join(bTicket.getMatches()) btime = bTicket.getBanTime(self.__banManager.getBanTime()) + # [todo] move merging to observer - here we could read already merged info from database (faster); if self._jail.database is not None: aInfo["ipmatches"] = lambda jail=self._jail: "\n".join( jail.database.getBansMerged(ip=ip).getMatches() @@ -373,30 +280,25 @@ class Actions(JailThread, Mapping): jail.database.getBansMerged(ip=ip).getAttempt() aInfo["ipjailfailures"] = lambda jail=self._jail: \ jail.database.getBansMerged(ip=ip, jail=jail).getAttempt() - try: - # if not permanent, not restored and ban time was not set: - if btime != -1 and not ticket.getRestored() and bTicket.getBanTime() is None: - btime = self.incrBanTime(bTicket) - bTicket.setBanTime(btime) - if bTicket.getRestored(): - ticket.setRestored(True) - except Exception as e: - logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - #logSys.error('%s', e, exc_info=True) if btime != -1: + bendtime = aInfo["time"] + btime logtime = (datetime.timedelta(seconds=int(btime)), - datetime.datetime.fromtimestamp(aInfo["time"] + btime).strftime("%Y-%m-%d %H:%M:%S")) + datetime.datetime.fromtimestamp(bendtime).strftime("%Y-%m-%d %H:%M:%S")) + # check ban is not too old : + if bendtime < MyTime.time(): + logSys.info('[%s] Ignore %s, expiered bantime - %s', self._jail.name, ip, logtime[1]) + return False else: logtime = ('permanent', 'infinite') + if self.__banManager.addBanTicket(bTicket): - if self._jail.database is not None: - # add to database always only after ban time was calculated an not yet already banned: - # if ticked was not restored from database - put it into database: - if not ticket.getRestored(): - self._jail.database.addBan(self._jail, bTicket) - logSys.notice("[%s] %sBan %s (%d # %s -> %s)" % ((self._jail.name, ('Resore ' if ticket.getRestored() else ''), - aInfo["ip"], bTicket.getBanCount()+1) + logtime)) + # report ticket to observer, to check time should be increased and hereafter observer writes ban to database (asynchronous) + if not bTicket.getRestored(): + Observers.Main.add('banFound', bTicket, self._jail, btime) + logSys.notice("[%s] %sBan %s (%d # %s -> %s)", self._jail.name, ('' if not bTicket.getRestored() else 'Restore '), + aInfo["ip"], bTicket.getBanCount()+(1 if not bTicket.getRestored() else 0), *logtime) + # do actions : for name, action in self._actions.iteritems(): try: action.ban(aInfo) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index e4a18f8c..fc7a9f25 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -84,7 +84,7 @@ class FailManager: finally: self.__lock.release() - def addFailure(self, ticket, count=1): + def addFailure(self, ticket, count=1, observed = False): try: self.__lock.acquire() ip = ticket.getIP() @@ -98,6 +98,9 @@ class FailManager: fData.inc(matches, count) fData.setLastTime(unixTime) else: + ## not found - already banned - prevent to add failure if comes from observer: + if observed: + return fData = FailData() fData.inc(matches, count) fData.setLastReset(unixTime) @@ -138,13 +141,13 @@ class FailManager: if self.__failList.has_key(ip): del self.__failList[ip] - def toBan(self): + def toBan(self, ip = None): try: self.__lock.acquire() - for ip in self.__failList: + for ip in ([ip] if ip != None and ip in self.__failList else self.__failList): data = self.__failList[ip] if data.getRetry() >= self.__maxRetry: - self.__delFailure(ip) + del self.__failList[ip] # Create a FailTicket from BanData failTicket = FailTicket(ip, data.getLastTime(), data.getMatches()) failTicket.setAttempt(data.getRetry()) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 70136392..0e13bba4 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -24,6 +24,7 @@ __license__ = "GPL" import logging, re, os, fcntl, sys, locale, codecs, datetime from .failmanager import FailManagerEmpty, FailManager +from .observer import Observers from .ticket import FailTicket from .jailthread import JailThread from .datedetector import DateDetector @@ -185,6 +186,7 @@ class Filter(JailThread): # @param value the time def setFindTime(self, value): + value = MyTime.str2seconds(value) self.__findTime = value self.failManager.setMaxTime(value) logSys.info("Set findtime = %s" % value) @@ -314,7 +316,7 @@ class Filter(JailThread): # Perform the banning of the IP now. try: # pragma: no branch - exception is the only way out while True: - ticket = self.failManager.toBan() + ticket = self.failManager.toBan(ip) self.jail.putFailTicket(ticket) except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) @@ -419,32 +421,13 @@ class Filter(JailThread): if self.inIgnoreIPList(ip): logSys.info("[%s] Ignore %s" % (self.jail.name, ip)) continue - # increase retry count for known (bad) ip, corresponding banCount of it (one try will count than 2, 3, 5, 9 ...) : - banCount = 0 - retryCount = 1 - timeOfBan = None - db = self.jail.database - if db is not None: - try: - for banCount, timeOfBan, lastBanTime in db.getBan(ip, self.jail): - retryCount = ((1 << (banCount if banCount < 20 else 20))/2 + 1) - # if lastBanTime == -1 or timeOfBan + lastBanTime * 2 > MyTime.time(): - # retryCount = self.failManager.getMaxRetry() - break - retryCount = min(retryCount, self.failManager.getMaxRetry()) - except Exception as e: - logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - #logSys.error('%s', e, exc_info=True) - # check this ticket already known (line was already processed and in the database and will be restored from there): - if timeOfBan is not None and unixTime <= timeOfBan: - logSys.debug("Ignore line for %s before last ban %s < %s" - % (ip, unixTime, timeOfBan)) - continue logSys.info( - ("[%s] Found %s - %s" % (self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S"))) - + ((", %s # -> %s" % (banCount, retryCount)) if banCount != 1 or retryCount != 1 else '') + "[%s] Found %s - %s", self.jail.name, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - self.failManager.addFailure(FailTicket(ip, unixTime, lines), retryCount) + tick = FailTicket(ip, unixTime, lines) + self.failManager.addFailure(tick) + # report to observer - failure was found, for possibly increasing of it retry counter (asynchronous) + Observers.Main.add('failureFound', self.failManager, self.jail, tick) ## # Returns true if the line should be ignored. diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 8e99e780..9d4eec29 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -23,9 +23,10 @@ __author__ = "Cyril Jaquier, Lee Clemens, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Lee Clemens, 2012 Yaroslav Halchenko" __license__ = "GPL" -import Queue, logging +import Queue, logging, math, random from .actions import Actions +from .mytime import MyTime # Gets the instance of the logger. logSys = logging.getLogger(__name__) @@ -72,8 +73,11 @@ class Jail: self.__name = name self.__queue = Queue.Queue() self.__filter = None + # Extra parameters for increase ban time + self._banExtra = {}; logSys.info("Creating new jail '%s'" % self.name) - self._setBackend(backend) + if backend is not None: + self._setBackend(backend) def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self.name) @@ -189,7 +193,7 @@ class Jail: """ self.__queue.put(ticket) # add ban to database moved to actions (should previously check not already banned - # and increase ticket time if "bantimeextra.enabled" set) + # and increase ticket time if "bantime.increment" set) def getFailTicket(self): """Get a fail ticket from the jail. @@ -201,6 +205,57 @@ class Jail: except Queue.Empty: return False + def setBanTimeExtra(self, opt, value): + # merge previous extra with new option: + be = self._banExtra; + if value == '': + value = None + if value is not None: + be[opt] = value; + elif opt in be: + del be[opt] + logSys.info('Set banTime.%s = %s', opt, value) + if opt == 'increment': + if isinstance(value, str): + be[opt] = value.lower() in ("yes", "true", "ok", "1") + if be[opt] and self.database is None: + logSys.warning("ban time increment is not available as long jail database is not set") + if opt in ['maxtime', 'rndtime']: + if not value is None: + be[opt] = MyTime.str2seconds(value) + # prepare formula lambda: + if opt in ['formula', 'factor', 'maxtime', 'rndtime', 'multipliers'] or be.get('evformula', None) is None: + # split multifiers to an array begins with 0 (or empty if not set): + if opt == 'multipliers': + be['evmultipliers'] = [int(i) for i in (value.split(' ') if value is not None and value != '' else [])] + # if we have multifiers - use it in lambda, otherwise compile and use formula within lambda + multipliers = be.get('evmultipliers', []) + banFactor = eval(be.get('factor', "1")) + if len(multipliers): + evformula = lambda ban, banFactor=banFactor: ( + ban.Time * banFactor * multipliers[ban.Count if ban.Count < len(multipliers) else -1] + ) + else: + formula = be.get('formula', 'ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor') + formula = compile(formula, '~inline-conf-expr~', 'eval') + evformula = lambda ban, banFactor=banFactor, formula=formula: max(ban.Time, eval(formula)) + # extend lambda with max time : + if not be.get('maxtime', None) is None: + maxtime = be['maxtime'] + evformula = lambda ban, evformula=evformula: min(evformula(ban), maxtime) + # mix lambda with random time (to prevent bot-nets to calculate exact time IP can be unbanned): + if not be.get('rndtime', None) is None: + rndtime = be['rndtime'] + evformula = lambda ban, evformula=evformula: (evformula(ban) + random.random() * rndtime) + # set to extra dict: + be['evformula'] = evformula + #logSys.info('banTimeExtra : %s' % json.dumps(be)) + + def getBanTimeExtra(self, opt=None): + if opt is not None: + return self._banExtra.get(opt, None) + return self._banExtra + def start(self): """Start the jail, by starting filter and actions threads. @@ -213,9 +268,8 @@ class Jail: try: if self.database is not None: forbantime = None; - if self.actions.getBanTimeExtra('enabled'): - forbantime = self.actions.getBanTimeExtra('findtime') - if forbantime is None: + # use ban time as search time if we have not enabled a increasing: + if not self.getBanTimeExtra('increment'): forbantime = self.actions.getBanTime() for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime): #logSys.debug('restored ticket: %s', ticket) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 43f5dabd..684a7a0c 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -94,7 +94,7 @@ class MyTime: # The string expression will be evaluated as mathematical expression, spaces between each groups # will be wrapped to "+" operand (only if any operand does not specified between). # Because of case insensitivity and overwriting with minutes ("m" or "mm"), the short replacement for month - # are "mo" or "mon" (like %b by date formating). + # are "mo" or "mon". # Ex: 1hour+30min = 5400 # 0d 1h 30m = 5400 # 1year-6mo = 15778800 @@ -109,8 +109,11 @@ class MyTime: #@staticmethod def str2seconds(val): + if isinstance(val, (int, long, float, complex)): + return val for rexp, rpl in ( - (r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), + (r"days?|da|dd?", 24*60*60), (r"week?|wee?|ww?", 7*24*60*60), (r"months?|mon?", (365*3+366)*24*60*60/4/12), + (r"years?|yea?|yy?", (365*3+366)*24*60*60/4), (r"seconds?|sec?|ss?", 1), (r"minutes?|min?|mm?", 60), (r"hours?|ho|hh?", 60*60), ): val = re.sub(r"(?i)(?<=[\d\s])(%s)\b" % rexp, "*"+str(rpl), val) diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py new file mode 100644 index 00000000..84981cdc --- /dev/null +++ b/fail2ban/server/observer.py @@ -0,0 +1,464 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Author: Serg G. Brester (sebres) +# +# This module was written as part of ban time increment feature. + +__author__ = "Serg G. Brester (sebres)" +__copyright__ = "Copyright (c) 2014 Serg G. Brester" +__license__ = "GPL" + +import time, logging +import threading +import os, datetime, math, json, random +import sys +if sys.version_info >= (3, 3): + import importlib.machinery +else: + import imp +from .jailthread import JailThread +from .mytime import MyTime + +# Gets the instance of the logger. +logSys = logging.getLogger(__name__) + +class ObserverThread(threading.Thread): + """Handles observing a database, managing bad ips and ban increment. + + Parameters + ---------- + + Attributes + ---------- + daemon + ident + name + status + active : bool + Control the state of the thread. + idle : bool + Control the idle state of the thread. + sleeptime : int + The time the thread sleeps for in the loop. + """ + + def __init__(self): + self.active = False + self.idle = False + ## Event queue + self._queue_lock = threading.RLock() + self._queue = [] + ## Event, be notified if anything added to event queue + self._notify = threading.Event() + ## Sleep for max 60 seconds, it possible to specify infinite to always sleep up to notifying via event, + ## but so we can later do some service "events" occurred infrequently directly in main loop of observer (not using queue) + self.sleeptime = 60 + # + self._started = False + self._timers = {} + self._paused = False + self.__db = None + self.__db_purge_interval = 60*60 + # start thread + super(ObserverThread, self).__init__(name='Observer') + # observer is a not main thread: + self.daemon = True + + def __getitem__(self, i): + try: + return self._queue[i] + except KeyError: + raise KeyError("Invalid event index : %s" % i) + + def __delitem__(self, name): + try: + del self._queue[i] + except KeyError: + raise KeyError("Invalid event index: %s" % i) + + def __iter__(self): + return iter(self._queue) + + def __len__(self): + return len(self._queue) + + def __eq__(self, other): # Required for Threading + return False + + def __hash__(self): # Required for Threading + return id(self) + + def add_named_timer(self, name, starttime, *event): + """Add a named timer event to queue will start (and wake) in 'starttime' seconds + + Previous timer event with same name will be canceled and trigger self into + queue after new 'starttime' value + """ + t = self._timers.get(name, None) + if t is not None: + t.cancel() + t = threading.Timer(starttime, self.add, event) + self._timers[name] = t + t.start() + + def add_timer(self, starttime, *event): + """Add a timer event to queue will start (and wake) in 'starttime' seconds + """ + t = threading.Timer(starttime, self.add, event) + t.start() + + def pulse_notify(self): + """Notify wakeup (sets and resets notify event) + """ + if not self._paused and self._notify: + self._notify.set() + self._notify.clear() + + def add(self, *event): + """Add a event to queue and notify thread to wake up. + """ + ## lock and add new event to queue: + with self._queue_lock: + self._queue.append(event) + self.pulse_notify() + + def call_lambda(self, l, *args): + l(*args) + + def run(self): + """Main loop for Threading. + + This function is the main loop of the thread. + + Returns + ------- + bool + True when the thread exits nicely. + """ + logSys.info("Observer start...") + ## first time create named timer to purge database each hour (clean old entries) ... + self.add_named_timer('DB_PURGE', self.__db_purge_interval, 'db_purge') + ## Mapping of all possible event types of observer: + __meth = { + 'failureFound': self.failureFound, + 'banFound': self.banFound, + # universal lambda: + 'call': self.call_lambda, + # system and service events: + 'db_set': self.db_set, + 'db_purge': self.db_purge, + # service events of observer self: + 'is_alive' : self.is_alive, + 'is_active': self.is_active, + 'start': self.start, + 'stop': self.stop, + 'shutdown': lambda:() + } + try: + ## check it self with sending is_alive event + self.add('is_alive') + ## if we should stop - break a main loop + while self.active: + ## going sleep, wait for events (in queue) + self.idle = True + self._notify.wait(self.sleeptime) + # does not clear notify event here - we use pulse (and clear it inside) ... + # ## wake up - reset signal now (we don't need it so long as we reed from queue) + # if self._notify: + # self._notify.clear() + if self._paused: + continue + self.idle = False + ## check events available and execute all events from queue + while not self._paused: + ## lock, check and pop one from begin of queue: + try: + ev = None + with self._queue_lock: + if len(self._queue): + ev = self._queue.pop(0) + if ev is None: + break + ## retrieve method by name + meth = __meth[ev[0]] + ## execute it with rest of event as variable arguments + meth(*ev[1:]) + except Exception as e: + #logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + logSys.error('%s', e, exc_info=True) + ## end of main loop - exit + except Exception as e: + logSys.error('Observer stopped after error: %s', e, exc_info=True) + #print("Observer stopped with error: %s" % str(e)) + self.idle = True + return True + logSys.info("Observer stopped, %s events remaining.", len(self._queue)) + #print("Observer stopped, %s events remaining." % len(self._queue)) + self.idle = True + return True + + def is_alive(self): + #logSys.debug("Observer alive...") + return True + + def is_active(self, fromStr=None): + # logSys.info("Observer alive, %s%s", + # 'active' if self.active else 'inactive', + # '' if fromStr is None else (", called from '%s'" % fromStr)) + return self.active + + def start(self): + with self._queue_lock: + if not self.active: + self.active = True + super(ObserverThread, self).start() + + def stop(self): + logSys.info("Observer stop ...") + #print("Observer stop ....") + self.active = False + if self._notify: + # just add shutdown job to make possible wait later until full (events remaining) + self.add('shutdown') + self.pulse_notify() + self._notify = None + # wait max 5 seconds until full (events remaining) + self.wait_empty(5) + + @property + def is_full(self): + with self._queue_lock: + return True if len(self._queue) else False + + def wait_empty(self, sleeptime=None): + """Wait observer is running and returns if observer has no more events (queue is empty) + """ + if not self.is_full: + return True + if sleeptime is not None: + e = MyTime.time() + sleeptime + while self.is_full: + if sleeptime is not None and MyTime.time() > e: + break + time.sleep(0.1) + return not self.is_full + + + def wait_idle(self, sleeptime=None): + """Wait observer is running and returns if observer idle (observer sleeps) + """ + time.sleep(0.001) + if self.idle: + return True + if sleeptime is not None: + e = MyTime.time() + sleeptime + while not self.idle: + if sleeptime is not None and MyTime.time() > e: + break + time.sleep(0.1) + return self.idle + + @property + def paused(self): + return self._paused; + + @paused.setter + def paused(self, pause): + if self._paused == pause: + return + self._paused = pause + # wake after pause ended + self.pulse_notify() + + + @property + def status(self): + """Status of observer to be implemented. [TODO] + """ + return ('', '') + + ## ----------------------------------------- + ## [Async] database service functionality ... + ## ----------------------------------------- + + def db_set(self, db): + self.__db = db + + def db_purge(self): + logSys.info("Purge database event occurred") + if self.__db is not None: + self.__db.purge() + # trigger timer again ... + self.add_named_timer('DB_PURGE', self.__db_purge_interval, 'db_purge') + + ## ----------------------------------------- + ## [Async] ban time increment functionality ... + ## ----------------------------------------- + + def failureFound(self, failManager, jail, ticket): + """ Notify observer a failure for ip was found + + Observer will check ip was known (bad) and possibly increase an retry count + """ + # check jail active : + if not jail.is_alive(): + return + ip = ticket.getIP() + unixTime = ticket.getTime() + logSys.info("[%s] Observer: failure found %s", jail.name, ip) + # increase retry count for known (bad) ip, corresponding banCount of it (one try will count than 2, 3, 5, 9 ...) : + banCount = 0 + retryCount = 1 + timeOfBan = None + try: + db = jail.database + if db is not None: + for banCount, timeOfBan, lastBanTime in db.getBan(ip, jail): + retryCount = ((1 << (banCount if banCount < 20 else 20))/2 + 1) + # if lastBanTime == -1 or timeOfBan + lastBanTime * 2 > MyTime.time(): + # retryCount = failManager.getMaxRetry() + break + retryCount = min(retryCount, failManager.getMaxRetry()) + # check this ticket already known (line was already processed and in the database and will be restored from there): + if timeOfBan is not None and unixTime <= timeOfBan: + logSys.info("[%s] Ignore failure %s before last ban %s < %s, restored" + % (jail.name, ip, unixTime, timeOfBan)) + return + # for not increased failures observer should not add it to fail manager, because was already added by filter self + if retryCount <= 1: + return + # retry counter was increased - add it again: + logSys.info("[%s] Found %s, bad - %s, %s # -> %s, ban", jail.name, ip, + datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S"), banCount, retryCount) + # remove matches from this ticket, because a ticket was already added by filter self + ticket.setMatches(None) + # retryCount-1, because a ticket was already once incremented by filter self + failManager.addFailure(ticket, retryCount - 1, True) + + # after observe we have increased count >= maxretry ... + if retryCount >= failManager.getMaxRetry(): + # perform the banning of the IP now (again) + # [todo]: this code part will be used multiple times - optimize it later. + try: # pragma: no branch - exception is the only way out + while True: + ticket = failManager.toBan(ip) + jail.putFailTicket(ticket) + except Exception: + failManager.cleanup(MyTime.time()) + + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + + + class BanTimeIncr: + def __init__(self, banTime, banCount): + self.Time = banTime + self.Count = banCount + + def calcBanTime(self, jail, banTime, banCount): + be = jail.getBanTimeExtra() + return be['evformula'](self.BanTimeIncr(banTime, banCount)) + + def incrBanTime(self, jail, banTime, ticket): + """Check for IP address to increment ban time (if was already banned). + + Returns + ------- + float + new ban time. + """ + # check jail active : + if not jail.is_alive(): + return + be = jail.getBanTimeExtra() + ip = ticket.getIP() + orgBanTime = banTime + # check ip was already banned (increment time of ban): + try: + if banTime > 0 and be.get('increment', False): + # search IP in database and increase time if found: + for banCount, timeOfBan, lastBanTime in \ + jail.database.getBan(ip, jail, overalljails=be.get('overalljails', False)) \ + : + logSys.debug('IP %s was already banned: %s #, %s' % (ip, banCount, timeOfBan)); + ticket.setBanCount(banCount); + # calculate new ban time + if banCount > 0: + banTime = be['evformula'](self.BanTimeIncr(banTime, banCount)) + ticket.setBanTime(banTime); + # check current ticket time to prevent increasing for twice read tickets (restored from log file besides database after restart) + if ticket.getTime() > timeOfBan: + logSys.info('[%s] IP %s is bad: %s # last %s - incr %s to %s' % (jail.name, ip, banCount, + datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), + datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + else: + ticket.setRestored(True) + break + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + return banTime + + def banFound(self, ticket, jail, btime): + """ Notify observer a ban occured for ip + + Observer will check ip was known (bad) and possibly increase/prolong a ban time + Secondary we will actualize the bans and bips (bad ip) in database + """ + oldbtime = btime + ip = ticket.getIP() + logSys.info("[%s] Observer: ban found %s, %s", jail.name, ip, btime) + try: + # if not permanent, not restored and ban time was not set - check time should be increased: + if btime != -1 and not ticket.getRestored() and ticket.getBanTime() is None: + btime = self.incrBanTime(jail, btime, ticket) + # if we should prolong ban time: + if btime == -1 or btime > oldbtime: + ticket.setBanTime(btime) + # if not permanent + if btime != -1: + bendtime = ticket.getTime() + btime + logtime = (datetime.timedelta(seconds=int(btime)), + datetime.datetime.fromtimestamp(bendtime).strftime("%Y-%m-%d %H:%M:%S")) + # check ban is not too old : + if bendtime < MyTime.time(): + logSys.info('Ignore old bantime %s', logtime[1]) + return False + else: + logtime = ('permanent', 'infinite') + # if ban time was prolonged - log again with new ban time: + if btime != oldbtime: + logSys.notice("[%s] Increase Ban %s (%d # %s -> %s)", jail.name, + ip, ticket.getBanCount()+1, *logtime) + # add ticket to database, but only if was not restored (not already read from database): + if jail.database is not None and not ticket.getRestored(): + # add to database always only after ban time was calculated an not yet already banned: + jail.database.addBan(jail, ticket) + except Exception as e: + logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + #logSys.error('%s', e, exc_info=True) + +# Global observer initial created in server (could be later rewriten via singleton) +class _Observers: + def __init__(self): + self.Main = None + +Observers = _Observers() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index b9efdfe8..34c984d9 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -27,6 +27,7 @@ __license__ = "GPL" from threading import Lock, RLock import logging, logging.handlers, sys, os, signal +from .observer import Observers, ObserverThread from .jails import Jails from .filter import FileFilter, JournalFilter from .transmitter import Transmitter @@ -101,6 +102,10 @@ class Server: os.remove(pidfile) except OSError, e: logSys.error("Unable to remove PID file: %s" % e) + # Stop observer and exit + if Observers.Main is not None: + Observers.Main.stop() + Observers.Main = None logSys.info("Exiting Fail2ban") def quit(self): @@ -124,10 +129,16 @@ class Server: def addJail(self, name, backend): + # Create an observer if not yet created and start it: + if Observers.Main is None: + Observers.Main = ObserverThread() + Observers.Main.start() + # Add jail hereafter: self.__jails.add(name, backend, self.__db) if self.__db is not None: self.__db.addJail(self.__jails[name]) - + Observers.Main.db_set(self.__db) + def delJail(self, name): if self.__db is not None: self.__db.delJail(self.__jails[name]) @@ -304,10 +315,10 @@ class Server: return self.__jails[name].actions.getBanTime() def setBanTimeExtra(self, name, opt, value): - self.__jails[name].actions.setBanTimeExtra(opt, value) + self.__jails[name].setBanTimeExtra(opt, value) def getBanTimeExtra(self, name, opt): - return self.__jails[name].actions.getBanTimeExtra(opt) + return self.__jails[name].getBanTimeExtra(opt) # Status def status(self): diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index d339dbb3..7d731637 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -105,6 +105,9 @@ class Ticket: def getAttempt(self): return self.__attempt + def setMatches(self, matches): + self.__matches = matches + def getMatches(self): return self.__matches diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 471cbea4..01c2bcd7 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -203,7 +203,7 @@ class Transmitter: return self.__server.getUseDns(name) elif command[1] == "findtime": value = command[2] - self.__server.setFindTime(name, int(value)) + self.__server.setFindTime(name, value) return self.__server.getFindTime(name) elif command[1] == "datepattern": value = command[2] @@ -220,11 +220,11 @@ class Transmitter: # command elif command[1] == "bantime": value = command[2] - self.__server.setBanTime(name, int(value)) + self.__server.setBanTime(name, value) return self.__server.getBanTime(name) - elif command[1].startswith("bantimeextra."): + elif command[1].startswith("bantime."): value = command[2] - opt = command[1][len("bantimeextra."):] + opt = command[1][len("bantime."):] self.__server.setBanTimeExtra(name, opt, value) return self.__server.getBanTimeExtra(name, opt) elif command[1] == "banip": @@ -305,8 +305,8 @@ class Transmitter: # Action elif command[1] == "bantime": return self.__server.getBanTime(name) - elif command[1].startswith("bantimeextra."): - opt = command[1][len("bantimeextra."):] + elif command[1].startswith("bantime."): + opt = command[1][len("bantime."):] return self.__server.getBanTimeExtra(name, opt) elif command[1] == "actions": return self.__server.getActions(name).keys() diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index d246c144..d4a54cdb 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -141,139 +141,3 @@ class ExecuteActions(LogCaptureTestCase): self.__actions.join() self.assertTrue(self._is_logged("Failed to stop")) - -# Author: Serg G. Brester (sebres) -# - -__author__ = "Serg Brester" -__copyright__ = "Copyright (c) 2014 Serg G. Brester" - -class BanTimeIncr(LogCaptureTestCase): - - def setUp(self): - """Call before every test case.""" - super(BanTimeIncr, self).setUp() - self.__jail = DummyJail() - self.__actions = Actions(self.__jail) - self.__tmpfile, self.__tmpfilename = tempfile.mkstemp() - - def tearDown(self): - super(BanTimeIncr, self).tearDown() - os.remove(self.__tmpfilename) - - def testDefault(self, multipliers = None): - a = self.__actions; - a.setBanTimeExtra('maxtime', '24*60*60') - a.setBanTimeExtra('rndtime', None) - a.setBanTimeExtra('factor', None) - # tests formulat or multipliers: - a.setBanTimeExtra('multipliers', multipliers) - # test algorithm and max time 24 hours : - self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], - [1200, 2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400] - ) - # with extra large max time (30 days): - a.setBanTimeExtra('maxtime', '30*24*60*60') - # using formula the ban time grows always, but using multipliers the growing will stops with last one: - arr = [1200, 2400, 4800, 9600, 19200, 38400, 76800, 153600, 307200, 614400] - if multipliers is not None: - multcnt = len(multipliers.split(' ')) - if multcnt < 11: - arr = arr[0:multcnt-1] + ([arr[multcnt-2]] * (11-multcnt)) - self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], - arr - ) - a.setBanTimeExtra('maxtime', '24*60*60') - # change factor : - a.setBanTimeExtra('factor', '2'); - self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], - [2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400, 86400] - ) - # factor is float : - a.setBanTimeExtra('factor', '1.33'); - self.assertEqual( - [int(a.calcBanTime(600, i)) for i in xrange(1, 11)], - [1596, 3192, 6384, 12768, 25536, 51072, 86400, 86400, 86400, 86400] - ) - a.setBanTimeExtra('factor', None); - # change max time : - a.setBanTimeExtra('maxtime', '12*60*60') - self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], - [1200, 2400, 4800, 9600, 19200, 38400, 43200, 43200, 43200, 43200] - ) - a.setBanTimeExtra('maxtime', '24*60*60') - ## test randomization - not possibe all 10 times we have random = 0: - a.setBanTimeExtra('rndtime', '5*60') - self.assertTrue( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] - ) - a.setBanTimeExtra('rndtime', None) - self.assertFalse( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] - ) - # restore default: - a.setBanTimeExtra('multipliers', None) - a.setBanTimeExtra('factor', None); - a.setBanTimeExtra('maxtime', '24*60*60') - a.setBanTimeExtra('rndtime', None) - - def testMultipliers(self): - # this multipliers has the same values as default formula, we test stop growing after count 9: - self.testDefault('1 2 4 8 16 32 64 128 256') - # this multipliers has exactly the same values as default formula, test endless growing (stops by count 31 only): - self.testDefault(' '.join([str(1<= (2,7): # pragma: no cover - raise unittest.SkipTest( - "Unable to import fail2ban database module as sqlite is not " - "available.") - elif Fail2BanDb is None: - return - _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") - self.db = Fail2BanDb(self.dbFilename) - - def tearDown(self): - """Call after every test case.""" - if Fail2BanDb is None: # pragma: no cover - return - # Cleanup - os.remove(self.dbFilename) - - def testBanTimeIncr(self): - if Fail2BanDb is None: # pragma: no cover - return - jail = DummyJail() - jail.database = self.db - self.db.addJail(jail) - a = jail.actions - # we tests with initial ban time = 10 seconds: - a.setBanTime(10) - a.setBanTimeExtra('enabled', 'true') - a.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" - # used as start and fromtime (like now but time independence, cause test case can run slow): - stime = int(MyTime.time()) - ticket = FailTicket(ip, stime, []) - # test ticket not yet found - self.assertEqual( - [a.incrBanTime(ticket) for i in xrange(3)], - [10, 10, 10] - ) - # add a ticket banned - self.db.addBan(jail, ticket) - # get a ticket already banned in this jail: - self.assertEqual( - [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, jail, None, False)], - [(1, stime, 10)] - ) - # incr time and ban a ticket again : - ticket.setTime(stime + 15) - self.assertEqual(a.incrBanTime(ticket), 20) - self.db.addBan(jail, ticket) - # get a ticket already banned in this jail: - self.assertEqual( - [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, jail, None, False)], - [(2, stime + 15, 20)] - ) - # get a ticket already banned in all jails: - self.assertEqual( - [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, '', None, True)], - [(2, stime + 15, 20)] - ) - # search currently banned and 1 day later (nothing should be found): - self.assertEqual( - self.db.getCurrentBans(forbantime=-24*60*60, fromtime=stime), - [] - ) - # search currently banned anywhere: - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual( - str(restored_tickets), - ('[FailTicket: ip=%s time=%s bantime=20 bancount=2 #attempts=0 matches=[]]' % (ip, stime + 15)) - ) - # search currently banned: - restored_tickets = self.db.getCurrentBans(jail=jail, fromtime=stime) - self.assertEqual( - str(restored_tickets), - ('[FailTicket: ip=%s time=%s bantime=20 bancount=2 #attempts=0 matches=[]]' % (ip, stime + 15)) - ) - restored_tickets[0].setRestored(True) - self.assertTrue(restored_tickets[0].getRestored()) - # increase ban multiple times: - lastBanTime = 20 - for i in xrange(10): - ticket.setTime(stime + lastBanTime + 5) - banTime = a.incrBanTime(ticket) - self.assertEqual(banTime, lastBanTime * 2) - self.db.addBan(jail, ticket) - lastBanTime = banTime - # increase again, but the last multiplier reached (time not increased): - ticket.setTime(stime + lastBanTime + 5) - banTime = a.incrBanTime(ticket) - self.assertNotEqual(banTime, lastBanTime * 2) - self.assertEqual(banTime, lastBanTime) - self.db.addBan(jail, ticket) - lastBanTime = banTime - # add two tickets from yesterday: one unbanned (bantime already out-dated): - ticket2 = FailTicket(ip+'2', stime-24*60*60, []) - ticket2.setBanTime(12*60*60) - self.db.addBan(jail, ticket2) - # and one from yesterday also, but still currently banned : - ticket2 = FailTicket(ip+'1', stime-24*60*60, []) - ticket2.setBanTime(36*60*60) - self.db.addBan(jail, ticket2) - # search currently banned: - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(len(restored_tickets), 2) - self.assertEqual( - str(restored_tickets[0]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=13 #attempts=0 matches=[]' % (ip, stime + lastBanTime + 5, lastBanTime) - ) - self.assertEqual( - str(restored_tickets[1]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'1', stime-24*60*60, 36*60*60) - ) - # search out-dated (give another fromtime now is -18 hours): - restored_tickets = self.db.getCurrentBans(fromtime=stime-18*60*60) - self.assertEqual(len(restored_tickets), 3) - self.assertEqual( - str(restored_tickets[2]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'2', stime-24*60*60, 12*60*60) - ) - # should be still banned - self.assertFalse(restored_tickets[1].isTimedOut(stime)) - self.assertFalse(restored_tickets[1].isTimedOut(stime)) - # the last should be timed out now - self.assertTrue(restored_tickets[2].isTimedOut(stime)) - self.assertFalse(restored_tickets[2].isTimedOut(stime-18*60*60)) - - # test permanent, create timed out: - ticket=FailTicket(ip+'3', stime-36*60*60, []) - self.assertTrue(ticket.isTimedOut(stime, 600)) - # not timed out - permanent jail: - self.assertFalse(ticket.isTimedOut(stime, -1)) - # not timed out - permanent ticket: - ticket.setBanTime(-1) - self.assertFalse(ticket.isTimedOut(stime, 600)) - self.assertFalse(ticket.isTimedOut(stime, -1)) - # timed out - permanent jail but ticket time (not really used behavior) - ticket.setBanTime(600) - self.assertTrue(ticket.isTimedOut(stime, -1)) - - # get currently banned pis with permanent one: - ticket.setBanTime(-1) - self.db.addBan(jail, ticket) - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(len(restored_tickets), 3) - self.assertEqual( - str(restored_tickets[2]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'3', stime-36*60*60, -1) - ) - # purge (nothing should be changed): - self.db.purge() - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(len(restored_tickets), 3) - # set short time and purge again: - ticket.setBanTime(600) - self.db.addBan(jail, ticket) - self.db.purge() - # this old ticket should be removed now: - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(len(restored_tickets), 2) - self.assertEqual(restored_tickets[0].getIP(), ip) - - # purge remove 1st ip - self.db._purgeAge = -48*60*60 - self.db.purge() - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(len(restored_tickets), 1) - self.assertEqual(restored_tickets[0].getIP(), ip+'1') - - # this should purge all bans, bips and logs - nothing should be found now - self.db._purgeAge = -240*60*60 - self.db.purge() - restored_tickets = self.db.getCurrentBans(fromtime=stime) - self.assertEqual(restored_tickets, []) - - # two separate jails : - jail1 = DummyJail() - jail1.database = self.db - self.db.addJail(jail1) - jail2 = DummyJail() - jail2.database = self.db - self.db.addJail(jail2) - ticket1 = FailTicket(ip, stime, []) - ticket1.setBanTime(6000) - self.db.addBan(jail1, ticket1) - ticket2 = FailTicket(ip, stime-6000, []) - ticket2.setBanTime(12000) - ticket2.setBanCount(1) - self.db.addBan(jail2, ticket2) - restored_tickets = self.db.getCurrentBans(jail=jail1, fromtime=stime) - self.assertEqual(len(restored_tickets), 1) - self.assertEqual( - str(restored_tickets[0]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip, stime, 6000) - ) - restored_tickets = self.db.getCurrentBans(jail=jail2, fromtime=stime) - self.assertEqual(len(restored_tickets), 1) - self.assertEqual( - str(restored_tickets[0]), - 'FailTicket: ip=%s time=%s bantime=%s bancount=2 #attempts=0 matches=[]' % (ip, stime-6000, 12000) - ) - # get last ban values for this ip separately for each jail: - for row in self.db.getBan(ip, jail1): - self.assertEqual(row, (1, stime, 6000)) - break - for row in self.db.getBan(ip, jail2): - self.assertEqual(row, (2, stime-6000, 12000)) - break - # get max values for this ip (over all jails): - for row in self.db.getBan(ip, overalljails=True): - self.assertEqual(row, (3, stime, 18000)) - break diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index 77ab785e..4ca7e108 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -24,17 +24,18 @@ __license__ = "GPL" from threading import Lock +from ..server.jail import Jail from ..server.actions import Actions -class DummyJail(object): +class DummyJail(Jail, object): """A simple 'jail' to suck in all the tickets generated by Filter's """ def __init__(self): self.lock = Lock() self.queue = [] - self.idle = False - self.database = None - self.actions = Actions(self) + super(DummyJail, self).__init__(name='DummyJail', backend=None) + self.__db = None + self.__actions = Actions(self) def __len__(self): try: @@ -63,3 +64,26 @@ class DummyJail(object): @property def name(self): return "DummyJail #%s with %d tickets" % (id(self), len(self)) + + @property + def idle(self): + return False; + + @idle.setter + def idle(self, value): + pass + + @property + def database(self): + return self.__db; + + @database.setter + def database(self, value): + self.__db = value; + + @property + def actions(self): + return self.__actions; + + def is_alive(self): + return True; diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py new file mode 100644 index 00000000..3e7c5fda --- /dev/null +++ b/fail2ban/tests/observertestcase.py @@ -0,0 +1,445 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Author: Serg G. Brester (sebres) +# + +__author__ = "Serg G. Brester (sebres)" +__copyright__ = "Copyright (c) 2014 Serg G. Brester" +__license__ = "GPL" + +import os +import sys +import unittest +import tempfile +import time + +from ..server.mytime import MyTime +from ..server.ticket import FailTicket +from ..server.observer import Observers, ObserverThread +from .utils import LogCaptureTestCase +from .dummyjail import DummyJail +try: + from ..server.database import Fail2BanDb +except ImportError: + Fail2BanDb = None + + +class BanTimeIncr(LogCaptureTestCase): + + def setUp(self): + """Call before every test case.""" + super(BanTimeIncr, self).setUp() + self.__jail = DummyJail() + self.__jail.calcBanTime = self.calcBanTime + self.Observer = ObserverThread() + + def tearDown(self): + super(BanTimeIncr, self).tearDown() + + def calcBanTime(self, banTime, banCount): + return self.Observer.calcBanTime(self.__jail, banTime, banCount) + + def testDefault(self, multipliers = None): + a = self.__jail; + a.setBanTimeExtra('increment', 'true') + a.setBanTimeExtra('maxtime', '1d') + a.setBanTimeExtra('rndtime', None) + a.setBanTimeExtra('factor', None) + # tests formulat or multipliers: + a.setBanTimeExtra('multipliers', multipliers) + # test algorithm and max time 24 hours : + self.assertEqual( + [a.calcBanTime(600, i) for i in xrange(1, 11)], + [1200, 2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400] + ) + # with extra large max time (30 days): + a.setBanTimeExtra('maxtime', '30d') + # using formula the ban time grows always, but using multipliers the growing will stops with last one: + arr = [1200, 2400, 4800, 9600, 19200, 38400, 76800, 153600, 307200, 614400] + if multipliers is not None: + multcnt = len(multipliers.split(' ')) + if multcnt < 11: + arr = arr[0:multcnt-1] + ([arr[multcnt-2]] * (11-multcnt)) + self.assertEqual( + [a.calcBanTime(600, i) for i in xrange(1, 11)], + arr + ) + a.setBanTimeExtra('maxtime', '1d') + # change factor : + a.setBanTimeExtra('factor', '2'); + self.assertEqual( + [a.calcBanTime(600, i) for i in xrange(1, 11)], + [2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400, 86400] + ) + # factor is float : + a.setBanTimeExtra('factor', '1.33'); + self.assertEqual( + [int(a.calcBanTime(600, i)) for i in xrange(1, 11)], + [1596, 3192, 6384, 12768, 25536, 51072, 86400, 86400, 86400, 86400] + ) + a.setBanTimeExtra('factor', None); + # change max time : + a.setBanTimeExtra('maxtime', '12h') + self.assertEqual( + [a.calcBanTime(600, i) for i in xrange(1, 11)], + [1200, 2400, 4800, 9600, 19200, 38400, 43200, 43200, 43200, 43200] + ) + a.setBanTimeExtra('maxtime', '24h') + ## test randomization - not possibe all 10 times we have random = 0: + a.setBanTimeExtra('rndtime', '5m') + self.assertTrue( + False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + ) + a.setBanTimeExtra('rndtime', None) + self.assertFalse( + False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + ) + # restore default: + a.setBanTimeExtra('multipliers', None) + a.setBanTimeExtra('factor', None); + a.setBanTimeExtra('maxtime', '24h') + a.setBanTimeExtra('rndtime', None) + + def testMultipliers(self): + # this multipliers has the same values as default formula, we test stop growing after count 9: + self.testDefault('1 2 4 8 16 32 64 128 256') + # this multipliers has exactly the same values as default formula, test endless growing (stops by count 31 only): + self.testDefault(' '.join([str(1<= (2,7): # pragma: no cover + raise unittest.SkipTest( + "Unable to import fail2ban database module as sqlite is not " + "available.") + elif Fail2BanDb is None: + return + _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") + self.db = Fail2BanDb(self.dbFilename) + self.jail = None + self.Observer = ObserverThread() + + def tearDown(self): + """Call after every test case.""" + super(BanTimeIncrDB, self).tearDown() + if Fail2BanDb is None: # pragma: no cover + return + # Cleanup + os.remove(self.dbFilename) + + def incrBanTime(self, ticket, banTime=None): + jail = self.jail; + if banTime is None: + banTime = ticket.getBanTime(jail.actions.getBanTime()) + ticket.setBanTime(None) + incrTime = self.Observer.incrBanTime(jail, banTime, ticket) + #print("!!!!!!!!! banTime: %s, %s, incr: %s " % (banTime, ticket.getBanCount(), incrTime)) + return incrTime + + + def testBanTimeIncr(self): + if Fail2BanDb is None: # pragma: no cover + return + jail = DummyJail() + self.jail = jail + jail.database = self.db + self.db.addJail(jail) + # we tests with initial ban time = 10 seconds: + jail.actions.setBanTime(10) + jail.setBanTimeExtra('increment', 'true') + jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') + ip = "127.0.0.2" + # used as start and fromtime (like now but time independence, cause test case can run slow): + stime = int(MyTime.time()) + ticket = FailTicket(ip, stime, []) + # test ticket not yet found + self.assertEqual( + [self.incrBanTime(ticket, 10) for i in xrange(3)], + [10, 10, 10] + ) + # add a ticket banned + self.db.addBan(jail, ticket) + # get a ticket already banned in this jail: + self.assertEqual( + [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, jail, None, False)], + [(1, stime, 10)] + ) + # incr time and ban a ticket again : + ticket.setTime(stime + 15) + self.assertEqual(self.incrBanTime(ticket, 10), 20) + self.db.addBan(jail, ticket) + # get a ticket already banned in this jail: + self.assertEqual( + [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, jail, None, False)], + [(2, stime + 15, 20)] + ) + # get a ticket already banned in all jails: + self.assertEqual( + [(banCount, timeOfBan, lastBanTime) for banCount, timeOfBan, lastBanTime in self.db.getBan(ip, '', None, True)], + [(2, stime + 15, 20)] + ) + # search currently banned and 1 day later (nothing should be found): + self.assertEqual( + self.db.getCurrentBans(forbantime=-24*60*60, fromtime=stime), + [] + ) + # search currently banned anywhere: + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual( + str(restored_tickets), + ('[FailTicket: ip=%s time=%s bantime=20 bancount=2 #attempts=0 matches=[]]' % (ip, stime + 15)) + ) + # search currently banned: + restored_tickets = self.db.getCurrentBans(jail=jail, fromtime=stime) + self.assertEqual( + str(restored_tickets), + ('[FailTicket: ip=%s time=%s bantime=20 bancount=2 #attempts=0 matches=[]]' % (ip, stime + 15)) + ) + restored_tickets[0].setRestored(True) + self.assertTrue(restored_tickets[0].getRestored()) + # increase ban multiple times: + lastBanTime = 20 + for i in xrange(10): + ticket.setTime(stime + lastBanTime + 5) + banTime = self.incrBanTime(ticket, 10) + self.assertEqual(banTime, lastBanTime * 2) + self.db.addBan(jail, ticket) + lastBanTime = banTime + # increase again, but the last multiplier reached (time not increased): + ticket.setTime(stime + lastBanTime + 5) + banTime = self.incrBanTime(ticket, 10) + self.assertNotEqual(banTime, lastBanTime * 2) + self.assertEqual(banTime, lastBanTime) + self.db.addBan(jail, ticket) + lastBanTime = banTime + # add two tickets from yesterday: one unbanned (bantime already out-dated): + ticket2 = FailTicket(ip+'2', stime-24*60*60, []) + ticket2.setBanTime(12*60*60) + self.db.addBan(jail, ticket2) + # and one from yesterday also, but still currently banned : + ticket2 = FailTicket(ip+'1', stime-24*60*60, []) + ticket2.setBanTime(36*60*60) + self.db.addBan(jail, ticket2) + # search currently banned: + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 2) + self.assertEqual( + str(restored_tickets[0]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=13 #attempts=0 matches=[]' % (ip, stime + lastBanTime + 5, lastBanTime) + ) + self.assertEqual( + str(restored_tickets[1]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'1', stime-24*60*60, 36*60*60) + ) + # search out-dated (give another fromtime now is -18 hours): + restored_tickets = self.db.getCurrentBans(fromtime=stime-18*60*60) + self.assertEqual(len(restored_tickets), 3) + self.assertEqual( + str(restored_tickets[2]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'2', stime-24*60*60, 12*60*60) + ) + # should be still banned + self.assertFalse(restored_tickets[1].isTimedOut(stime)) + self.assertFalse(restored_tickets[1].isTimedOut(stime)) + # the last should be timed out now + self.assertTrue(restored_tickets[2].isTimedOut(stime)) + self.assertFalse(restored_tickets[2].isTimedOut(stime-18*60*60)) + + # test permanent, create timed out: + ticket=FailTicket(ip+'3', stime-36*60*60, []) + self.assertTrue(ticket.isTimedOut(stime, 600)) + # not timed out - permanent jail: + self.assertFalse(ticket.isTimedOut(stime, -1)) + # not timed out - permanent ticket: + ticket.setBanTime(-1) + self.assertFalse(ticket.isTimedOut(stime, 600)) + self.assertFalse(ticket.isTimedOut(stime, -1)) + # timed out - permanent jail but ticket time (not really used behavior) + ticket.setBanTime(600) + self.assertTrue(ticket.isTimedOut(stime, -1)) + + # get currently banned pis with permanent one: + ticket.setBanTime(-1) + self.db.addBan(jail, ticket) + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 3) + self.assertEqual( + str(restored_tickets[2]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip+'3', stime-36*60*60, -1) + ) + # purge (nothing should be changed): + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 3) + # set short time and purge again: + ticket.setBanTime(600) + self.db.addBan(jail, ticket) + self.db.purge() + # this old ticket should be removed now: + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 2) + self.assertEqual(restored_tickets[0].getIP(), ip) + + # purge remove 1st ip + self.db._purgeAge = -48*60*60 + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual(restored_tickets[0].getIP(), ip+'1') + + # this should purge all bans, bips and logs - nothing should be found now + self.db._purgeAge = -240*60*60 + self.db.purge() + restored_tickets = self.db.getCurrentBans(fromtime=stime) + self.assertEqual(restored_tickets, []) + + # two separate jails : + jail1 = DummyJail() + jail1.database = self.db + self.db.addJail(jail1) + jail2 = DummyJail() + jail2.database = self.db + self.db.addJail(jail2) + ticket1 = FailTicket(ip, stime, []) + ticket1.setBanTime(6000) + self.db.addBan(jail1, ticket1) + ticket2 = FailTicket(ip, stime-6000, []) + ticket2.setBanTime(12000) + ticket2.setBanCount(1) + self.db.addBan(jail2, ticket2) + restored_tickets = self.db.getCurrentBans(jail=jail1, fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual( + str(restored_tickets[0]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=1 #attempts=0 matches=[]' % (ip, stime, 6000) + ) + restored_tickets = self.db.getCurrentBans(jail=jail2, fromtime=stime) + self.assertEqual(len(restored_tickets), 1) + self.assertEqual( + str(restored_tickets[0]), + 'FailTicket: ip=%s time=%s bantime=%s bancount=2 #attempts=0 matches=[]' % (ip, stime-6000, 12000) + ) + # get last ban values for this ip separately for each jail: + for row in self.db.getBan(ip, jail1): + self.assertEqual(row, (1, stime, 6000)) + break + for row in self.db.getBan(ip, jail2): + self.assertEqual(row, (2, stime-6000, 12000)) + break + # get max values for this ip (over all jails): + for row in self.db.getBan(ip, overalljails=True): + self.assertEqual(row, (3, stime, 18000)) + break + + +class ObserverTest(unittest.TestCase): + + def setUp(self): + """Call before every test case.""" + #super(ObserverTest, self).setUp() + pass + + def tearDown(self): + #super(ObserverTest, self).tearDown() + pass + + def testObserverBanTimeIncr(self): + obs = ObserverThread() + obs.start() + # wait for idle + obs.wait_idle(0.1) + # observer will sleep 0.5 second (in busy state): + o = set(['test']) + obs.add('call', o.clear) + obs.add('call', o.add, 'test2') + obs.wait_empty(1) + self.assertFalse(obs.is_full) + self.assertEqual(o, set(['test2'])) + # observer makes pause + obs.paused = True + # observer will sleep 0.5 second after pause ends: + obs.add('call', o.clear) + obs.add('call', o.add, 'test3') + obs.wait_empty(0.25) + self.assertTrue(obs.is_full) + self.assertEqual(o, set(['test2'])) + obs.paused = False + # wait running: + obs.wait_empty(1) + self.assertEqual(o, set(['test3'])) + + self.assertTrue(obs.is_active()) + self.assertTrue(obs.is_alive()) + obs.stop() + obs = None diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8e3252e2..75186ee1 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -68,6 +68,7 @@ def gatherTests(regexps=None, no_network=False): from . import sockettestcase from . import misctestcase from . import databasetestcase + from . import observertestcase from . import samplestestcase if not regexps: # pragma: no cover @@ -91,7 +92,6 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(servertestcase.RegexTests)) tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest)) tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions)) - tests.addTest(unittest.makeSuite(actionstestcase.BanTimeIncr)) # FailManager tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager @@ -110,7 +110,10 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest)) # Database tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest)) - tests.addTest(unittest.makeSuite(databasetestcase.BanTimeIncr)) + # Observer + tests.addTest(unittest.makeSuite(observertestcase.ObserverTest)) + tests.addTest(unittest.makeSuite(observertestcase.BanTimeIncr)) + tests.addTest(unittest.makeSuite(observertestcase.BanTimeIncrDB)) # Filter tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIP))