diff --git a/ChangeLog b/ChangeLog index d92aec4a..0515823a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger provides defaults for the chain, port, protocol and name tags - Fixes: + * start of file2ban aborted (on slow hosts, systemd considers the server has been + timed out and kills him), see gh-824 * UTF-8 fixes in pure-ftp thanks to Johannes Weberhofer. Closes gh-806. * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request @@ -64,6 +66,10 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Added Cloudflare API action - Enhancements + * Start performance of fail2ban-client (and tests) increased, start time + and cpu usage rapidly reduced. Introduced a shared storage logic, to bypass + reading lots of config files (see gh-824). + Thanks to Joost Molenaar for good catch (reported gh-820). * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options * Match non "Bye Bye" disconnect messages for sshd locked account regex diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 89e0a903..0c6999c1 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -409,6 +409,7 @@ class Fail2banClient: # TODO: get away from stew of return codes and exception # handling -- handle via exceptions try: + self.__configurator.Reload() self.__configurator.readAll() ret = self.__configurator.getOptions(jail) self.__configurator.convertToProtocol() diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 91f3322b..8022ecc1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..helpers import getLogger # Gets the instance of the logger. @@ -47,15 +47,19 @@ class ActionReader(DefinitionInitConfigReader): DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("action.d", fileName)) + + def getFile(self): + return self.__file + def setName(self, name): self._name = name def getName(self): return self._name - def read(self): - return ConfigReader.read(self, os.path.join("action.d", self._file)) - def convert(self): head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 80b99517..d819281b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -62,6 +62,7 @@ else: # pragma: no cover # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = 7 __all__ = ['SafeConfigParserWithIncludes'] @@ -98,30 +99,73 @@ after = 1.conf if sys.version_info >= (3,2): # overload constructor only for fancy new Python3's - def __init__(self, *args, **kwargs): + def __init__(self, share_config=None, *args, **kwargs): kwargs = kwargs.copy() kwargs['interpolation'] = BasicInterpolationWithName() kwargs['inline_comment_prefixes'] = ";" super(SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) + self._cfg_share = share_config - #@staticmethod - def getIncludes(resource, seen = []): + else: + def __init__(self, share_config=None, *args, **kwargs): + SafeConfigParser.__init__(self, *args, **kwargs) + self._cfg_share = share_config + + @property + def share_config(self): + return self._cfg_share + + def _getSharedSCPWI(self, filename): + SCPWI = SafeConfigParserWithIncludes + # read single one, add to return list, use sharing if possible: + if self._cfg_share: + # cache/share each file as include (ex: filter.d/common could be included in each filter config): + hashv = 'inc:'+(filename if not isinstance(filename, list) else '\x01'.join(filename)) + cfg, i = self._cfg_share.get(hashv, (None, None)) + if cfg is None: + cfg = SCPWI(share_config=self._cfg_share) + i = cfg.read(filename, get_includes=False) + self._cfg_share[hashv] = (cfg, i) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " Shared file: %s", filename) + else: + # don't have sharing: + cfg = SCPWI() + i = cfg.read(filename, get_includes=False) + return (cfg, i) + + def _getIncludes(self, filenames, seen=[]): + if not isinstance(filenames, list): + filenames = [ filenames ] + # retrieve or cache include paths: + if self._cfg_share: + # cache/share include list: + hashv = 'inc-path:'+('\x01'.join(filenames)) + fileNamesFull = self._cfg_share.get(hashv) + if fileNamesFull is None: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + self._cfg_share[hashv] = fileNamesFull + return fileNamesFull + # don't have sharing: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + return fileNamesFull + + def __getIncludesUncached(self, resource, seen=[]): """ Given 1 config resource returns list of included files (recursively) with the original one as well Simple loops are taken care about """ - - # Use a short class name ;) SCPWI = SafeConfigParserWithIncludes - - parser = SafeConfigParser() try: - if sys.version_info >= (3,2): # pragma: no cover - parser.read(resource, encoding='utf-8') - else: - parser.read(resource) + parser, i = self._getSharedSCPWI(resource) + if not i: + return [] except UnicodeDecodeError, e: logSys.error("Error decoding config file '%s': %s" % (resource, e)) return [] @@ -141,22 +185,60 @@ after = 1.conf if r in seen: continue s = seen + [resource] - option_list += SCPWI.getIncludes(r, s) + option_list += self._getIncludes(r, s) # combine lists return newFiles[0][1] + [resource] + newFiles[1][1] - #print "Includes list for " + resource + " is " + `resources` - getIncludes = staticmethod(getIncludes) + def get_defaults(self): + return self._defaults - def read(self, filenames): - fileNamesFull = [] + def get_sections(self): + return self._sections + + def read(self, filenames, get_includes=True): if not isinstance(filenames, list): filenames = [ filenames ] - for filename in filenames: - fileNamesFull += SafeConfigParserWithIncludes.getIncludes(filename) - logSys.debug("Reading files: %s" % fileNamesFull) + # retrieve (and cache) includes: + fileNamesFull = [] + if get_includes: + fileNamesFull += self._getIncludes(filenames) + else: + fileNamesFull = filenames + + if not fileNamesFull: + return [] + + logSys.info(" Loading files: %s", fileNamesFull) + + if get_includes or len(fileNamesFull) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in fileNamesFull: + # read single one, add to return list, use sharing if possible: + cfg, i = self._getSharedSCPWI(filename) + if i: + ret += i + # merge defaults and all sections to self: + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): + if isinstance(s, dict): + s2 = alls.get(n) + if isinstance(s2, dict): + s2.update(s) + else: + alls[n] = s.copy() + else: + alls[n] = s + + return ret + + # read one config : + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " Reading file: %s", fileNamesFull[0]) + # read file(s) : if sys.version_info >= (3,2): # pragma: no cover return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') else: return SafeConfigParser.read(self, fileNamesFull) - diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 22115d3a..82bf0fdc 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -27,24 +27,127 @@ __license__ = "GPL" import glob, os from ConfigParser import NoOptionError, NoSectionError -from .configparserinc import SafeConfigParserWithIncludes +from .configparserinc import SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class ConfigReader(SafeConfigParserWithIncludes): +class ConfigReader(): + """Generic config reader class. + + A caching adapter which automatically reuses already shared configuration. + """ + + def __init__(self, use_config=None, share_config=None, **kwargs): + # use given shared config if possible (see read): + self._cfg_share = None + self._cfg = None + if use_config is not None: + self._cfg = use_config + # share config if possible: + if share_config is not None: + self._cfg_share = share_config + self._cfg_share_kwargs = kwargs + self._cfg_share_basedir = None + elif self._cfg is None: + self._cfg = ConfigReaderUnshared(**kwargs) + + def setBaseDir(self, basedir): + if self._cfg: + self._cfg.setBaseDir(basedir) + else: + self._cfg_share_basedir = basedir + + def getBaseDir(self): + if self._cfg: + return self._cfg.getBaseDir() + else: + return self._cfg_share_basedir + + @property + def share_config(self): + return self._cfg_share + + def read(self, name, once=True): + """ Overloads a default (not shared) read of config reader. + + To prevent mutiple reads of config files with it includes, reads into + the config reader, if it was not yet cached/shared by 'name'. + """ + # already shared ? + if not self._cfg: + self.touch(name) + # performance feature - read once if using shared config reader: + if once and self._cfg.read_cfg_files is not None: + return self._cfg.read_cfg_files + + # load: + logSys.info("Loading configs for %s under %s ", name, self._cfg.getBaseDir()) + ret = self._cfg.read(name) + + # save already read and return: + self._cfg.read_cfg_files = ret + return ret + + def touch(self, name=''): + """ Allocates and share a config file by it name. + + Automatically allocates unshared or reuses shared handle by given 'name' and + init arguments inside a given shared storage. + """ + if not self._cfg and self._cfg_share is not None: + self._cfg = self._cfg_share.get(name) + if not self._cfg: + self._cfg = ConfigReaderUnshared(share_config=self._cfg_share, **self._cfg_share_kwargs) + if self._cfg_share_basedir is not None: + self._cfg.setBaseDir(self._cfg_share_basedir) + self._cfg_share[name] = self._cfg + else: + self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) + + def sections(self): + if self._cfg is not None: + return self._cfg.sections() + return [] + + def has_section(self, sec): + if self._cfg is not None: + return self._cfg.has_section(sec) + return False + + def options(self, *args): + if self._cfg is not None: + return self._cfg.options(*args) + return {} + + def get(self, sec, opt): + if self._cfg is not None: + return self._cfg.get(sec, opt) + return None + + def getOptions(self, *args, **kwargs): + if self._cfg is not None: + return self._cfg.getOptions(*args, **kwargs) + return {} + +class ConfigReaderUnshared(SafeConfigParserWithIncludes): + """Unshared config reader (previously ConfigReader). + + Do not use this class (internal not shared/cached represenation). + Use ConfigReader instead. + """ DEFAULT_BASEDIR = '/etc/fail2ban' - def __init__(self, basedir=None): - SafeConfigParserWithIncludes.__init__(self) + def __init__(self, basedir=None, *args, **kwargs): + SafeConfigParserWithIncludes.__init__(self, *args, **kwargs) + self.read_cfg_files = None self.setBaseDir(basedir) - self.__opts = None def setBaseDir(self, basedir): if basedir is None: - basedir = ConfigReader.DEFAULT_BASEDIR # stock system location + basedir = ConfigReaderUnshared.DEFAULT_BASEDIR # stock system location self._basedir = basedir.rstrip('/') def getBaseDir(self): @@ -55,7 +158,7 @@ class ConfigReader(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.info("Reading configs for %s under %s " % (basename, self._basedir)) + logSys.debug("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -71,14 +174,14 @@ class ConfigReader(SafeConfigParserWithIncludes): if len(config_files): # at least one config exists and accessible - logSys.debug("Reading config files: " + ', '.join(config_files)) + logSys.debug("Reading config files: %s", ', '.join(config_files)) config_files_read = SafeConfigParserWithIncludes.read(self, config_files) missed = [ cf for cf in config_files if cf not in config_files_read ] if missed: - logSys.error("Could not read config files: " + ', '.join(missed)) + logSys.error("Could not read config files: %s", ', '.join(missed)) if config_files_read: return True - logSys.error("Found no accessible config files for %r under %s" % + logSys.error("Found no accessible config files for %r under %s", ( filename, self.getBaseDir() )) return False else: @@ -98,7 +201,7 @@ class ConfigReader(SafeConfigParserWithIncludes): # 1 -> the name of the option # 2 -> the default value for the option - def getOptions(self, sec, options, pOptions = None): + def getOptions(self, sec, options, pOptions=None): values = dict() for option in options: try: @@ -121,10 +224,8 @@ class ConfigReader(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] - else: - logSys.debug( - "Non essential option '%s' not defined in '%s'.", - option[1], sec) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") @@ -133,12 +234,12 @@ class ConfigReader(SafeConfigParserWithIncludes): class DefinitionInitConfigReader(ConfigReader): """Config reader for files with options grouped in [Definition] and - [Init] sections. + [Init] sections. - Is a base class for readers of filters and actions, where definitions - in jails might provide custom values for options defined in [Init] - section. - """ + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 0dd9f955..8667501c 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,12 +33,20 @@ logSys = getLogger(__name__) class Configurator: - def __init__(self): + def __init__(self, force_enable=False, share_config=None): self.__settings = dict() self.__streams = dict() - self.__fail2ban = Fail2banReader() - self.__jails = JailsReader() - + # always share all config readers: + if share_config is None: + share_config = dict() + self.__share_config = share_config + self.__fail2ban = Fail2banReader(share_config=share_config) + self.__jails = JailsReader(force_enable=force_enable, share_config=share_config) + + def Reload(self): + # clear all shared handlers: + self.__share_config.clear() + def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) self.__jails.setBaseDir(folderName) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 40d74c45..fe657025 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os, shlex -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..server.action import CommandAction from ..helpers import getLogger @@ -40,8 +40,12 @@ class FilterReader(DefinitionInitConfigReader): ["string", "failregex", ""], ] - def read(self): - return ConfigReader.read(self, os.path.join("filter.d", self._file)) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("filter.d", fileName)) + + def getFile(self): + return self.__file def convert(self): stream = list() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index a4bf5174..ffdc5e26 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import re, glob, os.path import json -from .configreader import ConfigReader +from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger @@ -111,7 +111,7 @@ class JailReader(ConfigReader): filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( - filterName, self.__name, filterOpt, basedir=self.getBaseDir()) + filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -141,7 +141,7 @@ class JailReader(ConfigReader): else: action = ActionReader( actName, self.__name, actOpt, - basedir=self.getBaseDir()) + share_config=self.share_config, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -213,7 +213,7 @@ class JailReader(ConfigReader): if self.__filter: stream.extend(self.__filter.convert()) for action in self.__actions: - if isinstance(action, ConfigReader): + if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) else: stream.append(action) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 84c614b9..40255ce7 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -68,9 +68,10 @@ class JailsReader(ConfigReader): for sec in sections: if sec == 'INCLUDES': continue - jail = JailReader(sec, basedir=self.getBaseDir(), - force_enable=self.__force_enable) - jail.read() + # use the cfg_share for filter/action caching and the same config for all + # jails (use_config=...), therefore don't read it here: + jail = JailReader(sec, force_enable=self.__force_enable, + share_config=self.share_config, use_config=self._cfg) ret = jail.getOptions() if ret: if jail.isEnabled(): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0ad3c66e..e4dc2189 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,9 +21,9 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest - -from ..client.configreader import ConfigReader +import os, glob, shutil, tempfile, unittest, re, logging +from ..client.configreader import ConfigReaderUnshared +from ..client import configparserinc from ..client.jailreader import JailReader from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader @@ -44,7 +44,7 @@ class ConfigReaderTest(unittest.TestCase): def setUp(self): """Call before every test case.""" self.d = tempfile.mkdtemp(prefix="f2b-temp") - self.c = ConfigReader(basedir=self.d) + self.c = ConfigReaderUnshared(basedir=self.d) def tearDown(self): """Call after every test case.""" @@ -335,6 +335,64 @@ class FilterReaderTest(unittest.TestCase): self.assertRaises(ValueError, FilterReader.convert, filterReader) +class JailsReaderTestCache(LogCaptureTestCase): + + def _readWholeConf(self, basedir, force_enable=False, share_config=None): + # read whole configuration like a file2ban-client ... + configurator = Configurator(force_enable=force_enable, share_config=share_config) + configurator.setBaseDir(basedir) + configurator.readEarly() + configurator.getEarlyOptions() + configurator.readAll() + # from here we test a cache with all includes / before / after : + self.assertTrue(configurator.getOptions(None)) + + def _getLoggedReadCount(self, filematch): + cnt = 0 + for s in self.getLog().rsplit('\n'): + if re.match(r"^\s*Reading files?: .*/"+filematch, s): + cnt += 1 + return cnt + + def testTestJailConfCache(self): + saved_ll = configparserinc.logLevel + configparserinc.logLevel = logging.DEBUG + basedir = tempfile.mkdtemp("fail2ban_conf") + try: + shutil.rmtree(basedir) + shutil.copytree(CONFIG_DIR, basedir) + shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local') + shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') + + # common sharing handle for this test: + share_cfg = dict() + + # read whole configuration like a file2ban-client ... + self._readWholeConf(basedir, share_config=share_cfg) + # how many times jail.local was read: + cnt = self._getLoggedReadCount('jail.local') + # if cnt > 1: + # self.printLog() + self.assertTrue(cnt == 1, "Unexpected count by reading of jail files, cnt = %s" % cnt) + + # read whole configuration like a file2ban-client, again ... + # but this time force enable all jails, to check filter and action cached also: + self._readWholeConf(basedir, force_enable=True, share_config=share_cfg) + cnt = self._getLoggedReadCount(r'jail\.local') + # still one (no more reads): + self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt) + + # same with filter: + cnt = self._getLoggedReadCount(r'filter\.d/common\.conf') + self.assertTrue(cnt == 1, "Unexpected count by reading of filter files, cnt = %s" % cnt) + # same with action: + cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') + self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) + finally: + shutil.rmtree(basedir) + configparserinc.logLevel = saved_ll + + class JailsReaderTest(LogCaptureTestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 912c5a90..b56c0988 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -25,6 +25,7 @@ __license__ = "GPL" import logging import os import re +import sys import time import unittest from StringIO import StringIO @@ -111,6 +112,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTestCache)) # CSocket and AsyncServer tests.addTest(unittest.makeSuite(sockettestcase.Socket)) # Misc helpers @@ -204,6 +206,8 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] + if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! + logSys.handlers += self._old_handlers logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): @@ -216,5 +220,8 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() + def getLog(self): + return self._log.getvalue() + def printLog(self): print(self._log.getvalue())