diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 91f3322b..8022ecc1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..helpers import getLogger # Gets the instance of the logger. @@ -47,15 +47,19 @@ class ActionReader(DefinitionInitConfigReader): DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("action.d", fileName)) + + def getFile(self): + return self.__file + def setName(self, name): self._name = name def getName(self): return self._name - def read(self): - return ConfigReader.read(self, os.path.join("action.d", self._file)) - def convert(self): head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 80b99517..31945849 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -65,7 +65,136 @@ logSys = getLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] -class SafeConfigParserWithIncludes(SafeConfigParser): +class SafeConfigParserWithIncludes(object): + + SECTION_NAME = "INCLUDES" + CFG_CACHE = {} + CFG_INC_CACHE = {} + CFG_EMPY_CFG = None + + def __init__(self): + self.__cr = None + + def __check_read(self, attr): + if self.__cr is None: + # raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr) + if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None: + SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes() + self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG + + def __getattr__(self,attr): + # check we access local implementation + try: + orig_attr = self.__getattribute__(attr) + except AttributeError: + self.__check_read(attr) + orig_attr = self.__cr.__getattribute__(attr) + return orig_attr + + @staticmethod + def _resource_mtime(resource): + mt = [] + dirnames = [] + for filename in resource: + if os.path.exists(filename): + s = os.stat(filename) + mt.append(s.st_mtime) + mt.append(s.st_mode) + mt.append(s.st_size) + dirname = os.path.dirname(filename) + if dirname not in dirnames: + dirnames.append(dirname) + for dirname in dirnames: + if os.path.exists(dirname): + s = os.stat(dirname) + mt.append(s.st_mtime) + mt.append(s.st_mode) + mt.append(s.st_size) + return mt + + def read(self, resource, get_includes=True, log_info=None): + SCPWI = SafeConfigParserWithIncludes + # check includes : + fileNamesFull = [] + if not isinstance(resource, list): + resource = [ resource ] + if get_includes: + for filename in resource: + fileNamesFull += SCPWI.getIncludes(filename) + else: + fileNamesFull = resource + # check cache + hashv = '\x01'.join(fileNamesFull) + cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) + curmt = SCPWI._resource_mtime(fileNamesFull) + if cr is not None and mtime == curmt: + self.__cr = cr + logSys.debug("Cached config files: %s", resource) + #logSys.debug("Cached config files: %s", fileNamesFull) + return ret + # not yet in cache - create/read and add to cache: + if log_info is not None: + logSys.info(*log_info) + cr = _SafeConfigParserWithIncludes() + ret = cr.read(fileNamesFull) + SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt) + self.__cr = cr + return ret + + def getOptions(self, *args, **kwargs): + self.__check_read('getOptions') + return self.__cr.getOptions(*args, **kwargs) + + @staticmethod + def getIncludes(resource, seen = []): + """ + Given 1 config resource returns list of included files + (recursively) with the original one as well + Simple loops are taken care about + """ + + # Use a short class name ;) + SCPWI = SafeConfigParserWithIncludes + + resources = seen + [resource] + # check cache + hashv = '///'.join(resources) + cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0)) + curmt = SCPWI._resource_mtime(resources) + if cinc is not None and mtime == curmt: + return cinc + + parser = SCPWI() + try: + # read without includes + parser.read(resource, get_includes=False) + except UnicodeDecodeError, e: + logSys.error("Error decoding config file '%s': %s" % (resource, e)) + return [] + + resourceDir = os.path.dirname(resource) + + newFiles = [ ('before', []), ('after', []) ] + if SCPWI.SECTION_NAME in parser.sections(): + for option_name, option_list in newFiles: + if option_name in parser.options(SCPWI.SECTION_NAME): + newResources = parser.get(SCPWI.SECTION_NAME, option_name) + for newResource in newResources.split('\n'): + if os.path.isabs(newResource): + r = newResource + else: + r = os.path.join(resourceDir, newResource) + if r in seen: + continue + option_list += SCPWI.getIncludes(r, resources) + # combine lists + cinc = newFiles[0][1] + [resource] + newFiles[1][1] + # cache and return : + SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt) + return cinc + #print "Includes list for " + resource + " is " + `resources` + +class _SafeConfigParserWithIncludes(SafeConfigParser, object): """ Class adds functionality to SafeConfigParser to handle included other configuration files (or may be urls, whatever in the future) @@ -94,69 +223,53 @@ after = 1.conf """ - SECTION_NAME = "INCLUDES" - if sys.version_info >= (3,2): # overload constructor only for fancy new Python3's def __init__(self, *args, **kwargs): kwargs = kwargs.copy() kwargs['interpolation'] = BasicInterpolationWithName() kwargs['inline_comment_prefixes'] = ";" - super(SafeConfigParserWithIncludes, self).__init__( + super(_SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) - #@staticmethod - def getIncludes(resource, seen = []): - """ - Given 1 config resource returns list of included files - (recursively) with the original one as well - Simple loops are taken care about - """ - - # Use a short class name ;) - SCPWI = SafeConfigParserWithIncludes - - parser = SafeConfigParser() - try: - if sys.version_info >= (3,2): # pragma: no cover - parser.read(resource, encoding='utf-8') - else: - parser.read(resource) - except UnicodeDecodeError, e: - logSys.error("Error decoding config file '%s': %s" % (resource, e)) - return [] - - resourceDir = os.path.dirname(resource) - - newFiles = [ ('before', []), ('after', []) ] - if SCPWI.SECTION_NAME in parser.sections(): - for option_name, option_list in newFiles: - if option_name in parser.options(SCPWI.SECTION_NAME): - newResources = parser.get(SCPWI.SECTION_NAME, option_name) - for newResource in newResources.split('\n'): - if os.path.isabs(newResource): - r = newResource - else: - r = os.path.join(resourceDir, newResource) - if r in seen: - continue - s = seen + [resource] - option_list += SCPWI.getIncludes(r, s) - # combine lists - return newFiles[0][1] + [resource] + newFiles[1][1] - #print "Includes list for " + resource + " is " + `resources` - getIncludes = staticmethod(getIncludes) + def get_defaults(self): + return self._defaults + def get_sections(self): + return self._sections def read(self, filenames): - fileNamesFull = [] if not isinstance(filenames, list): filenames = [ filenames ] - for filename in filenames: - fileNamesFull += SafeConfigParserWithIncludes.getIncludes(filename) - logSys.debug("Reading files: %s" % fileNamesFull) - if sys.version_info >= (3,2): # pragma: no cover - return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') - else: - return SafeConfigParser.read(self, fileNamesFull) + if len(filenames) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in filenames: + # read single one, add to return list: + cfg = SafeConfigParserWithIncludes() + i = cfg.read(filename, get_includes=False) + if i: + ret += i + # merge defaults and all sections to self: + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): + if isinstance(s, dict): + s2 = alls.get(n) + if isinstance(s2, dict): + s2.update(s) + else: + alls[n] = s.copy() + else: + alls[n] = s + + return ret + + # read one config : + logSys.debug("Reading file: %s", filenames[0]) + if sys.version_info >= (3,2): # pragma: no cover + return SafeConfigParser.read(self, filenames, encoding='utf-8') + else: + return SafeConfigParser.read(self, filenames) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 22115d3a..d1cc7924 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -32,6 +32,65 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = 6 + +class ConfigWrapper(): + + def __init__(self, use_config=None, share_config=None, **kwargs): + # use given shared config if possible (see read): + self._cfg_share = None + self._cfg = None + if use_config is not None: + self._cfg = use_config + else: + # share config if possible: + if share_config is not None: + self._cfg_share = share_config + self._cfg_share_kwargs = kwargs + else: + self._cfg = ConfigReader(**kwargs) + + def setBaseDir(self, basedir): + self._cfg.setBaseDir(basedir) + + def getBaseDir(self): + return self._cfg.getBaseDir() + + def read(self, name, once=True): + # shared ? + if not self._cfg and self._cfg_share is not None: + self._cfg = self._cfg_share.get(name) + if not self._cfg: + self._cfg = ConfigReader(**self._cfg_share_kwargs) + self._cfg_share[name] = self._cfg + # performance feature - read once if using shared config reader: + rc = self._cfg.read_cfg_files + if once and rc.get(name) is not None: + return rc.get(name) + + # read: + ret = self._cfg.read(name) + + # save already read: + if once: + rc[name] = ret + return ret + + def sections(self): + return self._cfg.sections() + + def has_section(self, sec): + return self._cfg.has_section(sec) + + def options(self, *args): + return self._cfg.options(*args) + + def get(self, sec, opt): + return self._cfg.get(sec, opt) + + def getOptions(self, *args, **kwargs): + return self._cfg.getOptions(*args, **kwargs) + class ConfigReader(SafeConfigParserWithIncludes): @@ -39,8 +98,8 @@ class ConfigReader(SafeConfigParserWithIncludes): def __init__(self, basedir=None): SafeConfigParserWithIncludes.__init__(self) + self.read_cfg_files = dict() self.setBaseDir(basedir) - self.__opts = None def setBaseDir(self, basedir): if basedir is None: @@ -55,7 +114,7 @@ class ConfigReader(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.info("Reading configs for %s under %s " % (basename, self._basedir)) + logSys.debug("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -71,14 +130,15 @@ class ConfigReader(SafeConfigParserWithIncludes): if len(config_files): # at least one config exists and accessible - logSys.debug("Reading config files: " + ', '.join(config_files)) - config_files_read = SafeConfigParserWithIncludes.read(self, config_files) + logSys.debug("Reading config files: %s", ', '.join(config_files)) + config_files_read = SafeConfigParserWithIncludes.read(self, config_files, + log_info=("Cache configs for %s under %s " , filename, self._basedir)) missed = [ cf for cf in config_files if cf not in config_files_read ] if missed: - logSys.error("Could not read config files: " + ', '.join(missed)) + logSys.error("Could not read config files: %s", ', '.join(missed)) if config_files_read: return True - logSys.error("Found no accessible config files for %r under %s" % + logSys.error("Found no accessible config files for %r under %s", ( filename, self.getBaseDir() )) return False else: @@ -121,29 +181,27 @@ class ConfigReader(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] - else: - logSys.debug( - "Non essential option '%s' not defined in '%s'.", - option[1], sec) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values -class DefinitionInitConfigReader(ConfigReader): +class DefinitionInitConfigReader(ConfigWrapper): """Config reader for files with options grouped in [Definition] and - [Init] sections. + [Init] sections. - Is a base class for readers of filters and actions, where definitions - in jails might provide custom values for options defined in [Init] - section. - """ + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] def __init__(self, file_, jailName, initOpts, **kwargs): - ConfigReader.__init__(self, **kwargs) + ConfigWrapper.__init__(self, **kwargs) self.setFile(file_) self.setJailName(jailName) self._initOpts = initOpts @@ -162,14 +220,14 @@ class DefinitionInitConfigReader(ConfigReader): return self._jailName def read(self): - return ConfigReader.read(self, self._file) + return ConfigWrapper.read(self, self._file) # needed for fail2ban-regex that doesn't need fancy directories def readexplicit(self): return SafeConfigParserWithIncludes.read(self, self._file) def getOptions(self, pOpts): - self._opts = ConfigReader.getOptions( + self._opts = ConfigWrapper.getOptions( self, "Definition", self._configOpts, pOpts) if self.has_section("Init"): diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 361a5e54..a151ebf0 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,31 +24,32 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigWrapper from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class Fail2banReader(ConfigReader): +class Fail2banReader(ConfigWrapper): def __init__(self, **kwargs): - ConfigReader.__init__(self, **kwargs) + self.__opts = None + ConfigWrapper.__init__(self, **kwargs) def read(self): - ConfigReader.read(self, "fail2ban") + ConfigWrapper.read(self, "fail2ban") def getEarlyOptions(self): opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"], ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] - return ConfigReader.getOptions(self, "Definition", opts) + return ConfigWrapper.getOptions(self, "Definition", opts) def getOptions(self): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbpurgeage", 86400]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts) + self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) def convert(self): stream = list() diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 40d74c45..fe657025 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os, shlex -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..server.action import CommandAction from ..helpers import getLogger @@ -40,8 +40,12 @@ class FilterReader(DefinitionInitConfigReader): ["string", "failregex", ""], ] - def read(self): - return ConfigReader.read(self, os.path.join("filter.d", self._file)) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("filter.d", fileName)) + + def getFile(self): + return self.__file def convert(self): stream = list() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index a4bf5174..a3cc939c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import re, glob, os.path import json -from .configreader import ConfigReader +from .configreader import ConfigReader, ConfigWrapper from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger @@ -35,17 +35,19 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailReader(ConfigReader): +class JailReader(ConfigWrapper): optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") optionExtractRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') - def __init__(self, name, force_enable=False, **kwargs): - ConfigReader.__init__(self, **kwargs) + def __init__(self, name, force_enable=False, cfg_share=None, **kwargs): + # use shared config if possible: + ConfigWrapper.__init__(self, **kwargs) self.__name = name self.__filter = None self.__force_enable = force_enable + self.__cfg_share = cfg_share self.__actions = list() self.__opts = None @@ -60,7 +62,7 @@ class JailReader(ConfigReader): return self.__name def read(self): - out = ConfigReader.read(self, "jail") + out = ConfigWrapper.read(self, "jail") # Before returning -- verify that requested section # exists at all if not (self.__name in self.sections()): @@ -101,7 +103,7 @@ class JailReader(ConfigReader): ["string", "ignoreip", None], ["string", "filter", ""], ["string", "action", ""]] - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + self.__opts = ConfigWrapper.getOptions(self, self.__name, opts) if not self.__opts: return False @@ -111,7 +113,7 @@ class JailReader(ConfigReader): filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( - filterName, self.__name, filterOpt, basedir=self.getBaseDir()) + filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -141,7 +143,7 @@ class JailReader(ConfigReader): else: action = ActionReader( actName, self.__name, actOpt, - basedir=self.getBaseDir()) + share_config=self.__cfg_share, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -213,7 +215,7 @@ class JailReader(ConfigReader): if self.__filter: stream.extend(self.__filter.convert()) for action in self.__actions: - if isinstance(action, ConfigReader): + if isinstance(action, (ConfigReader, ConfigWrapper)): stream.extend(action.convert()) else: stream.append(action) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 84c614b9..37d37e01 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigReader, ConfigWrapper from .jailreader import JailReader from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailsReader(ConfigReader): +class JailsReader(ConfigWrapper): def __init__(self, force_enable=False, **kwargs): """ @@ -41,7 +41,9 @@ class JailsReader(ConfigReader): Passed to JailReader to force enable the jails. It is for internal use """ - ConfigReader.__init__(self, **kwargs) + # use shared config if possible: + ConfigWrapper.__init__(self, **kwargs) + self.__cfg_share = dict() self.__jails = list() self.__force_enable = force_enable @@ -50,13 +52,13 @@ class JailsReader(ConfigReader): return self.__jails def read(self): - return ConfigReader.read(self, "jail") + return ConfigWrapper.read(self, "jail") def getOptions(self, section=None): """Reads configuration for jail(s) and adds enabled jails to __jails """ opts = [] - self.__opts = ConfigReader.getOptions(self, "Definition", opts) + self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) if section is None: sections = self.sections() @@ -68,9 +70,10 @@ class JailsReader(ConfigReader): for sec in sections: if sec == 'INCLUDES': continue - jail = JailReader(sec, basedir=self.getBaseDir(), - force_enable=self.__force_enable) - jail.read() + # use the cfg_share for filter/action caching and the same config for all + # jails (use_config=...), therefore don't read it here: + jail = JailReader(sec, force_enable=self.__force_enable, + cfg_share=self.__cfg_share, use_config=self._cfg) ret = jail.getOptions() if ret: if jail.isEnabled():