Merge branch 'cache-config-read-820' into _tent/cache-config-read

pull/824/head
sebres 2014-10-09 18:01:31 +02:00
commit f6723a12ff
7 changed files with 287 additions and 102 deletions

View File

@ -26,7 +26,7 @@ __license__ = "GPL"
import os
from .configreader import ConfigReader, DefinitionInitConfigReader
from .configreader import DefinitionInitConfigReader
from ..helpers import getLogger
# Gets the instance of the logger.
@ -47,15 +47,19 @@ class ActionReader(DefinitionInitConfigReader):
DefinitionInitConfigReader.__init__(
self, file_, jailName, initOpts, **kwargs)
def setFile(self, fileName):
self.__file = fileName
DefinitionInitConfigReader.setFile(self, os.path.join("action.d", fileName))
def getFile(self):
return self.__file
def setName(self, name):
self._name = name
def getName(self):
return self._name
def read(self):
return ConfigReader.read(self, os.path.join("action.d", self._file))
def convert(self):
head = ["set", self._jailName]
stream = list()

View File

@ -65,7 +65,136 @@ logSys = getLogger(__name__)
__all__ = ['SafeConfigParserWithIncludes']
class SafeConfigParserWithIncludes(SafeConfigParser):
class SafeConfigParserWithIncludes(object):
SECTION_NAME = "INCLUDES"
CFG_CACHE = {}
CFG_INC_CACHE = {}
CFG_EMPY_CFG = None
def __init__(self):
self.__cr = None
def __check_read(self, attr):
if self.__cr is None:
# raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr)
if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None:
SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes()
self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG
def __getattr__(self,attr):
# check we access local implementation
try:
orig_attr = self.__getattribute__(attr)
except AttributeError:
self.__check_read(attr)
orig_attr = self.__cr.__getattribute__(attr)
return orig_attr
@staticmethod
def _resource_mtime(resource):
mt = []
dirnames = []
for filename in resource:
if os.path.exists(filename):
s = os.stat(filename)
mt.append(s.st_mtime)
mt.append(s.st_mode)
mt.append(s.st_size)
dirname = os.path.dirname(filename)
if dirname not in dirnames:
dirnames.append(dirname)
for dirname in dirnames:
if os.path.exists(dirname):
s = os.stat(dirname)
mt.append(s.st_mtime)
mt.append(s.st_mode)
mt.append(s.st_size)
return mt
def read(self, resource, get_includes=True, log_info=None):
SCPWI = SafeConfigParserWithIncludes
# check includes :
fileNamesFull = []
if not isinstance(resource, list):
resource = [ resource ]
if get_includes:
for filename in resource:
fileNamesFull += SCPWI.getIncludes(filename)
else:
fileNamesFull = resource
# check cache
hashv = '\x01'.join(fileNamesFull)
cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0))
curmt = SCPWI._resource_mtime(fileNamesFull)
if cr is not None and mtime == curmt:
self.__cr = cr
logSys.debug("Cached config files: %s", resource)
#logSys.debug("Cached config files: %s", fileNamesFull)
return ret
# not yet in cache - create/read and add to cache:
if log_info is not None:
logSys.info(*log_info)
cr = _SafeConfigParserWithIncludes()
ret = cr.read(fileNamesFull)
SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt)
self.__cr = cr
return ret
def getOptions(self, *args, **kwargs):
self.__check_read('getOptions')
return self.__cr.getOptions(*args, **kwargs)
@staticmethod
def getIncludes(resource, seen = []):
"""
Given 1 config resource returns list of included files
(recursively) with the original one as well
Simple loops are taken care about
"""
# Use a short class name ;)
SCPWI = SafeConfigParserWithIncludes
resources = seen + [resource]
# check cache
hashv = '///'.join(resources)
cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0))
curmt = SCPWI._resource_mtime(resources)
if cinc is not None and mtime == curmt:
return cinc
parser = SCPWI()
try:
# read without includes
parser.read(resource, get_includes=False)
except UnicodeDecodeError, e:
logSys.error("Error decoding config file '%s': %s" % (resource, e))
return []
resourceDir = os.path.dirname(resource)
newFiles = [ ('before', []), ('after', []) ]
if SCPWI.SECTION_NAME in parser.sections():
for option_name, option_list in newFiles:
if option_name in parser.options(SCPWI.SECTION_NAME):
newResources = parser.get(SCPWI.SECTION_NAME, option_name)
for newResource in newResources.split('\n'):
if os.path.isabs(newResource):
r = newResource
else:
r = os.path.join(resourceDir, newResource)
if r in seen:
continue
option_list += SCPWI.getIncludes(r, resources)
# combine lists
cinc = newFiles[0][1] + [resource] + newFiles[1][1]
# cache and return :
SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt)
return cinc
#print "Includes list for " + resource + " is " + `resources`
class _SafeConfigParserWithIncludes(SafeConfigParser, object):
"""
Class adds functionality to SafeConfigParser to handle included
other configuration files (or may be urls, whatever in the future)
@ -94,69 +223,53 @@ after = 1.conf
"""
SECTION_NAME = "INCLUDES"
if sys.version_info >= (3,2):
# overload constructor only for fancy new Python3's
def __init__(self, *args, **kwargs):
kwargs = kwargs.copy()
kwargs['interpolation'] = BasicInterpolationWithName()
kwargs['inline_comment_prefixes'] = ";"
super(SafeConfigParserWithIncludes, self).__init__(
super(_SafeConfigParserWithIncludes, self).__init__(
*args, **kwargs)
#@staticmethod
def getIncludes(resource, seen = []):
"""
Given 1 config resource returns list of included files
(recursively) with the original one as well
Simple loops are taken care about
"""
# Use a short class name ;)
SCPWI = SafeConfigParserWithIncludes
parser = SafeConfigParser()
try:
if sys.version_info >= (3,2): # pragma: no cover
parser.read(resource, encoding='utf-8')
else:
parser.read(resource)
except UnicodeDecodeError, e:
logSys.error("Error decoding config file '%s': %s" % (resource, e))
return []
resourceDir = os.path.dirname(resource)
newFiles = [ ('before', []), ('after', []) ]
if SCPWI.SECTION_NAME in parser.sections():
for option_name, option_list in newFiles:
if option_name in parser.options(SCPWI.SECTION_NAME):
newResources = parser.get(SCPWI.SECTION_NAME, option_name)
for newResource in newResources.split('\n'):
if os.path.isabs(newResource):
r = newResource
else:
r = os.path.join(resourceDir, newResource)
if r in seen:
continue
s = seen + [resource]
option_list += SCPWI.getIncludes(r, s)
# combine lists
return newFiles[0][1] + [resource] + newFiles[1][1]
#print "Includes list for " + resource + " is " + `resources`
getIncludes = staticmethod(getIncludes)
def get_defaults(self):
return self._defaults
def get_sections(self):
return self._sections
def read(self, filenames):
fileNamesFull = []
if not isinstance(filenames, list):
filenames = [ filenames ]
for filename in filenames:
fileNamesFull += SafeConfigParserWithIncludes.getIncludes(filename)
logSys.debug("Reading files: %s" % fileNamesFull)
if sys.version_info >= (3,2): # pragma: no cover
return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8')
else:
return SafeConfigParser.read(self, fileNamesFull)
if len(filenames) > 1:
# read multiple configs:
ret = []
alld = self.get_defaults()
alls = self.get_sections()
for filename in filenames:
# read single one, add to return list:
cfg = SafeConfigParserWithIncludes()
i = cfg.read(filename, get_includes=False)
if i:
ret += i
# merge defaults and all sections to self:
alld.update(cfg.get_defaults())
for n, s in cfg.get_sections().iteritems():
if isinstance(s, dict):
s2 = alls.get(n)
if isinstance(s2, dict):
s2.update(s)
else:
alls[n] = s.copy()
else:
alls[n] = s
return ret
# read one config :
logSys.debug("Reading file: %s", filenames[0])
if sys.version_info >= (3,2): # pragma: no cover
return SafeConfigParser.read(self, filenames, encoding='utf-8')
else:
return SafeConfigParser.read(self, filenames)

View File

@ -32,6 +32,65 @@ from ..helpers import getLogger
# Gets the instance of the logger.
logSys = getLogger(__name__)
logLevel = 6
class ConfigWrapper():
def __init__(self, use_config=None, share_config=None, **kwargs):
# use given shared config if possible (see read):
self._cfg_share = None
self._cfg = None
if use_config is not None:
self._cfg = use_config
else:
# share config if possible:
if share_config is not None:
self._cfg_share = share_config
self._cfg_share_kwargs = kwargs
else:
self._cfg = ConfigReader(**kwargs)
def setBaseDir(self, basedir):
self._cfg.setBaseDir(basedir)
def getBaseDir(self):
return self._cfg.getBaseDir()
def read(self, name, once=True):
# shared ?
if not self._cfg and self._cfg_share is not None:
self._cfg = self._cfg_share.get(name)
if not self._cfg:
self._cfg = ConfigReader(**self._cfg_share_kwargs)
self._cfg_share[name] = self._cfg
# performance feature - read once if using shared config reader:
rc = self._cfg.read_cfg_files
if once and rc.get(name) is not None:
return rc.get(name)
# read:
ret = self._cfg.read(name)
# save already read:
if once:
rc[name] = ret
return ret
def sections(self):
return self._cfg.sections()
def has_section(self, sec):
return self._cfg.has_section(sec)
def options(self, *args):
return self._cfg.options(*args)
def get(self, sec, opt):
return self._cfg.get(sec, opt)
def getOptions(self, *args, **kwargs):
return self._cfg.getOptions(*args, **kwargs)
class ConfigReader(SafeConfigParserWithIncludes):
@ -39,8 +98,8 @@ class ConfigReader(SafeConfigParserWithIncludes):
def __init__(self, basedir=None):
SafeConfigParserWithIncludes.__init__(self)
self.read_cfg_files = dict()
self.setBaseDir(basedir)
self.__opts = None
def setBaseDir(self, basedir):
if basedir is None:
@ -55,7 +114,7 @@ class ConfigReader(SafeConfigParserWithIncludes):
raise ValueError("Base configuration directory %s does not exist "
% self._basedir)
basename = os.path.join(self._basedir, filename)
logSys.info("Reading configs for %s under %s " % (basename, self._basedir))
logSys.debug("Reading configs for %s under %s " , filename, self._basedir)
config_files = [ basename + ".conf" ]
# possible further customizations under a .conf.d directory
@ -71,14 +130,15 @@ class ConfigReader(SafeConfigParserWithIncludes):
if len(config_files):
# at least one config exists and accessible
logSys.debug("Reading config files: " + ', '.join(config_files))
config_files_read = SafeConfigParserWithIncludes.read(self, config_files)
logSys.debug("Reading config files: %s", ', '.join(config_files))
config_files_read = SafeConfigParserWithIncludes.read(self, config_files,
log_info=("Cache configs for %s under %s " , filename, self._basedir))
missed = [ cf for cf in config_files if cf not in config_files_read ]
if missed:
logSys.error("Could not read config files: " + ', '.join(missed))
logSys.error("Could not read config files: %s", ', '.join(missed))
if config_files_read:
return True
logSys.error("Found no accessible config files for %r under %s" %
logSys.error("Found no accessible config files for %r under %s",
( filename, self.getBaseDir() ))
return False
else:
@ -121,29 +181,27 @@ class ConfigReader(SafeConfigParserWithIncludes):
logSys.warning("'%s' not defined in '%s'. Using default one: %r"
% (option[1], sec, option[2]))
values[option[1]] = option[2]
else:
logSys.debug(
"Non essential option '%s' not defined in '%s'.",
option[1], sec)
elif logSys.getEffectiveLevel() <= logLevel:
logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec)
except ValueError:
logSys.warning("Wrong value for '" + option[1] + "' in '" + sec +
"'. Using default one: '" + `option[2]` + "'")
values[option[1]] = option[2]
return values
class DefinitionInitConfigReader(ConfigReader):
class DefinitionInitConfigReader(ConfigWrapper):
"""Config reader for files with options grouped in [Definition] and
[Init] sections.
[Init] sections.
Is a base class for readers of filters and actions, where definitions
in jails might provide custom values for options defined in [Init]
section.
"""
Is a base class for readers of filters and actions, where definitions
in jails might provide custom values for options defined in [Init]
section.
"""
_configOpts = []
def __init__(self, file_, jailName, initOpts, **kwargs):
ConfigReader.__init__(self, **kwargs)
ConfigWrapper.__init__(self, **kwargs)
self.setFile(file_)
self.setJailName(jailName)
self._initOpts = initOpts
@ -162,14 +220,14 @@ class DefinitionInitConfigReader(ConfigReader):
return self._jailName
def read(self):
return ConfigReader.read(self, self._file)
return ConfigWrapper.read(self, self._file)
# needed for fail2ban-regex that doesn't need fancy directories
def readexplicit(self):
return SafeConfigParserWithIncludes.read(self, self._file)
def getOptions(self, pOpts):
self._opts = ConfigReader.getOptions(
self._opts = ConfigWrapper.getOptions(
self, "Definition", self._configOpts, pOpts)
if self.has_section("Init"):

View File

@ -24,31 +24,32 @@ __author__ = "Cyril Jaquier"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
from .configreader import ConfigReader
from .configreader import ConfigWrapper
from ..helpers import getLogger
# Gets the instance of the logger.
logSys = getLogger(__name__)
class Fail2banReader(ConfigReader):
class Fail2banReader(ConfigWrapper):
def __init__(self, **kwargs):
ConfigReader.__init__(self, **kwargs)
self.__opts = None
ConfigWrapper.__init__(self, **kwargs)
def read(self):
ConfigReader.read(self, "fail2ban")
ConfigWrapper.read(self, "fail2ban")
def getEarlyOptions(self):
opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"],
["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]]
return ConfigReader.getOptions(self, "Definition", opts)
return ConfigWrapper.getOptions(self, "Definition", opts)
def getOptions(self):
opts = [["string", "loglevel", "INFO" ],
["string", "logtarget", "STDERR"],
["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
["int", "dbpurgeage", 86400]]
self.__opts = ConfigReader.getOptions(self, "Definition", opts)
self.__opts = ConfigWrapper.getOptions(self, "Definition", opts)
def convert(self):
stream = list()

View File

@ -26,7 +26,7 @@ __license__ = "GPL"
import os, shlex
from .configreader import ConfigReader, DefinitionInitConfigReader
from .configreader import DefinitionInitConfigReader
from ..server.action import CommandAction
from ..helpers import getLogger
@ -40,8 +40,12 @@ class FilterReader(DefinitionInitConfigReader):
["string", "failregex", ""],
]
def read(self):
return ConfigReader.read(self, os.path.join("filter.d", self._file))
def setFile(self, fileName):
self.__file = fileName
DefinitionInitConfigReader.setFile(self, os.path.join("filter.d", fileName))
def getFile(self):
return self.__file
def convert(self):
stream = list()

View File

@ -27,7 +27,7 @@ __license__ = "GPL"
import re, glob, os.path
import json
from .configreader import ConfigReader
from .configreader import ConfigReader, ConfigWrapper
from .filterreader import FilterReader
from .actionreader import ActionReader
from ..helpers import getLogger
@ -35,17 +35,19 @@ from ..helpers import getLogger
# Gets the instance of the logger.
logSys = getLogger(__name__)
class JailReader(ConfigReader):
class JailReader(ConfigWrapper):
optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
optionExtractRE = re.compile(
r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
def __init__(self, name, force_enable=False, **kwargs):
ConfigReader.__init__(self, **kwargs)
def __init__(self, name, force_enable=False, cfg_share=None, **kwargs):
# use shared config if possible:
ConfigWrapper.__init__(self, **kwargs)
self.__name = name
self.__filter = None
self.__force_enable = force_enable
self.__cfg_share = cfg_share
self.__actions = list()
self.__opts = None
@ -60,7 +62,7 @@ class JailReader(ConfigReader):
return self.__name
def read(self):
out = ConfigReader.read(self, "jail")
out = ConfigWrapper.read(self, "jail")
# Before returning -- verify that requested section
# exists at all
if not (self.__name in self.sections()):
@ -101,7 +103,7 @@ class JailReader(ConfigReader):
["string", "ignoreip", None],
["string", "filter", ""],
["string", "action", ""]]
self.__opts = ConfigReader.getOptions(self, self.__name, opts)
self.__opts = ConfigWrapper.getOptions(self, self.__name, opts)
if not self.__opts:
return False
@ -111,7 +113,7 @@ class JailReader(ConfigReader):
filterName, filterOpt = JailReader.extractOptions(
self.__opts["filter"])
self.__filter = FilterReader(
filterName, self.__name, filterOpt, basedir=self.getBaseDir())
filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir())
ret = self.__filter.read()
if ret:
self.__filter.getOptions(self.__opts)
@ -141,7 +143,7 @@ class JailReader(ConfigReader):
else:
action = ActionReader(
actName, self.__name, actOpt,
basedir=self.getBaseDir())
share_config=self.__cfg_share, basedir=self.getBaseDir())
ret = action.read()
if ret:
action.getOptions(self.__opts)
@ -213,7 +215,7 @@ class JailReader(ConfigReader):
if self.__filter:
stream.extend(self.__filter.convert())
for action in self.__actions:
if isinstance(action, ConfigReader):
if isinstance(action, (ConfigReader, ConfigWrapper)):
stream.extend(action.convert())
else:
stream.append(action)

View File

@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
from .configreader import ConfigReader
from .configreader import ConfigReader, ConfigWrapper
from .jailreader import JailReader
from ..helpers import getLogger
# Gets the instance of the logger.
logSys = getLogger(__name__)
class JailsReader(ConfigReader):
class JailsReader(ConfigWrapper):
def __init__(self, force_enable=False, **kwargs):
"""
@ -41,7 +41,9 @@ class JailsReader(ConfigReader):
Passed to JailReader to force enable the jails.
It is for internal use
"""
ConfigReader.__init__(self, **kwargs)
# use shared config if possible:
ConfigWrapper.__init__(self, **kwargs)
self.__cfg_share = dict()
self.__jails = list()
self.__force_enable = force_enable
@ -50,13 +52,13 @@ class JailsReader(ConfigReader):
return self.__jails
def read(self):
return ConfigReader.read(self, "jail")
return ConfigWrapper.read(self, "jail")
def getOptions(self, section=None):
"""Reads configuration for jail(s) and adds enabled jails to __jails
"""
opts = []
self.__opts = ConfigReader.getOptions(self, "Definition", opts)
self.__opts = ConfigWrapper.getOptions(self, "Definition", opts)
if section is None:
sections = self.sections()
@ -68,9 +70,10 @@ class JailsReader(ConfigReader):
for sec in sections:
if sec == 'INCLUDES':
continue
jail = JailReader(sec, basedir=self.getBaseDir(),
force_enable=self.__force_enable)
jail.read()
# use the cfg_share for filter/action caching and the same config for all
# jails (use_config=...), therefore don't read it here:
jail = JailReader(sec, force_enable=self.__force_enable,
cfg_share=self.__cfg_share, use_config=self._cfg)
ret = jail.getOptions()
if ret:
if jail.isEnabled():