rewritten caching resp. sharing of ConfigReader and SafeConfigParserWithIncludes (v.2, first and second level cache, without fingerprinting etc.);

pull/824/head
sebres 2014-10-10 02:10:13 +02:00
parent 37952ab75f
commit c35b4b24d2
6 changed files with 178 additions and 204 deletions

View File

@ -65,136 +65,7 @@ logSys = getLogger(__name__)
__all__ = ['SafeConfigParserWithIncludes']
class SafeConfigParserWithIncludes(object):
SECTION_NAME = "INCLUDES"
CFG_CACHE = {}
CFG_INC_CACHE = {}
CFG_EMPY_CFG = None
def __init__(self):
self.__cr = None
def __check_read(self, attr):
if self.__cr is None:
# raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr)
if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None:
SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes()
self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG
def __getattr__(self,attr):
# check we access local implementation
try:
orig_attr = self.__getattribute__(attr)
except AttributeError:
self.__check_read(attr)
orig_attr = self.__cr.__getattribute__(attr)
return orig_attr
@staticmethod
def _get_resource_fingerprint(resource):
mt = []
dirnames = []
for filename in resource:
if os.path.exists(filename):
s = os.stat(filename)
mt.append(s.st_mtime)
mt.append(s.st_mode)
mt.append(s.st_size)
dirname = os.path.dirname(filename)
if dirname not in dirnames:
dirnames.append(dirname)
for dirname in dirnames:
if os.path.exists(dirname):
s = os.stat(dirname)
mt.append(s.st_mtime)
mt.append(s.st_mode)
mt.append(s.st_size)
return mt
def read(self, resource, get_includes=True, log_info=None):
SCPWI = SafeConfigParserWithIncludes
# check includes :
fileNamesFull = []
if not isinstance(resource, list):
resource = [ resource ]
if get_includes:
for filename in resource:
fileNamesFull += SCPWI.getIncludes(filename)
else:
fileNamesFull = resource
# check cache
hashv = '\x01'.join(fileNamesFull)
cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0))
curmt = SCPWI._get_resource_fingerprint(fileNamesFull)
if cr is not None and mtime == curmt:
self.__cr = cr
logSys.debug("Cached config files: %s", resource)
#logSys.debug("Cached config files: %s", fileNamesFull)
return ret
# not yet in cache - create/read and add to cache:
if log_info is not None:
logSys.info(*log_info)
cr = _SafeConfigParserWithIncludes()
ret = cr.read(fileNamesFull)
SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt)
self.__cr = cr
return ret
def getOptions(self, *args, **kwargs):
self.__check_read('getOptions')
return self.__cr.getOptions(*args, **kwargs)
@staticmethod
def getIncludes(resource, seen = []):
"""
Given 1 config resource returns list of included files
(recursively) with the original one as well
Simple loops are taken care about
"""
# Use a short class name ;)
SCPWI = SafeConfigParserWithIncludes
resources = seen + [resource]
# check cache
hashv = '///'.join(resources)
cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0))
curmt = SCPWI._get_resource_fingerprint(resources)
if cinc is not None and mtime == curmt:
return cinc
parser = SCPWI()
try:
# read without includes
parser.read(resource, get_includes=False)
except UnicodeDecodeError, e:
logSys.error("Error decoding config file '%s': %s" % (resource, e))
return []
resourceDir = os.path.dirname(resource)
newFiles = [ ('before', []), ('after', []) ]
if SCPWI.SECTION_NAME in parser.sections():
for option_name, option_list in newFiles:
if option_name in parser.options(SCPWI.SECTION_NAME):
newResources = parser.get(SCPWI.SECTION_NAME, option_name)
for newResource in newResources.split('\n'):
if os.path.isabs(newResource):
r = newResource
else:
r = os.path.join(resourceDir, newResource)
if r in seen:
continue
option_list += SCPWI.getIncludes(r, resources)
# combine lists
cinc = newFiles[0][1] + [resource] + newFiles[1][1]
# cache and return :
SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt)
return cinc
#print "Includes list for " + resource + " is " + `resources`
class _SafeConfigParserWithIncludes(SafeConfigParser, object):
class SafeConfigParserWithIncludes(SafeConfigParser):
"""
Class adds functionality to SafeConfigParser to handle included
other configuration files (or may be urls, whatever in the future)
@ -223,14 +94,99 @@ after = 1.conf
"""
SECTION_NAME = "INCLUDES"
if sys.version_info >= (3,2):
# overload constructor only for fancy new Python3's
def __init__(self, *args, **kwargs):
def __init__(self, share_config=None, *args, **kwargs):
kwargs = kwargs.copy()
kwargs['interpolation'] = BasicInterpolationWithName()
kwargs['inline_comment_prefixes'] = ";"
super(_SafeConfigParserWithIncludes, self).__init__(
super(SafeConfigParserWithIncludes, self).__init__(
*args, **kwargs)
self._cfg_share = share_config
else:
def __init__(self, share_config=None, *args, **kwargs):
SafeConfigParser.__init__(self, *args, **kwargs)
self._cfg_share = share_config
@property
def share_config(self):
return self._cfg_share
def _getSharedSCPWI(self, filename):
SCPWI = SafeConfigParserWithIncludes
# read single one, add to return list, use sharing if possible:
if self._cfg_share:
# cache/share each file as include (ex: filter.d/common could be included in each filter config):
hashv = 'inc:'+(filename if not isinstance(filename, list) else '\x01'.join(filename))
cfg, i = self._cfg_share.get(hashv, (None, None))
if cfg is None:
cfg = SCPWI(share_config=self._cfg_share)
i = cfg.read(filename, get_includes=False)
self._cfg_share[hashv] = (cfg, i)
else:
logSys.debug(" Shared file: %s", filename)
else:
# don't have sharing:
cfg = SCPWI()
i = cfg.read(filename, get_includes=False)
return (cfg, i)
def _getIncludes(self, filenames, seen=[]):
if not isinstance(filenames, list):
filenames = [ filenames ]
# retrieve or cache include paths:
if self._cfg_share:
# cache/share include list:
hashv = 'inc-path:'+('\x01'.join(filenames))
fileNamesFull = self._cfg_share.get(hashv)
if fileNamesFull is None:
fileNamesFull = []
for filename in filenames:
fileNamesFull += self.__getIncludesUncached(filename, seen)
self._cfg_share[hashv] = fileNamesFull
return fileNamesFull
# don't have sharing:
fileNamesFull = []
for filename in filenames:
fileNamesFull += self.__getIncludesUncached(filename, seen)
return fileNamesFull
def __getIncludesUncached(self, resource, seen=[]):
"""
Given 1 config resource returns list of included files
(recursively) with the original one as well
Simple loops are taken care about
"""
SCPWI = SafeConfigParserWithIncludes
try:
parser, i = self._getSharedSCPWI(resource)
if not i:
return []
except UnicodeDecodeError, e:
logSys.error("Error decoding config file '%s': %s" % (resource, e))
return []
resourceDir = os.path.dirname(resource)
newFiles = [ ('before', []), ('after', []) ]
if SCPWI.SECTION_NAME in parser.sections():
for option_name, option_list in newFiles:
if option_name in parser.options(SCPWI.SECTION_NAME):
newResources = parser.get(SCPWI.SECTION_NAME, option_name)
for newResource in newResources.split('\n'):
if os.path.isabs(newResource):
r = newResource
else:
r = os.path.join(resourceDir, newResource)
if r in seen:
continue
s = seen + [resource]
option_list += self._getIncludes(r, s)
# combine lists
return newFiles[0][1] + [resource] + newFiles[1][1]
def get_defaults(self):
return self._defaults
@ -238,18 +194,27 @@ after = 1.conf
def get_sections(self):
return self._sections
def read(self, filenames):
def read(self, filenames, get_includes=True, log_info=None):
if not isinstance(filenames, list):
filenames = [ filenames ]
if len(filenames) > 1:
# retrieve (and cache) includes:
fileNamesFull = []
if get_includes:
fileNamesFull += self._getIncludes(filenames)
else:
fileNamesFull = filenames
if self._cfg_share is not None:
logSys.debug(" Sharing files: %s", fileNamesFull)
if len(fileNamesFull) > 1:
# read multiple configs:
ret = []
alld = self.get_defaults()
alls = self.get_sections()
for filename in filenames:
# read single one, add to return list:
cfg = SafeConfigParserWithIncludes()
i = cfg.read(filename, get_includes=False)
for filename in fileNamesFull:
# read single one, add to return list, use sharing if possible:
cfg, i = self._getSharedSCPWI(filename)
if i:
ret += i
# merge defaults and all sections to self:
@ -267,9 +232,14 @@ after = 1.conf
return ret
# read one config :
logSys.debug("Reading file: %s", filenames[0])
if sys.version_info >= (3,2): # pragma: no cover
return SafeConfigParser.read(self, filenames, encoding='utf-8')
logSys.debug(" Reading file: %s", fileNamesFull[0])
else:
return SafeConfigParser.read(self, filenames)
# don't have sharing - read one or multiple at once:
logSys.debug(" Reading files: %s", fileNamesFull)
# read file(s) :
if sys.version_info >= (3,2): # pragma: no cover
return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8')
else:
return SafeConfigParser.read(self, fileNamesFull)

View File

@ -46,26 +46,38 @@ class ConfigReader():
self._cfg = None
if use_config is not None:
self._cfg = use_config
else:
# share config if possible:
if share_config is not None:
self._cfg_share = share_config
self._cfg_share_kwargs = kwargs
else:
self._cfg_share_basedir = None
elif self._cfg is None:
self._cfg = ConfigReaderUnshared(**kwargs)
def setBaseDir(self, basedir):
if self._cfg:
self._cfg.setBaseDir(basedir)
else:
self._cfg_share_basedir = basedir
def getBaseDir(self):
if self._cfg:
return self._cfg.getBaseDir()
else:
return self._cfg_share_basedir
@property
def share_config(self):
return self._cfg_share
def read(self, name, once=True):
# shared ?
if not self._cfg and self._cfg_share is not None:
self._cfg = self._cfg_share.get(name)
if not self._cfg:
self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs)
self._cfg = ConfigReaderUnshared(share_config=self._cfg_share, **self._cfg_share_kwargs)
if self._cfg_share_basedir is not None:
self._cfg.setBaseDir(self._cfg_share_basedir)
self._cfg_share[name] = self._cfg
# performance feature - read once if using shared config reader:
rc = self._cfg.read_cfg_files
@ -73,6 +85,8 @@ class ConfigReader():
return rc.get(name)
# read:
if self._cfg_share is not None:
logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir())
ret = self._cfg.read(name)
# save already read:
@ -105,8 +119,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
DEFAULT_BASEDIR = '/etc/fail2ban'
def __init__(self, basedir=None):
SafeConfigParserWithIncludes.__init__(self)
def __init__(self, basedir=None, *args, **kwargs):
SafeConfigParserWithIncludes.__init__(self, *args, **kwargs)
self.read_cfg_files = dict()
self.setBaseDir(basedir)
@ -123,7 +137,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
raise ValueError("Base configuration directory %s does not exist "
% self._basedir)
basename = os.path.join(self._basedir, filename)
logSys.debug("Reading configs for %s under %s " , filename, self._basedir)
logSys.info("Reading configs for %s under %s " , filename, self._basedir)
config_files = [ basename + ".conf" ]
# possible further customizations under a .conf.d directory
@ -140,8 +154,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
if len(config_files):
# at least one config exists and accessible
logSys.debug("Reading config files: %s", ', '.join(config_files))
config_files_read = SafeConfigParserWithIncludes.read(self, config_files,
log_info=("Cache configs for %s under %s " , filename, self._basedir))
config_files_read = SafeConfigParserWithIncludes.read(self, config_files)
missed = [ cf for cf in config_files if cf not in config_files_read ]
if missed:
logSys.error("Could not read config files: %s", ', '.join(missed))

View File

@ -33,11 +33,11 @@ logSys = getLogger(__name__)
class Configurator:
def __init__(self, force_enable=False):
def __init__(self, force_enable=False, share_config=None):
self.__settings = dict()
self.__streams = dict()
self.__fail2ban = Fail2banReader()
self.__jails = JailsReader(force_enable=force_enable)
self.__fail2ban = Fail2banReader(share_config=share_config)
self.__jails = JailsReader(force_enable=force_enable, share_config=share_config)
def setBaseDir(self, folderName):
self.__fail2ban.setBaseDir(folderName)

View File

@ -41,13 +41,12 @@ class JailReader(ConfigReader):
optionExtractRE = re.compile(
r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
def __init__(self, name, force_enable=False, cfg_share=None, **kwargs):
def __init__(self, name, force_enable=False, **kwargs):
# use shared config if possible:
ConfigReader.__init__(self, **kwargs)
self.__name = name
self.__filter = None
self.__force_enable = force_enable
self.__cfg_share = cfg_share
self.__actions = list()
self.__opts = None
@ -113,7 +112,7 @@ class JailReader(ConfigReader):
filterName, filterOpt = JailReader.extractOptions(
self.__opts["filter"])
self.__filter = FilterReader(
filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir())
filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
ret = self.__filter.read()
if ret:
self.__filter.getOptions(self.__opts)
@ -143,7 +142,7 @@ class JailReader(ConfigReader):
else:
action = ActionReader(
actName, self.__name, actOpt,
share_config=self.__cfg_share, basedir=self.getBaseDir())
share_config=self.share_config, basedir=self.getBaseDir())
ret = action.read()
if ret:
action.getOptions(self.__opts)

View File

@ -43,7 +43,6 @@ class JailsReader(ConfigReader):
"""
# use shared config if possible:
ConfigReader.__init__(self, **kwargs)
self.__cfg_share = dict()
self.__jails = list()
self.__force_enable = force_enable
@ -73,7 +72,7 @@ class JailsReader(ConfigReader):
# use the cfg_share for filter/action caching and the same config for all
# jails (use_config=...), therefore don't read it here:
jail = JailReader(sec, force_enable=self.__force_enable,
cfg_share=self.__cfg_share, use_config=self._cfg)
share_config=self.share_config, use_config=self._cfg)
ret = jail.getOptions()
if ret:
if jail.isEnabled():

View File

@ -39,8 +39,6 @@ STOCK = os.path.exists(os.path.join('config','fail2ban.conf'))
IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
LAST_WRITE_TIME = 0
class ConfigReaderTest(unittest.TestCase):
def setUp(self):
@ -59,8 +57,7 @@ class ConfigReaderTest(unittest.TestCase):
d_ = os.path.join(self.d, d)
if not os.path.exists(d_):
os.makedirs(d_)
fname = "%s/%s" % (self.d, fname)
f = open(fname, "w")
f = open("%s/%s" % (self.d, fname), "w")
if value is not None:
f.write("""
[section]
@ -69,14 +66,6 @@ option = %s
if content is not None:
f.write(content)
f.close()
# set modification time to another second to revalidate cache (if milliseconds not supported) :
global LAST_WRITE_TIME
mtime = os.path.getmtime(fname)
if LAST_WRITE_TIME == mtime:
mtime += 1
os.utime(fname, (mtime, mtime))
LAST_WRITE_TIME = mtime
def _remove(self, fname):
os.unlink("%s/%s" % (self.d, fname))
@ -102,6 +91,7 @@ option = %s
# raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform)
pass
def testOptionalDotDDir(self):
self.assertFalse(self.c.read('c')) # nothing is there yet
self._write("c.conf", "1")
@ -347,9 +337,9 @@ class FilterReaderTest(unittest.TestCase):
class JailsReaderTestCache(LogCaptureTestCase):
def _readWholeConf(self, basedir, force_enable=False):
def _readWholeConf(self, basedir, force_enable=False, share_config=None):
# read whole configuration like a file2ban-client ...
configurator = Configurator(force_enable=force_enable)
configurator = Configurator(force_enable=force_enable, share_config=share_config)
configurator.setBaseDir(basedir)
configurator.readEarly()
configurator.getEarlyOptions()
@ -360,7 +350,7 @@ class JailsReaderTestCache(LogCaptureTestCase):
def _getLoggedReadCount(self, filematch):
cnt = 0
for s in self.getLog().rsplit('\n'):
if re.match(r"^Reading files?: .*/"+filematch, s):
if re.match(r"^\s*Reading files?: .*/"+filematch, s):
cnt += 1
return cnt
@ -372,8 +362,11 @@ class JailsReaderTestCache(LogCaptureTestCase):
shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local')
shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local')
# common sharing handle for this test:
share_cfg = dict()
# read whole configuration like a file2ban-client ...
self._readWholeConf(basedir)
self._readWholeConf(basedir, share_config=share_cfg)
# how many times jail.local was read:
cnt = self._getLoggedReadCount('jail.local')
# if cnt > 1:
@ -382,7 +375,7 @@ class JailsReaderTestCache(LogCaptureTestCase):
# read whole configuration like a file2ban-client, again ...
# but this time force enable all jails, to check filter and action cached also:
self._readWholeConf(basedir, force_enable=True)
self._readWholeConf(basedir, force_enable=True, share_config=share_cfg)
cnt = self._getLoggedReadCount(r'jail\.local')
# still one (no more reads):
self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt)