Merge branch 'cache-config-read-820' into _tent/cache-config-read

fail2ban/client/actionreader.py

@@ -26,7 +26,7 @@ __license__ = "GPL"
 
 import os
 
-from .configreader import ConfigReader, DefinitionInitConfigReader
+from .configreader import DefinitionInitConfigReader
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -47,15 +47,19 @@ class ActionReader(DefinitionInitConfigReader):
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
 
+	def setFile(self, fileName):
+		self.__file = fileName
+		DefinitionInitConfigReader.setFile(self, os.path.join("action.d", fileName))
+	
+	def getFile(self):
+		return self.__file
+
 	def setName(self, name):
 		self._name = name
 
 	def getName(self):
 		return self._name
 
-	def read(self):
-		return ConfigReader.read(self, os.path.join("action.d", self._file))
-
 	def convert(self):
 		head = ["set", self._jailName]
 		stream = list()

fail2ban/client/configparserinc.py

@@ -65,7 +65,136 @@ logSys = getLogger(__name__)
 
 __all__ = ['SafeConfigParserWithIncludes']
 
-class SafeConfigParserWithIncludes(SafeConfigParser):
+class SafeConfigParserWithIncludes(object):
+
+	SECTION_NAME = "INCLUDES"
+	CFG_CACHE = {}
+	CFG_INC_CACHE = {}
+	CFG_EMPY_CFG = None
+
+	def __init__(self):
+		self.__cr = None
+
+	def __check_read(self, attr):
+		if self.__cr is None:
+			# raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr)
+			if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None: 
+				SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes()
+			self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG
+
+	def __getattr__(self,attr):
+		# check we access local implementation
+		try:
+			orig_attr = self.__getattribute__(attr)
+		except AttributeError:
+			self.__check_read(attr)
+			orig_attr = self.__cr.__getattribute__(attr)
+		return orig_attr
+
+	@staticmethod
+	def _resource_mtime(resource):
+		mt = []
+		dirnames = []
+		for filename in resource:
+			if os.path.exists(filename):
+				s = os.stat(filename)
+				mt.append(s.st_mtime)
+				mt.append(s.st_mode)
+				mt.append(s.st_size)
+				dirname = os.path.dirname(filename)
+				if dirname not in dirnames:
+					dirnames.append(dirname)
+		for dirname in dirnames:
+			if os.path.exists(dirname):
+				s = os.stat(dirname)
+				mt.append(s.st_mtime)
+				mt.append(s.st_mode)
+				mt.append(s.st_size)
+		return mt
+
+	def read(self, resource, get_includes=True, log_info=None):
+		SCPWI = SafeConfigParserWithIncludes
+		# check includes :
+		fileNamesFull = []
+		if not isinstance(resource, list):
+			resource = [ resource ]
+		if get_includes:
+			for filename in resource:
+				fileNamesFull += SCPWI.getIncludes(filename)
+		else:
+			fileNamesFull = resource
+		# check cache
+		hashv = '\x01'.join(fileNamesFull)
+		cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0))
+		curmt = SCPWI._resource_mtime(fileNamesFull)
+		if cr is not None and mtime == curmt:
+			self.__cr = cr
+			logSys.debug("Cached config files: %s", resource)
+			#logSys.debug("Cached config files: %s", fileNamesFull)
+			return ret
+		# not yet in cache - create/read and add to cache:
+		if log_info is not None:
+			logSys.info(*log_info)
+		cr = _SafeConfigParserWithIncludes()
+		ret = cr.read(fileNamesFull)
+		SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt)
+		self.__cr = cr
+		return ret
+
+	def getOptions(self, *args, **kwargs):
+		self.__check_read('getOptions')
+		return self.__cr.getOptions(*args, **kwargs)
+
+	@staticmethod
+	def getIncludes(resource, seen = []):
+		"""
+		Given 1 config resource returns list of included files
+		(recursively) with the original one as well
+		Simple loops are taken care about
+		"""
+		
+		# Use a short class name ;)
+		SCPWI = SafeConfigParserWithIncludes
+
+		resources = seen + [resource]
+		# check cache
+		hashv = '///'.join(resources)
+		cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0))
+		curmt = SCPWI._resource_mtime(resources)
+		if cinc is not None and mtime == curmt:
+			return cinc
+		
+		parser = SCPWI()
+		try:
+			# read without includes
+			parser.read(resource, get_includes=False)
+		except UnicodeDecodeError, e:
+			logSys.error("Error decoding config file '%s': %s" % (resource, e))
+			return []
+		
+		resourceDir = os.path.dirname(resource)
+
+		newFiles = [ ('before', []), ('after', []) ]
+		if SCPWI.SECTION_NAME in parser.sections():
+			for option_name, option_list in newFiles:
+				if option_name in parser.options(SCPWI.SECTION_NAME):
+					newResources = parser.get(SCPWI.SECTION_NAME, option_name)
+					for newResource in newResources.split('\n'):
+						if os.path.isabs(newResource):
+							r = newResource
+						else:
+							r = os.path.join(resourceDir, newResource)
+						if r in seen:
+							continue
+						option_list += SCPWI.getIncludes(r, resources)
+		# combine lists
+		cinc = newFiles[0][1] + [resource] + newFiles[1][1]
+		# cache and return :
+		SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt)
+		return cinc
+		#print "Includes list for " + resource + " is " + `resources`
+
+class _SafeConfigParserWithIncludes(SafeConfigParser, object):
 	"""
 	Class adds functionality to SafeConfigParser to handle included
 	other configuration files (or may be urls, whatever in the future)
@@ -94,69 +223,53 @@ after = 1.conf
 
 	"""
 
-	SECTION_NAME = "INCLUDES"
-
 	if sys.version_info >= (3,2):
 		# overload constructor only for fancy new Python3's
 		def __init__(self, *args, **kwargs):
 			kwargs = kwargs.copy()
 			kwargs['interpolation'] = BasicInterpolationWithName()
 			kwargs['inline_comment_prefixes'] = ";"
-			super(SafeConfigParserWithIncludes, self).__init__(
+			super(_SafeConfigParserWithIncludes, self).__init__(
 				*args, **kwargs)
 
-	#@staticmethod
+	def get_defaults(self):
-	def getIncludes(resource, seen = []):
+		return self._defaults
-		"""
-		Given 1 config resource returns list of included files
-		(recursively) with the original one as well
-		Simple loops are taken care about
-		"""
-		
-		# Use a short class name ;)
-		SCPWI = SafeConfigParserWithIncludes
-		
-		parser = SafeConfigParser()
-		try:
-			if sys.version_info >= (3,2): # pragma: no cover
-				parser.read(resource, encoding='utf-8')
-			else:
-				parser.read(resource)
-		except UnicodeDecodeError, e:
-			logSys.error("Error decoding config file '%s': %s" % (resource, e))
-			return []
-		
-		resourceDir = os.path.dirname(resource)
-
-		newFiles = [ ('before', []), ('after', []) ]
-		if SCPWI.SECTION_NAME in parser.sections():
-			for option_name, option_list in newFiles:
-				if option_name in parser.options(SCPWI.SECTION_NAME):
-					newResources = parser.get(SCPWI.SECTION_NAME, option_name)
-					for newResource in newResources.split('\n'):
-						if os.path.isabs(newResource):
-							r = newResource
-						else:
-							r = os.path.join(resourceDir, newResource)
-						if r in seen:
-							continue
-						s = seen + [resource]
-						option_list += SCPWI.getIncludes(r, s)
-		# combine lists
-		return newFiles[0][1] + [resource] + newFiles[1][1]
-		#print "Includes list for " + resource + " is " + `resources`
-	getIncludes = staticmethod(getIncludes)
 
+	def get_sections(self):
+		return self._sections
 
 	def read(self, filenames):
-		fileNamesFull = []
 		if not isinstance(filenames, list):
 			filenames = [ filenames ]
+		if len(filenames) > 1:
+			# read multiple configs:
+			ret = []
+			alld = self.get_defaults()
+			alls = self.get_sections()
 			for filename in filenames:
-			fileNamesFull += SafeConfigParserWithIncludes.getIncludes(filename)
+				# read single one, add to return list:
-		logSys.debug("Reading files: %s" % fileNamesFull)
+				cfg = SafeConfigParserWithIncludes()
-		if sys.version_info >= (3,2): # pragma: no cover
+				i = cfg.read(filename, get_includes=False)
-			return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8')
+				if i:
+					ret += i
+					# merge defaults and all sections to self:
+					alld.update(cfg.get_defaults())
+					for n, s in cfg.get_sections().iteritems():
+						if isinstance(s, dict):
+							s2 = alls.get(n)
+							if isinstance(s2, dict):
+								s2.update(s)
 							else:
-			return SafeConfigParser.read(self, fileNamesFull)
+								alls[n] = s.copy()
+						else:
+							alls[n] = s
+
+			return ret
+
+		# read one config :
+		logSys.debug("Reading file: %s", filenames[0])
+		if sys.version_info >= (3,2): # pragma: no cover
+			return SafeConfigParser.read(self, filenames, encoding='utf-8')
+		else:
+			return SafeConfigParser.read(self, filenames)
 

fail2ban/client/configreader.py

@@ -32,6 +32,65 @@ from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
+logLevel = 6
+
+class ConfigWrapper():
+
+	def __init__(self, use_config=None, share_config=None, **kwargs):
+		# use given shared config if possible (see read):
+		self._cfg_share = None
+		self._cfg = None
+		if use_config is not None:
+			self._cfg = use_config
+		else:
+			# share config if possible:
+			if share_config is not None:
+				self._cfg_share = share_config
+				self._cfg_share_kwargs = kwargs
+			else:
+				self._cfg = ConfigReader(**kwargs)
+
+	def setBaseDir(self, basedir):
+		self._cfg.setBaseDir(basedir)
+
+	def getBaseDir(self):
+		return self._cfg.getBaseDir()
+
+	def read(self, name, once=True):
+		# shared ?
+		if not self._cfg and self._cfg_share is not None:
+			self._cfg = self._cfg_share.get(name)
+			if not self._cfg:
+				self._cfg = ConfigReader(**self._cfg_share_kwargs)
+				self._cfg_share[name] = self._cfg
+		# performance feature - read once if using shared config reader:
+		rc = self._cfg.read_cfg_files
+		if once and rc.get(name) is not None:
+			return rc.get(name)
+
+		# read:
+		ret = self._cfg.read(name)
+
+		# save already read:
+		if once:
+			rc[name] = ret
+		return ret
+
+	def sections(self):
+		return self._cfg.sections()
+
+	def has_section(self, sec):
+		return self._cfg.has_section(sec)
+
+	def options(self, *args):
+		return self._cfg.options(*args)
+
+	def get(self, sec, opt):
+		return self._cfg.get(sec, opt)
+
+	def getOptions(self, *args, **kwargs):
+		return self._cfg.getOptions(*args, **kwargs)
+
 
 class ConfigReader(SafeConfigParserWithIncludes):
 
@@ -39,8 +98,8 @@ class ConfigReader(SafeConfigParserWithIncludes):
 	
 	def __init__(self, basedir=None):
 		SafeConfigParserWithIncludes.__init__(self)
+		self.read_cfg_files = dict()
 		self.setBaseDir(basedir)
-		self.__opts = None
 	
 	def setBaseDir(self, basedir):
 		if basedir is None:
@@ -55,7 +114,7 @@ class ConfigReader(SafeConfigParserWithIncludes):
 			raise ValueError("Base configuration directory %s does not exist "
 							  % self._basedir)
 		basename = os.path.join(self._basedir, filename)
-		logSys.info("Reading configs for %s under %s "  % (basename, self._basedir))
+		logSys.debug("Reading configs for %s under %s " , filename, self._basedir)
 		config_files = [ basename + ".conf" ]
 
 		# possible further customizations under a .conf.d directory
@@ -71,14 +130,15 @@ class ConfigReader(SafeConfigParserWithIncludes):
 
 		if len(config_files):
 			# at least one config exists and accessible
-			logSys.debug("Reading config files: " + ', '.join(config_files))
+			logSys.debug("Reading config files: %s", ', '.join(config_files))
-			config_files_read = SafeConfigParserWithIncludes.read(self, config_files)
+			config_files_read = SafeConfigParserWithIncludes.read(self, config_files,
+				log_info=("Cache configs for %s under %s " , filename, self._basedir))
 			missed = [ cf for cf in config_files if cf not in config_files_read ]
 			if missed:
-				logSys.error("Could not read config files: " + ', '.join(missed))
+				logSys.error("Could not read config files: %s", ', '.join(missed))
 			if config_files_read:
 				return True
-			logSys.error("Found no accessible config files for %r under %s" %
+			logSys.error("Found no accessible config files for %r under %s",
 						 ( filename, self.getBaseDir() ))
 			return False
 		else:
@@ -121,17 +181,15 @@ class ConfigReader(SafeConfigParserWithIncludes):
 					logSys.warning("'%s' not defined in '%s'. Using default one: %r"
 								% (option[1], sec, option[2]))
 					values[option[1]] = option[2]
-				else:
+				elif logSys.getEffectiveLevel() <= logLevel:
-					logSys.debug(
+					logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec)
-						"Non essential option '%s' not defined in '%s'.",
-						option[1], sec)
 			except ValueError:
 				logSys.warning("Wrong value for '" + option[1] + "' in '" + sec +
 							"'. Using default one: '" + `option[2]` + "'")
 				values[option[1]] = option[2]
 		return values
 
-class DefinitionInitConfigReader(ConfigReader):
+class DefinitionInitConfigReader(ConfigWrapper):
 	"""Config reader for files with options grouped in [Definition] and
 			 [Init] sections.
 
@@ -143,7 +201,7 @@ class DefinitionInitConfigReader(ConfigReader):
 	_configOpts = []
 	
 	def __init__(self, file_, jailName, initOpts, **kwargs):
-		ConfigReader.__init__(self, **kwargs)
+		ConfigWrapper.__init__(self, **kwargs)
 		self.setFile(file_)
 		self.setJailName(jailName)
 		self._initOpts = initOpts
@@ -162,14 +220,14 @@ class DefinitionInitConfigReader(ConfigReader):
 		return self._jailName
 	
 	def read(self):
-		return ConfigReader.read(self, self._file)
+		return ConfigWrapper.read(self, self._file)
 
 	# needed for fail2ban-regex that doesn't need fancy directories
 	def readexplicit(self):
 		return SafeConfigParserWithIncludes.read(self, self._file)
 	
 	def getOptions(self, pOpts):
-		self._opts = ConfigReader.getOptions(
+		self._opts = ConfigWrapper.getOptions(
 			self, "Definition", self._configOpts, pOpts)
 		
 		if self.has_section("Init"):

fail2ban/client/fail2banreader.py

@@ -24,31 +24,32 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-from .configreader import ConfigReader
+from .configreader import ConfigWrapper
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-class Fail2banReader(ConfigReader):
+class Fail2banReader(ConfigWrapper):
 	
 	def __init__(self, **kwargs):
-		ConfigReader.__init__(self, **kwargs)
+		self.__opts = None
+		ConfigWrapper.__init__(self, **kwargs)
 	
 	def read(self):
-		ConfigReader.read(self, "fail2ban")
+		ConfigWrapper.read(self, "fail2ban")
 	
 	def getEarlyOptions(self):
 		opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"],
 				["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]]
-		return ConfigReader.getOptions(self, "Definition", opts)
+		return ConfigWrapper.getOptions(self, "Definition", opts)
 	
 	def getOptions(self):
 		opts = [["string", "loglevel", "INFO" ],
 				["string", "logtarget", "STDERR"],
 				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
 				["int", "dbpurgeage", 86400]]
-		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
+		self.__opts = ConfigWrapper.getOptions(self, "Definition", opts)
 	
 	def convert(self):
 		stream = list()

fail2ban/client/filterreader.py

@@ -26,7 +26,7 @@ __license__ = "GPL"
 
 import os, shlex
 
-from .configreader import ConfigReader, DefinitionInitConfigReader
+from .configreader import DefinitionInitConfigReader
 from ..server.action import CommandAction
 from ..helpers import getLogger
 
@@ -40,8 +40,12 @@ class FilterReader(DefinitionInitConfigReader):
 		["string", "failregex", ""],
 	]
 
-	def read(self):
+	def setFile(self, fileName):
-		return ConfigReader.read(self, os.path.join("filter.d", self._file))
+		self.__file = fileName
+		DefinitionInitConfigReader.setFile(self, os.path.join("filter.d", fileName))
+	
+	def getFile(self):
+		return self.__file
 	
 	def convert(self):
 		stream = list()

fail2ban/client/jailreader.py

@@ -27,7 +27,7 @@ __license__ = "GPL"
 import re, glob, os.path
 import json
 
-from .configreader import ConfigReader
+from .configreader import ConfigReader, ConfigWrapper
 from .filterreader import FilterReader
 from .actionreader import ActionReader
 from ..helpers import getLogger
@@ -35,17 +35,19 @@ from ..helpers import getLogger
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-class JailReader(ConfigReader):
+class JailReader(ConfigWrapper):
 	
 	optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
 	optionExtractRE = re.compile(
 		r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)')
 	
-	def __init__(self, name, force_enable=False, **kwargs):
+	def __init__(self, name, force_enable=False, cfg_share=None, **kwargs):
-		ConfigReader.__init__(self, **kwargs)
+		# use shared config if possible:
+		ConfigWrapper.__init__(self, **kwargs)
 		self.__name = name
 		self.__filter = None
 		self.__force_enable = force_enable
+		self.__cfg_share = cfg_share
 		self.__actions = list()
 		self.__opts = None
 	
@@ -60,7 +62,7 @@ class JailReader(ConfigReader):
 		return self.__name
 	
 	def read(self):
-		out = ConfigReader.read(self, "jail")
+		out = ConfigWrapper.read(self, "jail")
 		# Before returning -- verify that requested section
 		# exists at all
 		if not (self.__name in self.sections()):
@@ -101,7 +103,7 @@ class JailReader(ConfigReader):
 				["string", "ignoreip", None],
 				["string", "filter", ""],
 				["string", "action", ""]]
-		self.__opts = ConfigReader.getOptions(self, self.__name, opts)
+		self.__opts = ConfigWrapper.getOptions(self, self.__name, opts)
 		if not self.__opts:
 			return False
 		
@@ -111,7 +113,7 @@ class JailReader(ConfigReader):
 				filterName, filterOpt = JailReader.extractOptions(
 					self.__opts["filter"])
 				self.__filter = FilterReader(
-					filterName, self.__name, filterOpt, basedir=self.getBaseDir())
+					filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				if ret:
 					self.__filter.getOptions(self.__opts)
@@ -141,7 +143,7 @@ class JailReader(ConfigReader):
 					else:
 						action = ActionReader(
 							actName, self.__name, actOpt,
-							basedir=self.getBaseDir())
+							share_config=self.__cfg_share, basedir=self.getBaseDir())
 						ret = action.read()
 						if ret:
 							action.getOptions(self.__opts)
@@ -213,7 +215,7 @@ class JailReader(ConfigReader):
 		if self.__filter:
 			stream.extend(self.__filter.convert())
 		for action in self.__actions:
-			if isinstance(action, ConfigReader):
+			if isinstance(action, (ConfigReader, ConfigWrapper)):
 				stream.extend(action.convert())
 			else:
 				stream.append(action)

fail2ban/client/jailsreader.py

@@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-from .configreader import ConfigReader
+from .configreader import ConfigReader, ConfigWrapper
 from .jailreader import JailReader
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-class JailsReader(ConfigReader):
+class JailsReader(ConfigWrapper):
 
 	def __init__(self, force_enable=False, **kwargs):
 		"""
@@ -41,7 +41,9 @@ class JailsReader(ConfigReader):
 		  Passed to JailReader to force enable the jails.
 		  It is for internal use
 		"""
-		ConfigReader.__init__(self, **kwargs)
+		# use shared config if possible:
+		ConfigWrapper.__init__(self, **kwargs)
+		self.__cfg_share = dict()
 		self.__jails = list()
 		self.__force_enable = force_enable
 
@@ -50,13 +52,13 @@ class JailsReader(ConfigReader):
 		return self.__jails
 
 	def read(self):
-		return ConfigReader.read(self, "jail")
+		return ConfigWrapper.read(self, "jail")
 
 	def getOptions(self, section=None):
 		"""Reads configuration for jail(s) and adds enabled jails to __jails
 		"""
 		opts = []
-		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
+		self.__opts = ConfigWrapper.getOptions(self, "Definition", opts)
 
 		if section is None:
 			sections = self.sections()
@@ -68,9 +70,10 @@ class JailsReader(ConfigReader):
 		for sec in sections:
 			if sec == 'INCLUDES':
 				continue
-			jail = JailReader(sec, basedir=self.getBaseDir(),
+			# use the cfg_share for filter/action caching and the same config for all 
-							  force_enable=self.__force_enable)
+			# jails (use_config=...), therefore don't read it here:
-			jail.read()
+			jail = JailReader(sec, force_enable=self.__force_enable, 
+				cfg_share=self.__cfg_share, use_config=self._cfg)
 			ret = jail.getOptions()
 			if ret:
 				if jail.isEnabled():