From d00af327c54eb57991600e5d2c61b15eb31f4c18 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 2 Oct 2014 22:29:09 +0200 Subject: [PATCH 01/36] caching of read config files, to make start of fail2ban faster, see issue #820 --- fail2ban/client/configparserinc.py | 186 ++++++++++++++++++------- fail2ban/client/configreader.py | 21 +-- fail2ban/tests/clientreadertestcase.py | 16 ++- 3 files changed, 158 insertions(+), 65 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 80b99517..ad0a255c 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -65,7 +65,136 @@ logSys = getLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] -class SafeConfigParserWithIncludes(SafeConfigParser): +class SafeConfigParserWithIncludes(object): + + SECTION_NAME = "INCLUDES" + CFG_CACHE = {} + CFG_INC_CACHE = {} + CFG_EMPY_CFG = None + + def __init__(self): + self.__cr = None + + def __check_read(self, attr): + if self.__cr is None: + # raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr) + if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None: + SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes() + self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG + + def __getattr__(self,attr): + # check we access local implementation + try: + orig_attr = self.__getattribute__(attr) + except AttributeError: + self.__check_read(attr) + orig_attr = self.__cr.__getattribute__(attr) + return orig_attr + + @staticmethod + def _resource_mtime(resource): + mt = [] + dirnames = [] + for filename in resource: + if os.path.exists(filename): + s = os.stat(filename) + mt.append(s.st_mtime) + mt.append(s.st_mode) + mt.append(s.st_size) + dirname = os.path.dirname(filename) + if dirname not in dirnames: + dirnames.append(dirname) + for dirname in dirnames: + if os.path.exists(dirname): + s = os.stat(dirname) + mt.append(s.st_mtime) + mt.append(s.st_mode) + mt.append(s.st_size) + return mt + + def read(self, resource, get_includes=True, log_info=None): + SCPWI = SafeConfigParserWithIncludes + # check includes : + fileNamesFull = [] + if not isinstance(resource, list): + resource = [ resource ] + if get_includes: + for filename in resource: + fileNamesFull += SCPWI.getIncludes(filename) + else: + fileNamesFull = resource + # check cache + hashv = '///'.join(fileNamesFull) + cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) + curmt = SCPWI._resource_mtime(fileNamesFull) + if cr is not None and mtime == curmt: + self.__cr = cr + logSys.debug("Cached config files: %s", resource) + #logSys.debug("Cached config files: %s", fileNamesFull) + return ret + # not yet in cache - create/read and add to cache: + if log_info is not None: + logSys.info(*log_info) + cr = _SafeConfigParserWithIncludes() + ret = cr.read(fileNamesFull) + SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt) + self.__cr = cr + return ret + + def getOptions(self, *args, **kwargs): + self.__check_read('getOptions') + return self.__cr.getOptions(*args, **kwargs) + + @staticmethod + def getIncludes(resource, seen = []): + """ + Given 1 config resource returns list of included files + (recursively) with the original one as well + Simple loops are taken care about + """ + + # Use a short class name ;) + SCPWI = SafeConfigParserWithIncludes + + resources = seen + [resource] + # check cache + hashv = '///'.join(resources) + cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0)) + curmt = SCPWI._resource_mtime(resources) + if cinc is not None and mtime == curmt: + return cinc + + parser = SCPWI() + try: + # read without includes + parser.read(resource, get_includes = False) + except UnicodeDecodeError, e: + logSys.error("Error decoding config file '%s': %s" % (resource, e)) + return [] + + resourceDir = os.path.dirname(resource) + + newFiles = [ ('before', []), ('after', []) ] + if SCPWI.SECTION_NAME in parser.sections(): + for option_name, option_list in newFiles: + if option_name in parser.options(SCPWI.SECTION_NAME): + newResources = parser.get(SCPWI.SECTION_NAME, option_name) + for newResource in newResources.split('\n'): + if os.path.isabs(newResource): + r = newResource + else: + r = os.path.join(resourceDir, newResource) + if r in seen: + continue + option_list += SCPWI.getIncludes(r, resources) + # combine lists + cinc = newFiles[0][1] + [resource] + newFiles[1][1] + # cache and return : + SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt) + return cinc + #print "Includes list for " + resource + " is " + `resources` + +class _SafeConfigParserWithIncludes(SafeConfigParser, object): """ Class adds functionality to SafeConfigParser to handle included other configuration files (or may be urls, whatever in the future) @@ -94,69 +223,22 @@ after = 1.conf """ - SECTION_NAME = "INCLUDES" - if sys.version_info >= (3,2): # overload constructor only for fancy new Python3's def __init__(self, *args, **kwargs): kwargs = kwargs.copy() kwargs['interpolation'] = BasicInterpolationWithName() kwargs['inline_comment_prefixes'] = ";" - super(SafeConfigParserWithIncludes, self).__init__( + super(_SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) - #@staticmethod - def getIncludes(resource, seen = []): - """ - Given 1 config resource returns list of included files - (recursively) with the original one as well - Simple loops are taken care about - """ - - # Use a short class name ;) - SCPWI = SafeConfigParserWithIncludes - - parser = SafeConfigParser() - try: - if sys.version_info >= (3,2): # pragma: no cover - parser.read(resource, encoding='utf-8') - else: - parser.read(resource) - except UnicodeDecodeError, e: - logSys.error("Error decoding config file '%s': %s" % (resource, e)) - return [] - - resourceDir = os.path.dirname(resource) - - newFiles = [ ('before', []), ('after', []) ] - if SCPWI.SECTION_NAME in parser.sections(): - for option_name, option_list in newFiles: - if option_name in parser.options(SCPWI.SECTION_NAME): - newResources = parser.get(SCPWI.SECTION_NAME, option_name) - for newResource in newResources.split('\n'): - if os.path.isabs(newResource): - r = newResource - else: - r = os.path.join(resourceDir, newResource) - if r in seen: - continue - s = seen + [resource] - option_list += SCPWI.getIncludes(r, s) - # combine lists - return newFiles[0][1] + [resource] + newFiles[1][1] - #print "Includes list for " + resource + " is " + `resources` - getIncludes = staticmethod(getIncludes) - def read(self, filenames): - fileNamesFull = [] if not isinstance(filenames, list): filenames = [ filenames ] - for filename in filenames: - fileNamesFull += SafeConfigParserWithIncludes.getIncludes(filename) - logSys.debug("Reading files: %s" % fileNamesFull) + logSys.debug("Reading files: %s", filenames) if sys.version_info >= (3,2): # pragma: no cover - return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') + return SafeConfigParser.read(self, filenames, encoding='utf-8') else: - return SafeConfigParser.read(self, fileNamesFull) + return SafeConfigParser.read(self, filenames) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 22115d3a..ada48803 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -55,7 +55,7 @@ class ConfigReader(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.info("Reading configs for %s under %s " % (basename, self._basedir)) + logSys.debug("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -71,14 +71,15 @@ class ConfigReader(SafeConfigParserWithIncludes): if len(config_files): # at least one config exists and accessible - logSys.debug("Reading config files: " + ', '.join(config_files)) - config_files_read = SafeConfigParserWithIncludes.read(self, config_files) + logSys.debug("Reading config files: %s", ', '.join(config_files)) + config_files_read = SafeConfigParserWithIncludes.read(self, config_files, + log_info=("Cache configs for %s under %s " , filename, self._basedir)) missed = [ cf for cf in config_files if cf not in config_files_read ] if missed: - logSys.error("Could not read config files: " + ', '.join(missed)) + logSys.error("Could not read config files: %s", ', '.join(missed)) if config_files_read: return True - logSys.error("Found no accessible config files for %r under %s" % + logSys.error("Found no accessible config files for %r under %s", ( filename, self.getBaseDir() )) return False else: @@ -133,12 +134,12 @@ class ConfigReader(SafeConfigParserWithIncludes): class DefinitionInitConfigReader(ConfigReader): """Config reader for files with options grouped in [Definition] and - [Init] sections. + [Init] sections. - Is a base class for readers of filters and actions, where definitions - in jails might provide custom values for options defined in [Init] - section. - """ + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0ad3c66e..b57a0562 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest +import os, glob, shutil, tempfile, unittest, time from ..client.configreader import ConfigReader from ..client.jailreader import JailReader @@ -39,6 +39,8 @@ STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') +LAST_WRITE_TIME = 0 + class ConfigReaderTest(unittest.TestCase): def setUp(self): @@ -57,7 +59,8 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - f = open("%s/%s" % (self.d, fname), "w") + fname = "%s/%s" % (self.d, fname) + f = open(fname, "w") if value is not None: f.write(""" [section] @@ -66,6 +69,14 @@ option = %s if content is not None: f.write(content) f.close() + # set modification time to another second to revalidate cache (if milliseconds not supported) : + global LAST_WRITE_TIME + mtime = os.path.getmtime(fname) + if LAST_WRITE_TIME == mtime: + mtime += 1 + os.utime(fname, (mtime, mtime)) + LAST_WRITE_TIME = mtime + def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) @@ -91,7 +102,6 @@ option = %s # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) pass - def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet self._write("c.conf", "1") From 37acc6b8323230d3f314daeeb9c233e694461de9 Mon Sep 17 00:00:00 2001 From: pacop Date: Mon, 30 Jun 2014 14:29:57 +0200 Subject: [PATCH 02/36] ENH: Add dateTime format for PortSentry Added dateTime format for PortSentry with EPOCH format --- fail2ban/tests/datedetectortestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 726e73f8..c82f92ba 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -86,6 +86,7 @@ class DateDetectorTest(unittest.TestCase): (False, "23-Jan-2005 21:59:59.02"), (False, "23-Jan-2005 21:59:59 +0100"), (False, "23-01-2005 21:59:59"), + (True, "1106513999"), # Portsetry (False, "01-23-2005 21:59:59.252"), # reported on f2b, causes Feb29 fix to break (False, "@4000000041f4104f00000000"), # TAI64N (False, "2005-01-23T20:59:59.252Z"), #ISO 8601 (UTC) From ce4f2d1c88f448b303adda8a19b6e7ae23cd8411 Mon Sep 17 00:00:00 2001 From: pacop Date: Sat, 4 Oct 2014 15:08:12 +0200 Subject: [PATCH 03/36] added filter for PortSentry with jail and samples --- config/filter.d/portsentry.conf | 10 ++++++++++ config/jail.conf | 5 +++++ fail2ban/tests/files/logs/portsentry | 4 ++++ 3 files changed, 19 insertions(+) create mode 100644 config/filter.d/portsentry.conf create mode 100644 fail2ban/tests/files/logs/portsentry diff --git a/config/filter.d/portsentry.conf b/config/filter.d/portsentry.conf new file mode 100644 index 00000000..1ee9531c --- /dev/null +++ b/config/filter.d/portsentry.conf @@ -0,0 +1,10 @@ +# Fail2Ban filter for failure attempts in Counter Strike-1.6 +# +# + +[Definition] + +failregex = \/ Port\: [0-9]+ (TCP|UDP) Blocked$ + +# Author: Pacop + diff --git a/config/jail.conf b/config/jail.conf index c42952d8..2fe3dea2 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -709,3 +709,8 @@ enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current maxretry = 6 banaction = iptables-allports + +[portsentry] +enabled = false +logpath = /var/lib/portsentry/portsentry.history +maxretry = 1 diff --git a/fail2ban/tests/files/logs/portsentry b/fail2ban/tests/files/logs/portsentry new file mode 100644 index 00000000..19c917a0 --- /dev/null +++ b/fail2ban/tests/files/logs/portsentry @@ -0,0 +1,4 @@ +# failJSON: { "time": "2014-06-27T17:51:19", "match": true , "host": "192.168.56.1" } +1403884279 - 06/27/2014 17:51:19 Host: 192.168.56.1/192.168.56.1 Port: 1 TCP Blocked +# failJSON: { "time": "2014-06-27T17:51:19", "match": true , "host": "192.168.56.1" } +1403884279 - 06/27/2014 17:51:19 Host: 192.168.56.1/192.168.56.1 Port: 1 UDP Blocked \ No newline at end of file From af4b48e841f6f2a6754c891f391e419c998819d5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 7 Oct 2014 14:07:50 +0200 Subject: [PATCH 04/36] test case for check the read of config files will be cached; --- fail2ban/tests/clientreadertestcase.py | 32 +++++++++++++++++++++++++- fail2ban/tests/utils.py | 4 ++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index b57a0562..68de2e8b 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest, time +import os, glob, shutil, tempfile, unittest, time, re from ..client.configreader import ConfigReader from ..client.jailreader import JailReader @@ -345,6 +345,36 @@ class FilterReaderTest(unittest.TestCase): self.assertRaises(ValueError, FilterReader.convert, filterReader) +class JailsReaderTestCache(LogCaptureTestCase): + + def testTestJailConfCache(self): + basedir = tempfile.mkdtemp("fail2ban_conf") + try: + shutil.rmtree(basedir) + shutil.copytree(CONFIG_DIR, basedir) + shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local') + shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') + + # read whole configuration like a file2ban-client ... + configurator = Configurator() + configurator.setBaseDir(basedir) + configurator.readEarly() + configurator.getEarlyOptions() + configurator.readAll() + # from here we test a cache : + self.assertTrue(configurator.getOptions(None)) + cnt = 0 + for s in self.getLog().rsplit('\n'): + if re.match(r"^Reading files: .*jail.local", s): + cnt += 1 + # if cnt > 2: + # self.printLog() + self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt) + self.assertFalse(cnt <= 0) + finally: + shutil.rmtree(basedir) + + class JailsReaderTest(LogCaptureTestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 912c5a90..ad504703 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -111,6 +111,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTestCache)) # CSocket and AsyncServer tests.addTest(unittest.makeSuite(sockettestcase.Socket)) # Misc helpers @@ -216,5 +217,8 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() + def getLog(self): + return self._log.getvalue() + def printLog(self): print(self._log.getvalue()) From 2a54e612382609495b50579c2a75cfb8999aaa8e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Oct 2014 15:44:32 +0200 Subject: [PATCH 05/36] config cache optimized - prevent to read the same config file inside different resources multiple times; test case: read jail file only once; --- fail2ban/client/configparserinc.py | 40 ++++++++++++++++++++++++-- fail2ban/tests/clientreadertestcase.py | 6 ++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index ad0a255c..7aac111b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -124,7 +124,7 @@ class SafeConfigParserWithIncludes(object): else: fileNamesFull = resource # check cache - hashv = '///'.join(fileNamesFull) + hashv = '\x01'.join(fileNamesFull) cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) curmt = SCPWI._resource_mtime(fileNamesFull) if cr is not None and mtime == curmt: @@ -167,7 +167,7 @@ class SafeConfigParserWithIncludes(object): parser = SCPWI() try: # read without includes - parser.read(resource, get_includes = False) + parser.read(resource, get_includes=False) except UnicodeDecodeError, e: logSys.error("Error decoding config file '%s': %s" % (resource, e)) return [] @@ -232,11 +232,45 @@ after = 1.conf super(_SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) + def get_defaults(self): + return self._defaults + + def get_sections(self): + return self._sections def read(self, filenames): if not isinstance(filenames, list): filenames = [ filenames ] - logSys.debug("Reading files: %s", filenames) + if len(filenames) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in filenames: + # read single one, add to return list: + cfg = SafeConfigParserWithIncludes() + i = cfg.read(filename, get_includes=False) + if i: + ret += i + # merge defaults and all sections to self: + for (n, v) in cfg.get_defaults().items(): + alld[n] = v + for (n, s) in cfg.get_sections().items(): + if isinstance(s, dict): + s2 = alls.get(n) + if s2 is not None: + for (n, v) in s.items(): + s2[n] = v + else: + s2 = s.copy() + alls[n] = s2 + else: + alls[n] = s + + return ret + + # read one config : + logSys.debug("Reading file: %s", filenames[0]) if sys.version_info >= (3,2): # pragma: no cover return SafeConfigParser.read(self, filenames, encoding='utf-8') else: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 68de2e8b..5c5c75bc 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -365,11 +365,11 @@ class JailsReaderTestCache(LogCaptureTestCase): self.assertTrue(configurator.getOptions(None)) cnt = 0 for s in self.getLog().rsplit('\n'): - if re.match(r"^Reading files: .*jail.local", s): + if re.match(r"^Reading files?: .*jail.local", s): cnt += 1 - # if cnt > 2: + # if cnt > 1: # self.printLog() - self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt) + self.assertFalse(cnt > 1, "Too many times reading of config files, cnt = %s" % cnt) self.assertFalse(cnt <= 0) finally: shutil.rmtree(basedir) From 4244c87802f0cc88072a7bac3574d1fb1377eb3d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Oct 2014 14:51:08 +0200 Subject: [PATCH 06/36] ConfigWrapper class introduced: sharing of the same ConfigReader object between JailsReader and JailReader (don't read jail config each jail); sharing of the same DefinitionInitConfigReader (ActionReader, FilterReader) between all jails using that; cache of read a config files was optimized; test case extended for all types of config readers; --- fail2ban/client/actionreader.py | 12 +++-- fail2ban/client/configparserinc.py | 13 ++--- fail2ban/client/configreader.py | 75 ++++++++++++++++++++++---- fail2ban/client/configurator.py | 4 +- fail2ban/client/fail2banreader.py | 13 ++--- fail2ban/client/filterreader.py | 10 ++-- fail2ban/client/jailreader.py | 20 +++---- fail2ban/client/jailsreader.py | 19 ++++--- fail2ban/tests/clientreadertestcase.py | 49 ++++++++++++----- 9 files changed, 153 insertions(+), 62 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 91f3322b..8022ecc1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..helpers import getLogger # Gets the instance of the logger. @@ -47,15 +47,19 @@ class ActionReader(DefinitionInitConfigReader): DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("action.d", fileName)) + + def getFile(self): + return self.__file + def setName(self, name): self._name = name def getName(self): return self._name - def read(self): - return ConfigReader.read(self, os.path.join("action.d", self._file)) - def convert(self): head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7aac111b..31945849 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -253,17 +253,14 @@ after = 1.conf if i: ret += i # merge defaults and all sections to self: - for (n, v) in cfg.get_defaults().items(): - alld[n] = v - for (n, s) in cfg.get_sections().items(): + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): if isinstance(s, dict): s2 = alls.get(n) - if s2 is not None: - for (n, v) in s.items(): - s2[n] = v + if isinstance(s2, dict): + s2.update(s) else: - s2 = s.copy() - alls[n] = s2 + alls[n] = s.copy() else: alls[n] = s diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index ada48803..d1cc7924 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -32,6 +32,65 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = 6 + +class ConfigWrapper(): + + def __init__(self, use_config=None, share_config=None, **kwargs): + # use given shared config if possible (see read): + self._cfg_share = None + self._cfg = None + if use_config is not None: + self._cfg = use_config + else: + # share config if possible: + if share_config is not None: + self._cfg_share = share_config + self._cfg_share_kwargs = kwargs + else: + self._cfg = ConfigReader(**kwargs) + + def setBaseDir(self, basedir): + self._cfg.setBaseDir(basedir) + + def getBaseDir(self): + return self._cfg.getBaseDir() + + def read(self, name, once=True): + # shared ? + if not self._cfg and self._cfg_share is not None: + self._cfg = self._cfg_share.get(name) + if not self._cfg: + self._cfg = ConfigReader(**self._cfg_share_kwargs) + self._cfg_share[name] = self._cfg + # performance feature - read once if using shared config reader: + rc = self._cfg.read_cfg_files + if once and rc.get(name) is not None: + return rc.get(name) + + # read: + ret = self._cfg.read(name) + + # save already read: + if once: + rc[name] = ret + return ret + + def sections(self): + return self._cfg.sections() + + def has_section(self, sec): + return self._cfg.has_section(sec) + + def options(self, *args): + return self._cfg.options(*args) + + def get(self, sec, opt): + return self._cfg.get(sec, opt) + + def getOptions(self, *args, **kwargs): + return self._cfg.getOptions(*args, **kwargs) + class ConfigReader(SafeConfigParserWithIncludes): @@ -39,8 +98,8 @@ class ConfigReader(SafeConfigParserWithIncludes): def __init__(self, basedir=None): SafeConfigParserWithIncludes.__init__(self) + self.read_cfg_files = dict() self.setBaseDir(basedir) - self.__opts = None def setBaseDir(self, basedir): if basedir is None: @@ -122,17 +181,15 @@ class ConfigReader(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] - else: - logSys.debug( - "Non essential option '%s' not defined in '%s'.", - option[1], sec) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values -class DefinitionInitConfigReader(ConfigReader): +class DefinitionInitConfigReader(ConfigWrapper): """Config reader for files with options grouped in [Definition] and [Init] sections. @@ -144,7 +201,7 @@ class DefinitionInitConfigReader(ConfigReader): _configOpts = [] def __init__(self, file_, jailName, initOpts, **kwargs): - ConfigReader.__init__(self, **kwargs) + ConfigWrapper.__init__(self, **kwargs) self.setFile(file_) self.setJailName(jailName) self._initOpts = initOpts @@ -163,14 +220,14 @@ class DefinitionInitConfigReader(ConfigReader): return self._jailName def read(self): - return ConfigReader.read(self, self._file) + return ConfigWrapper.read(self, self._file) # needed for fail2ban-regex that doesn't need fancy directories def readexplicit(self): return SafeConfigParserWithIncludes.read(self, self._file) def getOptions(self, pOpts): - self._opts = ConfigReader.getOptions( + self._opts = ConfigWrapper.getOptions( self, "Definition", self._configOpts, pOpts) if self.has_section("Init"): diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 0dd9f955..a29fe94d 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,11 +33,11 @@ logSys = getLogger(__name__) class Configurator: - def __init__(self): + def __init__(self, force_enable=False): self.__settings = dict() self.__streams = dict() self.__fail2ban = Fail2banReader() - self.__jails = JailsReader() + self.__jails = JailsReader(force_enable=force_enable) def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 361a5e54..a151ebf0 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,31 +24,32 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigWrapper from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class Fail2banReader(ConfigReader): +class Fail2banReader(ConfigWrapper): def __init__(self, **kwargs): - ConfigReader.__init__(self, **kwargs) + self.__opts = None + ConfigWrapper.__init__(self, **kwargs) def read(self): - ConfigReader.read(self, "fail2ban") + ConfigWrapper.read(self, "fail2ban") def getEarlyOptions(self): opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"], ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] - return ConfigReader.getOptions(self, "Definition", opts) + return ConfigWrapper.getOptions(self, "Definition", opts) def getOptions(self): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbpurgeage", 86400]] - self.__opts = ConfigReader.getOptions(self, "Definition", opts) + self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) def convert(self): stream = list() diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 40d74c45..fe657025 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -26,7 +26,7 @@ __license__ = "GPL" import os, shlex -from .configreader import ConfigReader, DefinitionInitConfigReader +from .configreader import DefinitionInitConfigReader from ..server.action import CommandAction from ..helpers import getLogger @@ -40,8 +40,12 @@ class FilterReader(DefinitionInitConfigReader): ["string", "failregex", ""], ] - def read(self): - return ConfigReader.read(self, os.path.join("filter.d", self._file)) + def setFile(self, fileName): + self.__file = fileName + DefinitionInitConfigReader.setFile(self, os.path.join("filter.d", fileName)) + + def getFile(self): + return self.__file def convert(self): stream = list() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index a4bf5174..a3cc939c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import re, glob, os.path import json -from .configreader import ConfigReader +from .configreader import ConfigReader, ConfigWrapper from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger @@ -35,17 +35,19 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailReader(ConfigReader): +class JailReader(ConfigWrapper): optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") optionExtractRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') - def __init__(self, name, force_enable=False, **kwargs): - ConfigReader.__init__(self, **kwargs) + def __init__(self, name, force_enable=False, cfg_share=None, **kwargs): + # use shared config if possible: + ConfigWrapper.__init__(self, **kwargs) self.__name = name self.__filter = None self.__force_enable = force_enable + self.__cfg_share = cfg_share self.__actions = list() self.__opts = None @@ -60,7 +62,7 @@ class JailReader(ConfigReader): return self.__name def read(self): - out = ConfigReader.read(self, "jail") + out = ConfigWrapper.read(self, "jail") # Before returning -- verify that requested section # exists at all if not (self.__name in self.sections()): @@ -101,7 +103,7 @@ class JailReader(ConfigReader): ["string", "ignoreip", None], ["string", "filter", ""], ["string", "action", ""]] - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + self.__opts = ConfigWrapper.getOptions(self, self.__name, opts) if not self.__opts: return False @@ -111,7 +113,7 @@ class JailReader(ConfigReader): filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( - filterName, self.__name, filterOpt, basedir=self.getBaseDir()) + filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -141,7 +143,7 @@ class JailReader(ConfigReader): else: action = ActionReader( actName, self.__name, actOpt, - basedir=self.getBaseDir()) + share_config=self.__cfg_share, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -213,7 +215,7 @@ class JailReader(ConfigReader): if self.__filter: stream.extend(self.__filter.convert()) for action in self.__actions: - if isinstance(action, ConfigReader): + if isinstance(action, (ConfigReader, ConfigWrapper)): stream.extend(action.convert()) else: stream.append(action) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 84c614b9..37d37e01 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader +from .configreader import ConfigReader, ConfigWrapper from .jailreader import JailReader from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailsReader(ConfigReader): +class JailsReader(ConfigWrapper): def __init__(self, force_enable=False, **kwargs): """ @@ -41,7 +41,9 @@ class JailsReader(ConfigReader): Passed to JailReader to force enable the jails. It is for internal use """ - ConfigReader.__init__(self, **kwargs) + # use shared config if possible: + ConfigWrapper.__init__(self, **kwargs) + self.__cfg_share = dict() self.__jails = list() self.__force_enable = force_enable @@ -50,13 +52,13 @@ class JailsReader(ConfigReader): return self.__jails def read(self): - return ConfigReader.read(self, "jail") + return ConfigWrapper.read(self, "jail") def getOptions(self, section=None): """Reads configuration for jail(s) and adds enabled jails to __jails """ opts = [] - self.__opts = ConfigReader.getOptions(self, "Definition", opts) + self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) if section is None: sections = self.sections() @@ -68,9 +70,10 @@ class JailsReader(ConfigReader): for sec in sections: if sec == 'INCLUDES': continue - jail = JailReader(sec, basedir=self.getBaseDir(), - force_enable=self.__force_enable) - jail.read() + # use the cfg_share for filter/action caching and the same config for all + # jails (use_config=...), therefore don't read it here: + jail = JailReader(sec, force_enable=self.__force_enable, + cfg_share=self.__cfg_share, use_config=self._cfg) ret = jail.getOptions() if ret: if jail.isEnabled(): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 5c5c75bc..a5fe7b6e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -347,6 +347,23 @@ class FilterReaderTest(unittest.TestCase): class JailsReaderTestCache(LogCaptureTestCase): + def _readWholeConf(self, basedir, force_enable=False): + # read whole configuration like a file2ban-client ... + configurator = Configurator(force_enable=force_enable) + configurator.setBaseDir(basedir) + configurator.readEarly() + configurator.getEarlyOptions() + configurator.readAll() + # from here we test a cache with all includes / before / after : + self.assertTrue(configurator.getOptions(None)) + + def _getLoggedReadCount(self, filematch): + cnt = 0 + for s in self.getLog().rsplit('\n'): + if re.match(r"^Reading files?: .*/"+filematch, s): + cnt += 1 + return cnt + def testTestJailConfCache(self): basedir = tempfile.mkdtemp("fail2ban_conf") try: @@ -356,21 +373,27 @@ class JailsReaderTestCache(LogCaptureTestCase): shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') # read whole configuration like a file2ban-client ... - configurator = Configurator() - configurator.setBaseDir(basedir) - configurator.readEarly() - configurator.getEarlyOptions() - configurator.readAll() - # from here we test a cache : - self.assertTrue(configurator.getOptions(None)) - cnt = 0 - for s in self.getLog().rsplit('\n'): - if re.match(r"^Reading files?: .*jail.local", s): - cnt += 1 + self._readWholeConf(basedir) + # how many times jail.local was read: + cnt = self._getLoggedReadCount('jail.local') # if cnt > 1: # self.printLog() - self.assertFalse(cnt > 1, "Too many times reading of config files, cnt = %s" % cnt) - self.assertFalse(cnt <= 0) + self.assertFalse(cnt > 1, "Too many times reading of jail files, cnt = %s" % cnt) + self.assertNotEqual(cnt, 0) + + # read whole configuration like a file2ban-client, again ... + # but this time force enable all jails, to check filter and action cached also: + self._readWholeConf(basedir, force_enable=True) + cnt = self._getLoggedReadCount(r'jail\.local') + # still one (no more reads): + self.assertFalse(cnt > 1, "Too many times second reading of jail files, cnt = %s" % cnt) + + # same with filter: + cnt = self._getLoggedReadCount(r'filter\.d/common\.conf') + self.assertFalse(cnt > 1, "Too many times reading of filter files, cnt = %s" % cnt) + # same with action: + cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') + self.assertFalse(cnt > 1, "Too many times reading of action files, cnt = %s" % cnt) finally: shutil.rmtree(basedir) From 51cae63bf0cbdcfe1bdc142a65166e2002101b50 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Oct 2014 15:39:58 +0200 Subject: [PATCH 07/36] more precise by test --- fail2ban/tests/clientreadertestcase.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index a5fe7b6e..a24856da 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -378,22 +378,21 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt = self._getLoggedReadCount('jail.local') # if cnt > 1: # self.printLog() - self.assertFalse(cnt > 1, "Too many times reading of jail files, cnt = %s" % cnt) - self.assertNotEqual(cnt, 0) + self.assertTrue(cnt == 1, "Unexpected count by reading of jail files, cnt = %s" % cnt) # read whole configuration like a file2ban-client, again ... # but this time force enable all jails, to check filter and action cached also: self._readWholeConf(basedir, force_enable=True) cnt = self._getLoggedReadCount(r'jail\.local') # still one (no more reads): - self.assertFalse(cnt > 1, "Too many times second reading of jail files, cnt = %s" % cnt) + self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt) # same with filter: cnt = self._getLoggedReadCount(r'filter\.d/common\.conf') - self.assertFalse(cnt > 1, "Too many times reading of filter files, cnt = %s" % cnt) + self.assertTrue(cnt == 1, "Unexpected count by reading of filter files, cnt = %s" % cnt) # same with action: cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') - self.assertFalse(cnt > 1, "Too many times reading of action files, cnt = %s" % cnt) + self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: shutil.rmtree(basedir) From f31607ded102c8fb8edca29b16af5994f78b999c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 7 Oct 2014 14:07:50 +0200 Subject: [PATCH 08/36] test case for check the read of config files will be cached; Conflicts: fail2ban/tests/clientreadertestcase.py -- removed not needed time in imports --- fail2ban/tests/clientreadertestcase.py | 32 +++++++++++++++++++++++++- fail2ban/tests/utils.py | 4 ++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0ad3c66e..f65ef87a 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest +import os, glob, shutil, tempfile, unittest, re from ..client.configreader import ConfigReader from ..client.jailreader import JailReader @@ -335,6 +335,36 @@ class FilterReaderTest(unittest.TestCase): self.assertRaises(ValueError, FilterReader.convert, filterReader) +class JailsReaderTestCache(LogCaptureTestCase): + + def testTestJailConfCache(self): + basedir = tempfile.mkdtemp("fail2ban_conf") + try: + shutil.rmtree(basedir) + shutil.copytree(CONFIG_DIR, basedir) + shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local') + shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') + + # read whole configuration like a file2ban-client ... + configurator = Configurator() + configurator.setBaseDir(basedir) + configurator.readEarly() + configurator.getEarlyOptions() + configurator.readAll() + # from here we test a cache : + self.assertTrue(configurator.getOptions(None)) + cnt = 0 + for s in self.getLog().rsplit('\n'): + if re.match(r"^Reading files: .*jail.local", s): + cnt += 1 + # if cnt > 2: + # self.printLog() + self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt) + self.assertFalse(cnt <= 0) + finally: + shutil.rmtree(basedir) + + class JailsReaderTest(LogCaptureTestCase): def testProvidingBadBasedir(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 912c5a90..ad504703 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -111,6 +111,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest)) + tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTestCache)) # CSocket and AsyncServer tests.addTest(unittest.makeSuite(sockettestcase.Socket)) # Misc helpers @@ -216,5 +217,8 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() + def getLog(self): + return self._log.getvalue() + def printLog(self): print(self._log.getvalue()) From 0c5f11079c6d1cb817a4218f707be37cd1897398 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 9 Oct 2014 10:46:17 -0400 Subject: [PATCH 09/36] ENH: keep spitting out logging to the screen in LogCaptureTestCases if HEAVYDEBUG --- fail2ban/tests/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index ad504703..b56c0988 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -25,6 +25,7 @@ __license__ = "GPL" import logging import os import re +import sys import time import unittest from StringIO import StringIO @@ -205,6 +206,8 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] + if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! + logSys.handlers += self._old_handlers logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): From b62ce14ccd4dd0307264493122caac824c0a9e49 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Oct 2014 18:00:45 +0200 Subject: [PATCH 10/36] Partially merge remote-tracking from 'sebres:cache-config-read-820': test cases extended, configurator.py adapted for test case. --- fail2ban/client/configurator.py | 4 +- fail2ban/tests/clientreadertestcase.py | 66 +++++++++++++++++++------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 0dd9f955..a29fe94d 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,11 +33,11 @@ logSys = getLogger(__name__) class Configurator: - def __init__(self): + def __init__(self, force_enable=False): self.__settings = dict() self.__streams = dict() self.__fail2ban = Fail2banReader() - self.__jails = JailsReader() + self.__jails = JailsReader(force_enable=force_enable) def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index f65ef87a..a24856da 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest, re +import os, glob, shutil, tempfile, unittest, time, re from ..client.configreader import ConfigReader from ..client.jailreader import JailReader @@ -39,6 +39,8 @@ STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') +LAST_WRITE_TIME = 0 + class ConfigReaderTest(unittest.TestCase): def setUp(self): @@ -57,7 +59,8 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - f = open("%s/%s" % (self.d, fname), "w") + fname = "%s/%s" % (self.d, fname) + f = open(fname, "w") if value is not None: f.write(""" [section] @@ -66,6 +69,14 @@ option = %s if content is not None: f.write(content) f.close() + # set modification time to another second to revalidate cache (if milliseconds not supported) : + global LAST_WRITE_TIME + mtime = os.path.getmtime(fname) + if LAST_WRITE_TIME == mtime: + mtime += 1 + os.utime(fname, (mtime, mtime)) + LAST_WRITE_TIME = mtime + def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) @@ -91,7 +102,6 @@ option = %s # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) pass - def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet self._write("c.conf", "1") @@ -337,6 +347,23 @@ class FilterReaderTest(unittest.TestCase): class JailsReaderTestCache(LogCaptureTestCase): + def _readWholeConf(self, basedir, force_enable=False): + # read whole configuration like a file2ban-client ... + configurator = Configurator(force_enable=force_enable) + configurator.setBaseDir(basedir) + configurator.readEarly() + configurator.getEarlyOptions() + configurator.readAll() + # from here we test a cache with all includes / before / after : + self.assertTrue(configurator.getOptions(None)) + + def _getLoggedReadCount(self, filematch): + cnt = 0 + for s in self.getLog().rsplit('\n'): + if re.match(r"^Reading files?: .*/"+filematch, s): + cnt += 1 + return cnt + def testTestJailConfCache(self): basedir = tempfile.mkdtemp("fail2ban_conf") try: @@ -346,21 +373,26 @@ class JailsReaderTestCache(LogCaptureTestCase): shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') # read whole configuration like a file2ban-client ... - configurator = Configurator() - configurator.setBaseDir(basedir) - configurator.readEarly() - configurator.getEarlyOptions() - configurator.readAll() - # from here we test a cache : - self.assertTrue(configurator.getOptions(None)) - cnt = 0 - for s in self.getLog().rsplit('\n'): - if re.match(r"^Reading files: .*jail.local", s): - cnt += 1 - # if cnt > 2: + self._readWholeConf(basedir) + # how many times jail.local was read: + cnt = self._getLoggedReadCount('jail.local') + # if cnt > 1: # self.printLog() - self.assertFalse(cnt > 2, "Too many times reading of config files, cnt = %s" % cnt) - self.assertFalse(cnt <= 0) + self.assertTrue(cnt == 1, "Unexpected count by reading of jail files, cnt = %s" % cnt) + + # read whole configuration like a file2ban-client, again ... + # but this time force enable all jails, to check filter and action cached also: + self._readWholeConf(basedir, force_enable=True) + cnt = self._getLoggedReadCount(r'jail\.local') + # still one (no more reads): + self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt) + + # same with filter: + cnt = self._getLoggedReadCount(r'filter\.d/common\.conf') + self.assertTrue(cnt == 1, "Unexpected count by reading of filter files, cnt = %s" % cnt) + # same with action: + cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') + self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: shutil.rmtree(basedir) From f67053c2ece28888f38c4f19d19c92e4cbfa91a1 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Oct 2014 19:01:49 +0200 Subject: [PATCH 11/36] ConfigReader/ConfigWrapper renamed as suggested from @yarikoptic; + code clarifying (suggested also); --- fail2ban/client/configparserinc.py | 6 ++--- fail2ban/client/configreader.py | 33 ++++++++++++++++---------- fail2ban/client/fail2banreader.py | 12 +++++----- fail2ban/client/jailreader.py | 12 +++++----- fail2ban/client/jailsreader.py | 10 ++++---- fail2ban/tests/clientreadertestcase.py | 6 ++--- 6 files changed, 44 insertions(+), 35 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 31945849..25e15213 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -92,7 +92,7 @@ class SafeConfigParserWithIncludes(object): return orig_attr @staticmethod - def _resource_mtime(resource): + def _get_resource_fingerprint(resource): mt = [] dirnames = [] for filename in resource: @@ -126,7 +126,7 @@ class SafeConfigParserWithIncludes(object): # check cache hashv = '\x01'.join(fileNamesFull) cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) - curmt = SCPWI._resource_mtime(fileNamesFull) + curmt = SCPWI._get_resource_fingerprint(fileNamesFull) if cr is not None and mtime == curmt: self.__cr = cr logSys.debug("Cached config files: %s", resource) @@ -160,7 +160,7 @@ class SafeConfigParserWithIncludes(object): # check cache hashv = '///'.join(resources) cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0)) - curmt = SCPWI._resource_mtime(resources) + curmt = SCPWI._get_resource_fingerprint(resources) if cinc is not None and mtime == curmt: return cinc diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d1cc7924..a3b5645e 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -32,9 +32,13 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -logLevel = 6 +_logLevel = 6 -class ConfigWrapper(): +class ConfigReader(): + """Config reader class (previously ConfigWrapper). + + Automatically shares or use already shared instance of ConfigReaderUnshared. + """ def __init__(self, use_config=None, share_config=None, **kwargs): # use given shared config if possible (see read): @@ -48,7 +52,7 @@ class ConfigWrapper(): self._cfg_share = share_config self._cfg_share_kwargs = kwargs else: - self._cfg = ConfigReader(**kwargs) + self._cfg = ConfigReaderUnshared(**kwargs) def setBaseDir(self, basedir): self._cfg.setBaseDir(basedir) @@ -61,7 +65,7 @@ class ConfigWrapper(): if not self._cfg and self._cfg_share is not None: self._cfg = self._cfg_share.get(name) if not self._cfg: - self._cfg = ConfigReader(**self._cfg_share_kwargs) + self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) self._cfg_share[name] = self._cfg # performance feature - read once if using shared config reader: rc = self._cfg.read_cfg_files @@ -92,7 +96,12 @@ class ConfigWrapper(): return self._cfg.getOptions(*args, **kwargs) -class ConfigReader(SafeConfigParserWithIncludes): +class ConfigReaderUnshared(SafeConfigParserWithIncludes): + """Unshared config reader (previously ConfigReader). + + Does not use this class (internal not shared/cached represenation). + Use ConfigReader instead. + """ DEFAULT_BASEDIR = '/etc/fail2ban' @@ -103,7 +112,7 @@ class ConfigReader(SafeConfigParserWithIncludes): def setBaseDir(self, basedir): if basedir is None: - basedir = ConfigReader.DEFAULT_BASEDIR # stock system location + basedir = ConfigReaderUnshared.DEFAULT_BASEDIR # stock system location self._basedir = basedir.rstrip('/') def getBaseDir(self): @@ -181,15 +190,15 @@ class ConfigReader(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] - elif logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) + elif logSys.getEffectiveLevel() <= _logLevel: + logSys.log(_logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") values[option[1]] = option[2] return values -class DefinitionInitConfigReader(ConfigWrapper): +class DefinitionInitConfigReader(ConfigReader): """Config reader for files with options grouped in [Definition] and [Init] sections. @@ -201,7 +210,7 @@ class DefinitionInitConfigReader(ConfigWrapper): _configOpts = [] def __init__(self, file_, jailName, initOpts, **kwargs): - ConfigWrapper.__init__(self, **kwargs) + ConfigReader.__init__(self, **kwargs) self.setFile(file_) self.setJailName(jailName) self._initOpts = initOpts @@ -220,14 +229,14 @@ class DefinitionInitConfigReader(ConfigWrapper): return self._jailName def read(self): - return ConfigWrapper.read(self, self._file) + return ConfigReader.read(self, self._file) # needed for fail2ban-regex that doesn't need fancy directories def readexplicit(self): return SafeConfigParserWithIncludes.read(self, self._file) def getOptions(self, pOpts): - self._opts = ConfigWrapper.getOptions( + self._opts = ConfigReader.getOptions( self, "Definition", self._configOpts, pOpts) if self.has_section("Init"): diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index a151ebf0..4d46c156 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,32 +24,32 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigWrapper +from .configreader import ConfigReader from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class Fail2banReader(ConfigWrapper): +class Fail2banReader(ConfigReader): def __init__(self, **kwargs): self.__opts = None - ConfigWrapper.__init__(self, **kwargs) + ConfigReader.__init__(self, **kwargs) def read(self): - ConfigWrapper.read(self, "fail2ban") + ConfigReader.read(self, "fail2ban") def getEarlyOptions(self): opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"], ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] - return ConfigWrapper.getOptions(self, "Definition", opts) + return ConfigReader.getOptions(self, "Definition", opts) def getOptions(self): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbpurgeage", 86400]] - self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) + self.__opts = ConfigReader.getOptions(self, "Definition", opts) def convert(self): stream = list() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index a3cc939c..e4394e59 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -27,7 +27,7 @@ __license__ = "GPL" import re, glob, os.path import json -from .configreader import ConfigReader, ConfigWrapper +from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger @@ -35,7 +35,7 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailReader(ConfigWrapper): +class JailReader(ConfigReader): optionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") optionExtractRE = re.compile( @@ -43,7 +43,7 @@ class JailReader(ConfigWrapper): def __init__(self, name, force_enable=False, cfg_share=None, **kwargs): # use shared config if possible: - ConfigWrapper.__init__(self, **kwargs) + ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None self.__force_enable = force_enable @@ -62,7 +62,7 @@ class JailReader(ConfigWrapper): return self.__name def read(self): - out = ConfigWrapper.read(self, "jail") + out = ConfigReader.read(self, "jail") # Before returning -- verify that requested section # exists at all if not (self.__name in self.sections()): @@ -103,7 +103,7 @@ class JailReader(ConfigWrapper): ["string", "ignoreip", None], ["string", "filter", ""], ["string", "action", ""]] - self.__opts = ConfigWrapper.getOptions(self, self.__name, opts) + self.__opts = ConfigReader.getOptions(self, self.__name, opts) if not self.__opts: return False @@ -215,7 +215,7 @@ class JailReader(ConfigWrapper): if self.__filter: stream.extend(self.__filter.convert()) for action in self.__actions: - if isinstance(action, (ConfigReader, ConfigWrapper)): + if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) else: stream.append(action) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 37d37e01..3bb713f6 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from .configreader import ConfigReader, ConfigWrapper +from .configreader import ConfigReader from .jailreader import JailReader from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -class JailsReader(ConfigWrapper): +class JailsReader(ConfigReader): def __init__(self, force_enable=False, **kwargs): """ @@ -42,7 +42,7 @@ class JailsReader(ConfigWrapper): It is for internal use """ # use shared config if possible: - ConfigWrapper.__init__(self, **kwargs) + ConfigReader.__init__(self, **kwargs) self.__cfg_share = dict() self.__jails = list() self.__force_enable = force_enable @@ -52,13 +52,13 @@ class JailsReader(ConfigWrapper): return self.__jails def read(self): - return ConfigWrapper.read(self, "jail") + return ConfigReader.read(self, "jail") def getOptions(self, section=None): """Reads configuration for jail(s) and adds enabled jails to __jails """ opts = [] - self.__opts = ConfigWrapper.getOptions(self, "Definition", opts) + self.__opts = ConfigReader.getOptions(self, "Definition", opts) if section is None: sections = self.sections() diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index a24856da..0aaa8890 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,9 +21,9 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest, time, re +import os, glob, shutil, tempfile, unittest, re -from ..client.configreader import ConfigReader +from ..client.configreader import ConfigReaderUnshared from ..client.jailreader import JailReader from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader @@ -46,7 +46,7 @@ class ConfigReaderTest(unittest.TestCase): def setUp(self): """Call before every test case.""" self.d = tempfile.mkdtemp(prefix="f2b-temp") - self.c = ConfigReader(basedir=self.d) + self.c = ConfigReaderUnshared(basedir=self.d) def tearDown(self): """Call after every test case.""" From 37952ab75f4cb34ffa9180a5a35344a2dfa53475 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Oct 2014 19:51:53 +0200 Subject: [PATCH 12/36] code review --- fail2ban/client/configreader.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index a3b5645e..096f8e78 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -35,9 +35,9 @@ logSys = getLogger(__name__) _logLevel = 6 class ConfigReader(): - """Config reader class (previously ConfigWrapper). + """Generic config reader class. - Automatically shares or use already shared instance of ConfigReaderUnshared. + A caching adapter which automatically reuses already shared configuration. """ def __init__(self, use_config=None, share_config=None, **kwargs): @@ -200,12 +200,12 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): class DefinitionInitConfigReader(ConfigReader): """Config reader for files with options grouped in [Definition] and - [Init] sections. + [Init] sections. - Is a base class for readers of filters and actions, where definitions - in jails might provide custom values for options defined in [Init] - section. - """ + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] From c35b4b24d21445690e4c51a335adf270136bd7d3 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 02:10:13 +0200 Subject: [PATCH 13/36] rewritten caching resp. sharing of ConfigReader and SafeConfigParserWithIncludes (v.2, first and second level cache, without fingerprinting etc.); --- fail2ban/client/configparserinc.py | 296 +++++++++++-------------- fail2ban/client/configreader.py | 43 ++-- fail2ban/client/configurator.py | 6 +- fail2ban/client/jailreader.py | 7 +- fail2ban/client/jailsreader.py | 3 +- fail2ban/tests/clientreadertestcase.py | 27 +-- 6 files changed, 178 insertions(+), 204 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 25e15213..d6ad4e34 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -65,136 +65,7 @@ logSys = getLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] -class SafeConfigParserWithIncludes(object): - - SECTION_NAME = "INCLUDES" - CFG_CACHE = {} - CFG_INC_CACHE = {} - CFG_EMPY_CFG = None - - def __init__(self): - self.__cr = None - - def __check_read(self, attr): - if self.__cr is None: - # raise RuntimeError("Access to wrapped attribute \"%s\" before read call" % attr) - if SafeConfigParserWithIncludes.CFG_EMPY_CFG is None: - SafeConfigParserWithIncludes.CFG_EMPY_CFG = _SafeConfigParserWithIncludes() - self.__cr = SafeConfigParserWithIncludes.CFG_EMPY_CFG - - def __getattr__(self,attr): - # check we access local implementation - try: - orig_attr = self.__getattribute__(attr) - except AttributeError: - self.__check_read(attr) - orig_attr = self.__cr.__getattribute__(attr) - return orig_attr - - @staticmethod - def _get_resource_fingerprint(resource): - mt = [] - dirnames = [] - for filename in resource: - if os.path.exists(filename): - s = os.stat(filename) - mt.append(s.st_mtime) - mt.append(s.st_mode) - mt.append(s.st_size) - dirname = os.path.dirname(filename) - if dirname not in dirnames: - dirnames.append(dirname) - for dirname in dirnames: - if os.path.exists(dirname): - s = os.stat(dirname) - mt.append(s.st_mtime) - mt.append(s.st_mode) - mt.append(s.st_size) - return mt - - def read(self, resource, get_includes=True, log_info=None): - SCPWI = SafeConfigParserWithIncludes - # check includes : - fileNamesFull = [] - if not isinstance(resource, list): - resource = [ resource ] - if get_includes: - for filename in resource: - fileNamesFull += SCPWI.getIncludes(filename) - else: - fileNamesFull = resource - # check cache - hashv = '\x01'.join(fileNamesFull) - cr, ret, mtime = SCPWI.CFG_CACHE.get(hashv, (None, False, 0)) - curmt = SCPWI._get_resource_fingerprint(fileNamesFull) - if cr is not None and mtime == curmt: - self.__cr = cr - logSys.debug("Cached config files: %s", resource) - #logSys.debug("Cached config files: %s", fileNamesFull) - return ret - # not yet in cache - create/read and add to cache: - if log_info is not None: - logSys.info(*log_info) - cr = _SafeConfigParserWithIncludes() - ret = cr.read(fileNamesFull) - SCPWI.CFG_CACHE[hashv] = (cr, ret, curmt) - self.__cr = cr - return ret - - def getOptions(self, *args, **kwargs): - self.__check_read('getOptions') - return self.__cr.getOptions(*args, **kwargs) - - @staticmethod - def getIncludes(resource, seen = []): - """ - Given 1 config resource returns list of included files - (recursively) with the original one as well - Simple loops are taken care about - """ - - # Use a short class name ;) - SCPWI = SafeConfigParserWithIncludes - - resources = seen + [resource] - # check cache - hashv = '///'.join(resources) - cinc, mtime = SCPWI.CFG_INC_CACHE.get(hashv, (None, 0)) - curmt = SCPWI._get_resource_fingerprint(resources) - if cinc is not None and mtime == curmt: - return cinc - - parser = SCPWI() - try: - # read without includes - parser.read(resource, get_includes=False) - except UnicodeDecodeError, e: - logSys.error("Error decoding config file '%s': %s" % (resource, e)) - return [] - - resourceDir = os.path.dirname(resource) - - newFiles = [ ('before', []), ('after', []) ] - if SCPWI.SECTION_NAME in parser.sections(): - for option_name, option_list in newFiles: - if option_name in parser.options(SCPWI.SECTION_NAME): - newResources = parser.get(SCPWI.SECTION_NAME, option_name) - for newResource in newResources.split('\n'): - if os.path.isabs(newResource): - r = newResource - else: - r = os.path.join(resourceDir, newResource) - if r in seen: - continue - option_list += SCPWI.getIncludes(r, resources) - # combine lists - cinc = newFiles[0][1] + [resource] + newFiles[1][1] - # cache and return : - SCPWI.CFG_INC_CACHE[hashv] = (cinc, curmt) - return cinc - #print "Includes list for " + resource + " is " + `resources` - -class _SafeConfigParserWithIncludes(SafeConfigParser, object): +class SafeConfigParserWithIncludes(SafeConfigParser): """ Class adds functionality to SafeConfigParser to handle included other configuration files (or may be urls, whatever in the future) @@ -223,14 +94,99 @@ after = 1.conf """ + SECTION_NAME = "INCLUDES" + if sys.version_info >= (3,2): # overload constructor only for fancy new Python3's - def __init__(self, *args, **kwargs): + def __init__(self, share_config=None, *args, **kwargs): kwargs = kwargs.copy() kwargs['interpolation'] = BasicInterpolationWithName() kwargs['inline_comment_prefixes'] = ";" - super(_SafeConfigParserWithIncludes, self).__init__( + super(SafeConfigParserWithIncludes, self).__init__( *args, **kwargs) + self._cfg_share = share_config + + else: + def __init__(self, share_config=None, *args, **kwargs): + SafeConfigParser.__init__(self, *args, **kwargs) + self._cfg_share = share_config + + @property + def share_config(self): + return self._cfg_share + + def _getSharedSCPWI(self, filename): + SCPWI = SafeConfigParserWithIncludes + # read single one, add to return list, use sharing if possible: + if self._cfg_share: + # cache/share each file as include (ex: filter.d/common could be included in each filter config): + hashv = 'inc:'+(filename if not isinstance(filename, list) else '\x01'.join(filename)) + cfg, i = self._cfg_share.get(hashv, (None, None)) + if cfg is None: + cfg = SCPWI(share_config=self._cfg_share) + i = cfg.read(filename, get_includes=False) + self._cfg_share[hashv] = (cfg, i) + else: + logSys.debug(" Shared file: %s", filename) + else: + # don't have sharing: + cfg = SCPWI() + i = cfg.read(filename, get_includes=False) + return (cfg, i) + + def _getIncludes(self, filenames, seen=[]): + if not isinstance(filenames, list): + filenames = [ filenames ] + # retrieve or cache include paths: + if self._cfg_share: + # cache/share include list: + hashv = 'inc-path:'+('\x01'.join(filenames)) + fileNamesFull = self._cfg_share.get(hashv) + if fileNamesFull is None: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + self._cfg_share[hashv] = fileNamesFull + return fileNamesFull + # don't have sharing: + fileNamesFull = [] + for filename in filenames: + fileNamesFull += self.__getIncludesUncached(filename, seen) + return fileNamesFull + + def __getIncludesUncached(self, resource, seen=[]): + """ + Given 1 config resource returns list of included files + (recursively) with the original one as well + Simple loops are taken care about + """ + SCPWI = SafeConfigParserWithIncludes + try: + parser, i = self._getSharedSCPWI(resource) + if not i: + return [] + except UnicodeDecodeError, e: + logSys.error("Error decoding config file '%s': %s" % (resource, e)) + return [] + + resourceDir = os.path.dirname(resource) + + newFiles = [ ('before', []), ('after', []) ] + if SCPWI.SECTION_NAME in parser.sections(): + for option_name, option_list in newFiles: + if option_name in parser.options(SCPWI.SECTION_NAME): + newResources = parser.get(SCPWI.SECTION_NAME, option_name) + for newResource in newResources.split('\n'): + if os.path.isabs(newResource): + r = newResource + else: + r = os.path.join(resourceDir, newResource) + if r in seen: + continue + s = seen + [resource] + option_list += self._getIncludes(r, s) + # combine lists + return newFiles[0][1] + [resource] + newFiles[1][1] def get_defaults(self): return self._defaults @@ -238,38 +194,52 @@ after = 1.conf def get_sections(self): return self._sections - def read(self, filenames): + def read(self, filenames, get_includes=True, log_info=None): if not isinstance(filenames, list): filenames = [ filenames ] - if len(filenames) > 1: - # read multiple configs: - ret = [] - alld = self.get_defaults() - alls = self.get_sections() - for filename in filenames: - # read single one, add to return list: - cfg = SafeConfigParserWithIncludes() - i = cfg.read(filename, get_includes=False) - if i: - ret += i - # merge defaults and all sections to self: - alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().iteritems(): - if isinstance(s, dict): - s2 = alls.get(n) - if isinstance(s2, dict): - s2.update(s) - else: - alls[n] = s.copy() - else: - alls[n] = s - - return ret - - # read one config : - logSys.debug("Reading file: %s", filenames[0]) - if sys.version_info >= (3,2): # pragma: no cover - return SafeConfigParser.read(self, filenames, encoding='utf-8') + # retrieve (and cache) includes: + fileNamesFull = [] + if get_includes: + fileNamesFull += self._getIncludes(filenames) else: - return SafeConfigParser.read(self, filenames) + fileNamesFull = filenames + + if self._cfg_share is not None: + logSys.debug(" Sharing files: %s", fileNamesFull) + + if len(fileNamesFull) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in fileNamesFull: + # read single one, add to return list, use sharing if possible: + cfg, i = self._getSharedSCPWI(filename) + if i: + ret += i + # merge defaults and all sections to self: + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): + if isinstance(s, dict): + s2 = alls.get(n) + if isinstance(s2, dict): + s2.update(s) + else: + alls[n] = s.copy() + else: + alls[n] = s + + return ret + + # read one config : + logSys.debug(" Reading file: %s", fileNamesFull[0]) + else: + # don't have sharing - read one or multiple at once: + logSys.debug(" Reading files: %s", fileNamesFull) + + # read file(s) : + if sys.version_info >= (3,2): # pragma: no cover + return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') + else: + return SafeConfigParser.read(self, fileNamesFull) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 096f8e78..657800fa 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -46,26 +46,38 @@ class ConfigReader(): self._cfg = None if use_config is not None: self._cfg = use_config - else: - # share config if possible: - if share_config is not None: - self._cfg_share = share_config - self._cfg_share_kwargs = kwargs - else: - self._cfg = ConfigReaderUnshared(**kwargs) + # share config if possible: + if share_config is not None: + self._cfg_share = share_config + self._cfg_share_kwargs = kwargs + self._cfg_share_basedir = None + elif self._cfg is None: + self._cfg = ConfigReaderUnshared(**kwargs) def setBaseDir(self, basedir): - self._cfg.setBaseDir(basedir) + if self._cfg: + self._cfg.setBaseDir(basedir) + else: + self._cfg_share_basedir = basedir def getBaseDir(self): - return self._cfg.getBaseDir() + if self._cfg: + return self._cfg.getBaseDir() + else: + return self._cfg_share_basedir + + @property + def share_config(self): + return self._cfg_share def read(self, name, once=True): # shared ? if not self._cfg and self._cfg_share is not None: self._cfg = self._cfg_share.get(name) if not self._cfg: - self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) + self._cfg = ConfigReaderUnshared(share_config=self._cfg_share, **self._cfg_share_kwargs) + if self._cfg_share_basedir is not None: + self._cfg.setBaseDir(self._cfg_share_basedir) self._cfg_share[name] = self._cfg # performance feature - read once if using shared config reader: rc = self._cfg.read_cfg_files @@ -73,6 +85,8 @@ class ConfigReader(): return rc.get(name) # read: + if self._cfg_share is not None: + logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir()) ret = self._cfg.read(name) # save already read: @@ -105,8 +119,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): DEFAULT_BASEDIR = '/etc/fail2ban' - def __init__(self, basedir=None): - SafeConfigParserWithIncludes.__init__(self) + def __init__(self, basedir=None, *args, **kwargs): + SafeConfigParserWithIncludes.__init__(self, *args, **kwargs) self.read_cfg_files = dict() self.setBaseDir(basedir) @@ -123,7 +137,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.debug("Reading configs for %s under %s " , filename, self._basedir) + logSys.info("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -140,8 +154,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if len(config_files): # at least one config exists and accessible logSys.debug("Reading config files: %s", ', '.join(config_files)) - config_files_read = SafeConfigParserWithIncludes.read(self, config_files, - log_info=("Cache configs for %s under %s " , filename, self._basedir)) + config_files_read = SafeConfigParserWithIncludes.read(self, config_files) missed = [ cf for cf in config_files if cf not in config_files_read ] if missed: logSys.error("Could not read config files: %s", ', '.join(missed)) diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index a29fe94d..4c0c0428 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -33,11 +33,11 @@ logSys = getLogger(__name__) class Configurator: - def __init__(self, force_enable=False): + def __init__(self, force_enable=False, share_config=None): self.__settings = dict() self.__streams = dict() - self.__fail2ban = Fail2banReader() - self.__jails = JailsReader(force_enable=force_enable) + self.__fail2ban = Fail2banReader(share_config=share_config) + self.__jails = JailsReader(force_enable=force_enable, share_config=share_config) def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index e4394e59..99891c0f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -41,13 +41,12 @@ class JailReader(ConfigReader): optionExtractRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') - def __init__(self, name, force_enable=False, cfg_share=None, **kwargs): + def __init__(self, name, force_enable=False, **kwargs): # use shared config if possible: ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None self.__force_enable = force_enable - self.__cfg_share = cfg_share self.__actions = list() self.__opts = None @@ -113,7 +112,7 @@ class JailReader(ConfigReader): filterName, filterOpt = JailReader.extractOptions( self.__opts["filter"]) self.__filter = FilterReader( - filterName, self.__name, filterOpt, share_config=self.__cfg_share, basedir=self.getBaseDir()) + filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -143,7 +142,7 @@ class JailReader(ConfigReader): else: action = ActionReader( actName, self.__name, actOpt, - share_config=self.__cfg_share, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 3bb713f6..7043aa67 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -43,7 +43,6 @@ class JailsReader(ConfigReader): """ # use shared config if possible: ConfigReader.__init__(self, **kwargs) - self.__cfg_share = dict() self.__jails = list() self.__force_enable = force_enable @@ -73,7 +72,7 @@ class JailsReader(ConfigReader): # use the cfg_share for filter/action caching and the same config for all # jails (use_config=...), therefore don't read it here: jail = JailReader(sec, force_enable=self.__force_enable, - cfg_share=self.__cfg_share, use_config=self._cfg) + share_config=self.share_config, use_config=self._cfg) ret = jail.getOptions() if ret: if jail.isEnabled(): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 0aaa8890..dadbef0f 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -39,8 +39,6 @@ STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') -LAST_WRITE_TIME = 0 - class ConfigReaderTest(unittest.TestCase): def setUp(self): @@ -59,8 +57,7 @@ class ConfigReaderTest(unittest.TestCase): d_ = os.path.join(self.d, d) if not os.path.exists(d_): os.makedirs(d_) - fname = "%s/%s" % (self.d, fname) - f = open(fname, "w") + f = open("%s/%s" % (self.d, fname), "w") if value is not None: f.write(""" [section] @@ -69,14 +66,6 @@ option = %s if content is not None: f.write(content) f.close() - # set modification time to another second to revalidate cache (if milliseconds not supported) : - global LAST_WRITE_TIME - mtime = os.path.getmtime(fname) - if LAST_WRITE_TIME == mtime: - mtime += 1 - os.utime(fname, (mtime, mtime)) - LAST_WRITE_TIME = mtime - def _remove(self, fname): os.unlink("%s/%s" % (self.d, fname)) @@ -102,6 +91,7 @@ option = %s # raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform) pass + def testOptionalDotDDir(self): self.assertFalse(self.c.read('c')) # nothing is there yet self._write("c.conf", "1") @@ -347,9 +337,9 @@ class FilterReaderTest(unittest.TestCase): class JailsReaderTestCache(LogCaptureTestCase): - def _readWholeConf(self, basedir, force_enable=False): + def _readWholeConf(self, basedir, force_enable=False, share_config=None): # read whole configuration like a file2ban-client ... - configurator = Configurator(force_enable=force_enable) + configurator = Configurator(force_enable=force_enable, share_config=share_config) configurator.setBaseDir(basedir) configurator.readEarly() configurator.getEarlyOptions() @@ -360,7 +350,7 @@ class JailsReaderTestCache(LogCaptureTestCase): def _getLoggedReadCount(self, filematch): cnt = 0 for s in self.getLog().rsplit('\n'): - if re.match(r"^Reading files?: .*/"+filematch, s): + if re.match(r"^\s*Reading files?: .*/"+filematch, s): cnt += 1 return cnt @@ -372,8 +362,11 @@ class JailsReaderTestCache(LogCaptureTestCase): shutil.copy(CONFIG_DIR + '/jail.conf', basedir + '/jail.local') shutil.copy(CONFIG_DIR + '/fail2ban.conf', basedir + '/fail2ban.local') + # common sharing handle for this test: + share_cfg = dict() + # read whole configuration like a file2ban-client ... - self._readWholeConf(basedir) + self._readWholeConf(basedir, share_config=share_cfg) # how many times jail.local was read: cnt = self._getLoggedReadCount('jail.local') # if cnt > 1: @@ -382,7 +375,7 @@ class JailsReaderTestCache(LogCaptureTestCase): # read whole configuration like a file2ban-client, again ... # but this time force enable all jails, to check filter and action cached also: - self._readWholeConf(basedir, force_enable=True) + self._readWholeConf(basedir, force_enable=True, share_config=share_cfg) cnt = self._getLoggedReadCount(r'jail\.local') # still one (no more reads): self.assertTrue(cnt == 1, "Unexpected count by second reading of jail files, cnt = %s" % cnt) From e0eb4f2358049b4427282aaa2a8ddf65972ed05b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 02:47:42 +0200 Subject: [PATCH 14/36] code review: use the same code (corresponding test cases - with sharing on and without it); --- fail2ban/client/configparserinc.py | 57 +++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index d6ad4e34..196c2ae6 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -204,39 +204,40 @@ after = 1.conf else: fileNamesFull = filenames + if not fileNamesFull: + return [] + if self._cfg_share is not None: logSys.debug(" Sharing files: %s", fileNamesFull) - - if len(fileNamesFull) > 1: - # read multiple configs: - ret = [] - alld = self.get_defaults() - alls = self.get_sections() - for filename in fileNamesFull: - # read single one, add to return list, use sharing if possible: - cfg, i = self._getSharedSCPWI(filename) - if i: - ret += i - # merge defaults and all sections to self: - alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().iteritems(): - if isinstance(s, dict): - s2 = alls.get(n) - if isinstance(s2, dict): - s2.update(s) - else: - alls[n] = s.copy() - else: - alls[n] = s - - return ret - - # read one config : - logSys.debug(" Reading file: %s", fileNamesFull[0]) else: - # don't have sharing - read one or multiple at once: logSys.debug(" Reading files: %s", fileNamesFull) + if len(fileNamesFull) > 1: + # read multiple configs: + ret = [] + alld = self.get_defaults() + alls = self.get_sections() + for filename in fileNamesFull: + # read single one, add to return list, use sharing if possible: + cfg, i = self._getSharedSCPWI(filename) + if i: + ret += i + # merge defaults and all sections to self: + alld.update(cfg.get_defaults()) + for n, s in cfg.get_sections().iteritems(): + if isinstance(s, dict): + s2 = alls.get(n) + if isinstance(s2, dict): + s2.update(s) + else: + alls[n] = s.copy() + else: + alls[n] = s + + return ret + + # read one config : + logSys.debug(" Reading file: %s", fileNamesFull[0]) # read file(s) : if sys.version_info >= (3,2): # pragma: no cover return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') From 02a46d0901b99aedfebaae6b2eb6070d97ad4246 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 12:05:49 +0200 Subject: [PATCH 15/36] code review; more stable config sharing, configurator always shares it config readers now; --- fail2ban/client/configparserinc.py | 16 +++---- fail2ban/client/configreader.py | 64 +++++++++++++++----------- fail2ban/client/configurator.py | 3 ++ fail2ban/client/fail2banreader.py | 1 - fail2ban/client/jailreader.py | 1 - fail2ban/client/jailsreader.py | 1 - fail2ban/tests/clientreadertestcase.py | 7 ++- 7 files changed, 54 insertions(+), 39 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 196c2ae6..1687f87f 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -62,6 +62,7 @@ else: # pragma: no cover # Gets the instance of the logger. logSys = getLogger(__name__) +logLevel = 7 __all__ = ['SafeConfigParserWithIncludes'] @@ -126,8 +127,8 @@ after = 1.conf cfg = SCPWI(share_config=self._cfg_share) i = cfg.read(filename, get_includes=False) self._cfg_share[hashv] = (cfg, i) - else: - logSys.debug(" Shared file: %s", filename) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " Shared file: %s", filename) else: # don't have sharing: cfg = SCPWI() @@ -207,10 +208,9 @@ after = 1.conf if not fileNamesFull: return [] - if self._cfg_share is not None: - logSys.debug(" Sharing files: %s", fileNamesFull) - else: - logSys.debug(" Reading files: %s", fileNamesFull) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, (" Sharing files: %s" if self._cfg_share is not None else \ + " Reading files: %s"), fileNamesFull) if len(fileNamesFull) > 1: # read multiple configs: @@ -237,10 +237,10 @@ after = 1.conf return ret # read one config : - logSys.debug(" Reading file: %s", fileNamesFull[0]) + if logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, " Reading file: %s", fileNamesFull[0]) # read file(s) : if sys.version_info >= (3,2): # pragma: no cover return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') else: return SafeConfigParser.read(self, fileNamesFull) - diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 657800fa..bd2e5f0c 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -27,12 +27,11 @@ __license__ = "GPL" import glob, os from ConfigParser import NoOptionError, NoSectionError -from .configparserinc import SafeConfigParserWithIncludes +from .configparserinc import SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -_logLevel = 6 class ConfigReader(): """Generic config reader class. @@ -72,6 +71,22 @@ class ConfigReader(): def read(self, name, once=True): # shared ? + if not self._cfg: + self.touch(name) + # performance feature - read once if using shared config reader: + if once and self._cfg.read_cfg_files is not None: + return self._cfg.read_cfg_files + + # read: + if self._cfg_share is not None: + logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir()) + ret = self._cfg.read(name) + + # save already read and return: + self._cfg.read_cfg_files = ret + return ret + + def touch(self, name = ''): if not self._cfg and self._cfg_share is not None: self._cfg = self._cfg_share.get(name) if not self._cfg: @@ -79,36 +94,33 @@ class ConfigReader(): if self._cfg_share_basedir is not None: self._cfg.setBaseDir(self._cfg_share_basedir) self._cfg_share[name] = self._cfg - # performance feature - read once if using shared config reader: - rc = self._cfg.read_cfg_files - if once and rc.get(name) is not None: - return rc.get(name) - - # read: - if self._cfg_share is not None: - logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir()) - ret = self._cfg.read(name) - - # save already read: - if once: - rc[name] = ret - return ret + else: + self._cfg = ConfigReaderUnshared(**self._cfg_share_kwargs) def sections(self): - return self._cfg.sections() + if self._cfg is not None: + return self._cfg.sections() + return [] def has_section(self, sec): - return self._cfg.has_section(sec) + if self._cfg is not None: + return self._cfg.has_section(sec) + return False def options(self, *args): - return self._cfg.options(*args) + if self._cfg is not None: + return self._cfg.options(*args) + return {} def get(self, sec, opt): - return self._cfg.get(sec, opt) + if self._cfg is not None: + return self._cfg.get(sec, opt) + return None def getOptions(self, *args, **kwargs): - return self._cfg.getOptions(*args, **kwargs) - + if self._cfg is not None: + return self._cfg.getOptions(*args, **kwargs) + return {} class ConfigReaderUnshared(SafeConfigParserWithIncludes): """Unshared config reader (previously ConfigReader). @@ -121,7 +133,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): def __init__(self, basedir=None, *args, **kwargs): SafeConfigParserWithIncludes.__init__(self, *args, **kwargs) - self.read_cfg_files = dict() + self.read_cfg_files = None self.setBaseDir(basedir) def setBaseDir(self, basedir): @@ -137,7 +149,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): raise ValueError("Base configuration directory %s does not exist " % self._basedir) basename = os.path.join(self._basedir, filename) - logSys.info("Reading configs for %s under %s " , filename, self._basedir) + logSys.debug("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] # possible further customizations under a .conf.d directory @@ -203,8 +215,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] - elif logSys.getEffectiveLevel() <= _logLevel: - logSys.log(_logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) + elif logSys.getEffectiveLevel() <= logLevel: + logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 4c0c0428..b3ddc5d0 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -36,6 +36,9 @@ class Configurator: def __init__(self, force_enable=False, share_config=None): self.__settings = dict() self.__streams = dict() + # always share all config readers: + if share_config is None: + share_config = dict() self.__fail2ban = Fail2banReader(share_config=share_config) self.__jails = JailsReader(force_enable=force_enable, share_config=share_config) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 4d46c156..361a5e54 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -33,7 +33,6 @@ logSys = getLogger(__name__) class Fail2banReader(ConfigReader): def __init__(self, **kwargs): - self.__opts = None ConfigReader.__init__(self, **kwargs) def read(self): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 99891c0f..ffdc5e26 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -42,7 +42,6 @@ class JailReader(ConfigReader): r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,]*))(?:,|$)') def __init__(self, name, force_enable=False, **kwargs): - # use shared config if possible: ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 7043aa67..40255ce7 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -41,7 +41,6 @@ class JailsReader(ConfigReader): Passed to JailReader to force enable the jails. It is for internal use """ - # use shared config if possible: ConfigReader.__init__(self, **kwargs) self.__jails = list() self.__force_enable = force_enable diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index dadbef0f..e4dc2189 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -21,9 +21,9 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import os, glob, shutil, tempfile, unittest, re - +import os, glob, shutil, tempfile, unittest, re, logging from ..client.configreader import ConfigReaderUnshared +from ..client import configparserinc from ..client.jailreader import JailReader from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader @@ -355,6 +355,8 @@ class JailsReaderTestCache(LogCaptureTestCase): return cnt def testTestJailConfCache(self): + saved_ll = configparserinc.logLevel + configparserinc.logLevel = logging.DEBUG basedir = tempfile.mkdtemp("fail2ban_conf") try: shutil.rmtree(basedir) @@ -388,6 +390,7 @@ class JailsReaderTestCache(LogCaptureTestCase): self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: shutil.rmtree(basedir) + configparserinc.logLevel = saved_ll class JailsReaderTest(LogCaptureTestCase): From 95bdcdecaaa0419785614c22afc57165fcbb29dc Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 16:49:08 +0200 Subject: [PATCH 16/36] cache-config-read-v2 merged; logging normalized, set log level for loading (read or use shared) file(s) to INFO; prevent to read some files twice by read inside "_getIncludes" and by "read" self (occurred by only one file); --- fail2ban/client/configparserinc.py | 10 ++++------ fail2ban/client/configreader.py | 5 ++--- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 1687f87f..d819281b 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -195,7 +195,7 @@ after = 1.conf def get_sections(self): return self._sections - def read(self, filenames, get_includes=True, log_info=None): + def read(self, filenames, get_includes=True): if not isinstance(filenames, list): filenames = [ filenames ] # retrieve (and cache) includes: @@ -208,11 +208,9 @@ after = 1.conf if not fileNamesFull: return [] - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, (" Sharing files: %s" if self._cfg_share is not None else \ - " Reading files: %s"), fileNamesFull) + logSys.info(" Loading files: %s", fileNamesFull) - if len(fileNamesFull) > 1: + if get_includes or len(fileNamesFull) > 1: # read multiple configs: ret = [] alld = self.get_defaults() @@ -238,7 +236,7 @@ after = 1.conf # read one config : if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " Reading file: %s", fileNamesFull[0]) + logSys.log(logLevel, " Reading file: %s", fileNamesFull[0]) # read file(s) : if sys.version_info >= (3,2): # pragma: no cover return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index bd2e5f0c..9e3bf90f 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -77,9 +77,8 @@ class ConfigReader(): if once and self._cfg.read_cfg_files is not None: return self._cfg.read_cfg_files - # read: - if self._cfg_share is not None: - logSys.info("Sharing configs for %s under %s ", name, self._cfg.getBaseDir()) + # load: + logSys.info("Loading configs for %s under %s ", name, self._cfg.getBaseDir()) ret = self._cfg.read(name) # save already read and return: From 7f5d4aa7a6c5acc5f29893e9d0f5b4c78e5c4053 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 16:59:40 +0200 Subject: [PATCH 17/36] normalize tabs/spaces in docstrings; --- fail2ban/client/configreader.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 9e3bf90f..ea86a36f 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -36,8 +36,8 @@ logSys = getLogger(__name__) class ConfigReader(): """Generic config reader class. - A caching adapter which automatically reuses already shared configuration. - """ + A caching adapter which automatically reuses already shared configuration. + """ def __init__(self, use_config=None, share_config=None, **kwargs): # use given shared config if possible (see read): @@ -126,7 +126,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): Does not use this class (internal not shared/cached represenation). Use ConfigReader instead. - """ + """ DEFAULT_BASEDIR = '/etc/fail2ban' @@ -224,12 +224,12 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): class DefinitionInitConfigReader(ConfigReader): """Config reader for files with options grouped in [Definition] and - [Init] sections. + [Init] sections. - Is a base class for readers of filters and actions, where definitions - in jails might provide custom values for options defined in [Init] - section. - """ + Is a base class for readers of filters and actions, where definitions + in jails might provide custom values for options defined in [Init] + section. + """ _configOpts = [] From 73a06d55a8ed5996e959952a616c9d9a8b209dc7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 18:50:24 +0200 Subject: [PATCH 18/36] reset share/cache storage (if we use 'reload' in client with interactive mode) --- bin/fail2ban-client | 1 + fail2ban/client/configurator.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 89e0a903..0c6999c1 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -409,6 +409,7 @@ class Fail2banClient: # TODO: get away from stew of return codes and exception # handling -- handle via exceptions try: + self.__configurator.Reload() self.__configurator.readAll() ret = self.__configurator.getOptions(jail) self.__configurator.convertToProtocol() diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index b3ddc5d0..8667501c 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -39,9 +39,14 @@ class Configurator: # always share all config readers: if share_config is None: share_config = dict() + self.__share_config = share_config self.__fail2ban = Fail2banReader(share_config=share_config) self.__jails = JailsReader(force_enable=force_enable, share_config=share_config) - + + def Reload(self): + # clear all shared handlers: + self.__share_config.clear() + def setBaseDir(self, folderName): self.__fail2ban.setBaseDir(folderName) self.__jails.setBaseDir(folderName) From 7d3e6e9935a22de7c7225576e649ed8592d65e55 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Oct 2014 20:06:58 +0200 Subject: [PATCH 19/36] code review, change log entries added; --- ChangeLog | 6 ++++++ fail2ban/client/configreader.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index d92aec4a..0515823a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger provides defaults for the chain, port, protocol and name tags - Fixes: + * start of file2ban aborted (on slow hosts, systemd considers the server has been + timed out and kills him), see gh-824 * UTF-8 fixes in pure-ftp thanks to Johannes Weberhofer. Closes gh-806. * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request @@ -64,6 +66,10 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Added Cloudflare API action - Enhancements + * Start performance of fail2ban-client (and tests) increased, start time + and cpu usage rapidly reduced. Introduced a shared storage logic, to bypass + reading lots of config files (see gh-824). + Thanks to Joost Molenaar for good catch (reported gh-820). * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options * Match non "Bye Bye" disconnect messages for sshd locked account regex diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index ea86a36f..82bf0fdc 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -70,7 +70,12 @@ class ConfigReader(): return self._cfg_share def read(self, name, once=True): - # shared ? + """ Overloads a default (not shared) read of config reader. + + To prevent mutiple reads of config files with it includes, reads into + the config reader, if it was not yet cached/shared by 'name'. + """ + # already shared ? if not self._cfg: self.touch(name) # performance feature - read once if using shared config reader: @@ -85,7 +90,12 @@ class ConfigReader(): self._cfg.read_cfg_files = ret return ret - def touch(self, name = ''): + def touch(self, name=''): + """ Allocates and share a config file by it name. + + Automatically allocates unshared or reuses shared handle by given 'name' and + init arguments inside a given shared storage. + """ if not self._cfg and self._cfg_share is not None: self._cfg = self._cfg_share.get(name) if not self._cfg: @@ -124,7 +134,7 @@ class ConfigReader(): class ConfigReaderUnshared(SafeConfigParserWithIncludes): """Unshared config reader (previously ConfigReader). - Does not use this class (internal not shared/cached represenation). + Do not use this class (internal not shared/cached represenation). Use ConfigReader instead. """ @@ -191,7 +201,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # 1 -> the name of the option # 2 -> the default value for the option - def getOptions(self, sec, options, pOptions = None): + def getOptions(self, sec, options, pOptions=None): values = dict() for option in options: try: From 5ac496d0309c0901fef00efdea39fa1eec40b4f4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 12 Oct 2014 17:28:35 -0400 Subject: [PATCH 20/36] We better check that installation doesn't cause any errors as well --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9a92a7f6..bd2d294c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,8 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi +# test installation + - sudo python setup.py install after_success: # Coverage config file must be .coveragerc for coveralls - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cp -v .travis_coveragerc .coveragerc; fi From e2f49b7334c20f01ddc3f88a1a4fc50799d7775b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Oct 2014 14:34:17 -0400 Subject: [PATCH 21/36] DOC: very minor (tabs/spaces) --- ChangeLog | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0515823a..ad92a55e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,8 +16,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger provides defaults for the chain, port, protocol and name tags - Fixes: - * start of file2ban aborted (on slow hosts, systemd considers the server has been - timed out and kills him), see gh-824 + * start of file2ban aborted (on slow hosts, systemd considers the server has + been timed out and kills him), see gh-824 * UTF-8 fixes in pure-ftp thanks to Johannes Weberhofer. Closes gh-806. * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request @@ -67,8 +67,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Enhancements * Start performance of fail2ban-client (and tests) increased, start time - and cpu usage rapidly reduced. Introduced a shared storage logic, to bypass - reading lots of config files (see gh-824). + and cpu usage rapidly reduced. Introduced a shared storage logic, to + bypass reading lots of config files (see gh-824). Thanks to Joost Molenaar for good catch (reported gh-820). * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options From d4015d6566e15a005ad6fd6cdfb2c477bce5e946 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Oct 2014 14:51:51 -0400 Subject: [PATCH 22/36] ENH: remove obsolete code for python < 2.6 (we support >= 2.6) --- fail2ban/server/asyncserver.py | 8 ++------ fail2ban/tests/samplestestcase.py | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 14673a99..6e71da77 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -149,12 +149,8 @@ class AsyncServer(asyncore.dispatcher): self.__init = True # TODO Add try..catch # There's a bug report for Python 2.6/3.0 that use_poll=True yields some 2.5 incompatibilities: - if sys.version_info >= (2, 6): # if python 2.6 or greater... - logSys.debug("Detected Python 2.6 or greater. asyncore.loop() not using poll") - asyncore.loop(use_poll = False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 - else: # pragma: no cover - logSys.debug("NOT Python 2.6/3.* - asyncore.loop() using poll") - asyncore.loop(use_poll = True) + logSys.debug("Detected Python 2.6 or greater. asyncore.loop() not using poll") + asyncore.loop(use_poll=False) # fixes the "Unexpected communication problem" issue on Python 2.6 and 3.0 ## # Stops the communication server. diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index e0831184..2c18a504 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -24,11 +24,7 @@ __license__ = "GPL" import unittest, sys, os, fileinput, re, time, datetime, inspect -if sys.version_info >= (2, 6): - import json -else: - import simplejson as json - next = lambda x: x.next() +import json from ..server.filter import Filter from ..client.filterreader import FilterReader From 0b0ea41f8773bf15e5a3eb9bb37778ee66255149 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Oct 2014 04:59:44 +0200 Subject: [PATCH 23/36] fix: fail2ban-regex with filter file failed (after merging #824, because test case missing); test case for 'readexplicit' added; --- fail2ban/client/configreader.py | 4 +++- fail2ban/tests/clientreadertestcase.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 82bf0fdc..d9bfb271 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -267,7 +267,9 @@ class DefinitionInitConfigReader(ConfigReader): # needed for fail2ban-regex that doesn't need fancy directories def readexplicit(self): - return SafeConfigParserWithIncludes.read(self, self._file) + if not self._cfg: + self.touch(self._file) + return SafeConfigParserWithIncludes.read(self._cfg, self._file) def getOptions(self, pOpts): self._opts = ConfigReader.getOptions( diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e4dc2189..fde9a915 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -334,6 +334,22 @@ class FilterReaderTest(unittest.TestCase): filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) + def testFilterReaderExplicit(self): + # read explicit uses absolute path: + path_ = os.path.abspath(os.path.join(TEST_FILES_DIR, "filter.d")) + filterReader = FilterReader(os.path.join(path_, "testcase01.conf"), "testcase01", {}) + self.assertEqual(filterReader.readexplicit(), + [os.path.join(path_, "testcase-common.conf"), os.path.join(path_, "testcase01.conf")] + ) + try: + filterReader.getOptions(None) + # from included common + filterReader.get('Definition', '__prefix_line') + # from testcase01 + filterReader.get('Definition', 'failregex') + filterReader.get('Definition', 'ignoreregex') + except Exception, e: + self.fail('unexpected options after readexplicit: %s' % (e)) class JailsReaderTestCache(LogCaptureTestCase): From bef0502e6b04b4221245b1f4a2408fd075d11eb6 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 24 Oct 2014 05:28:35 +0200 Subject: [PATCH 24/36] coverage: no cover (for failed except) --- fail2ban/tests/clientreadertestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index fde9a915..17a90cc8 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -348,7 +348,7 @@ class FilterReaderTest(unittest.TestCase): # from testcase01 filterReader.get('Definition', 'failregex') filterReader.get('Definition', 'ignoreregex') - except Exception, e: + except Exception, e: # pragma: no cover - failed if reachable self.fail('unexpected options after readexplicit: %s' % (e)) class JailsReaderTestCache(LogCaptureTestCase): From 07d4badfd069bb79d8d3a4562cc8de943c185c7b Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jun 2014 13:45:29 +0200 Subject: [PATCH 25/36] testExecuteTimeout fixed: give a test still 1 second, because system could be too busy --- fail2ban/tests/actiontestcase.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index f1ea77ce..5a58149f 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -186,8 +186,10 @@ class CommandActionTest(LogCaptureTestCase): # Should take a minute self.assertRaises( RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) - self.assertAlmostEqual(time.time() - stime, 2, places=0) - self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds')) + # give a test still 1 second, because system could be too busy + self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) + self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') + or self._is_logged('sleep 60 -- timed out after 3 seconds')) self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) def testCaptureStdOutErr(self): From caa6006a3179513962f960d57a8fca9e6199951b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 25 Oct 2014 09:25:18 -0400 Subject: [PATCH 26/36] ENH: do use @staticmethod (we are well beyond support of 2.4 now) --- bin/fail2ban-client | 3 +-- fail2ban/client/csocket.py | 3 +-- fail2ban/client/jailreader.py | 3 +-- fail2ban/server/asyncserver.py | 3 +-- fail2ban/server/banmanager.py | 3 +-- fail2ban/server/filter.py | 21 +++++++-------------- fail2ban/server/mytime.py | 14 +++++--------- 7 files changed, 17 insertions(+), 33 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 0c6999c1..866a5287 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -419,12 +419,11 @@ class Fail2banClient: ret = False return ret - #@staticmethod + @staticmethod def dumpConfig(cmd): for c in cmd: print c return True - dumpConfig = staticmethod(dumpConfig) class ServerExecutionException(Exception): diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index 1d522f6c..921b0de5 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -57,7 +57,7 @@ class CSocket: self.__csock.close() return ret - #@staticmethod + @staticmethod def receive(sock): msg = EMPTY_BYTES while msg.rfind(CSocket.END_STRING) == -1: @@ -66,4 +66,3 @@ class CSocket: raise RuntimeError, "socket connection broken" msg = msg + chunk return loads(msg) - receive = staticmethod(receive) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index ffdc5e26..84cc5e2a 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -220,7 +220,7 @@ class JailReader(ConfigReader): stream.insert(0, ["add", self.__name, backend]) return stream - #@staticmethod + @staticmethod def extractOptions(option): match = JailReader.optionCRE.match(option) if not match: @@ -235,4 +235,3 @@ class JailReader(ConfigReader): val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() return option_name, option_opts - extractOptions = staticmethod(extractOptions) diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 6e71da77..a54d41a1 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -171,12 +171,11 @@ class AsyncServer(asyncore.dispatcher): # @param sock: socket file. - #@staticmethod + @staticmethod def __markCloseOnExec(sock): fd = sock.fileno() flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags|fcntl.FD_CLOEXEC) - __markCloseOnExec = staticmethod(__markCloseOnExec) ## # AsyncServerException is used to wrap communication exceptions. diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index b24fa9e5..c21cad45 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -126,7 +126,7 @@ class BanManager: # @param ticket the FailTicket # @return a BanTicket - #@staticmethod + @staticmethod def createBanTicket(ticket): ip = ticket.getIP() #lastTime = ticket.getTime() @@ -134,7 +134,6 @@ class BanManager: banTicket = BanTicket(ip, lastTime, ticket.getMatches()) banTicket.setAttempt(ticket.getAttempt()) return banTicket - createBanTicket = staticmethod(createBanTicket) ## # Add a ban ticket. diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d1c9d2ad..c886bf35 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -838,7 +838,7 @@ class DNSUtils: IP_CRE = re.compile("^(?:\d{1,3}\.){3}\d{1,3}$") - #@staticmethod + @staticmethod def dnsToIp(dns): """ Convert a DNS into an IP address using the Python socket module. Thanks to Kevin Drapel. @@ -853,9 +853,8 @@ class DNSUtils: logSys.warning("Socket error raised trying to resolve hostname %s: %s" % (dns, e)) return list() - dnsToIp = staticmethod(dnsToIp) - #@staticmethod + @staticmethod def searchIP(text): """ Search if an IP address if directly available and return it. @@ -865,9 +864,8 @@ class DNSUtils: return match else: return None - searchIP = staticmethod(searchIP) - #@staticmethod + @staticmethod def isValidIP(string): """ Return true if str is a valid IP """ @@ -877,9 +875,8 @@ class DNSUtils: return True except socket.error: return False - isValidIP = staticmethod(isValidIP) - #@staticmethod + @staticmethod def textToIp(text, useDns): """ Return the IP of DNS found in a given text. """ @@ -901,9 +898,8 @@ class DNSUtils: text, ipList) return ipList - textToIp = staticmethod(textToIp) - #@staticmethod + @staticmethod def cidr(i, n): """ Convert an IP address string with a CIDR mask into a 32-bit integer. @@ -911,18 +907,15 @@ class DNSUtils: # 32-bit IPv4 address mask MASK = 0xFFFFFFFFL return ~(MASK >> n) & MASK & DNSUtils.addr2bin(i) - cidr = staticmethod(cidr) - #@staticmethod + @staticmethod def addr2bin(string): """ Convert a string IPv4 address into an unsigned integer. """ return struct.unpack("!L", socket.inet_aton(string))[0] - addr2bin = staticmethod(addr2bin) - #@staticmethod + @staticmethod def bin2addr(addr): """ Convert a numeric IPv4 address into string n.n.n.n form. """ return socket.inet_ntoa(struct.pack("!L", addr)) - bin2addr = staticmethod(bin2addr) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 96c7f8ab..a27f575b 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -43,48 +43,44 @@ class MyTime: # # @param t the time to set or None - #@staticmethod + @staticmethod def setTime(t): MyTime.myTime = t - setTime = staticmethod(setTime) ## # Equivalent to time.time() # # @return time.time() if setTime was called with None - #@staticmethod + @staticmethod def time(): if MyTime.myTime is None: return time.time() else: return MyTime.myTime - time = staticmethod(time) ## # Equivalent to time.gmtime() # # @return time.gmtime() if setTime was called with None - #@staticmethod + @staticmethod def gmtime(): if MyTime.myTime is None: return time.gmtime() else: return time.gmtime(MyTime.myTime) - gmtime = staticmethod(gmtime) - #@staticmethod + @staticmethod def now(): if MyTime.myTime is None: return datetime.datetime.now() else: return datetime.datetime.fromtimestamp(MyTime.myTime) - now = staticmethod(now) + @staticmethod def localtime(x=None): if MyTime.myTime is None or x is not None: return time.localtime(x) else: return time.localtime(MyTime.myTime) - localtime = staticmethod(localtime) From e1a5decc00a49bf24702b02de9f5ece81a1b84b9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 25 Oct 2014 09:34:37 -0400 Subject: [PATCH 27/36] DOC: adjust docs in mytime to place docs into docstrings --- fail2ban/server/mytime.py | 62 +++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index a27f575b..f284379e 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -26,46 +26,48 @@ import time, datetime ## # MyTime class. # -# This class is a wrapper around time.time() and time.gmtime(). When -# performing unit test, it is very useful to get a fixed value from these -# functions. -# Thus, time.time() and time.gmtime() should never be called directly. -# This wrapper should be called instead. The API are equivalent. class MyTime: - + """A wrapper around time module primarily for testing purposes + + This class is a wrapper around time.time() and time.gmtime(). When + performing unit test, it is very useful to get a fixed value from + these functions. Thus, time.time() and time.gmtime() should never + be called directly. This wrapper should be called instead. The API + are equivalent. + """ + myTime = None - - ## - # Sets the current time. - # - # Use None in order to always get the real current time. - # - # @param t the time to set or None - + @staticmethod def setTime(t): + """Set current time. + + Use None in order to always get the real current time. + + @param t the time to set or None + """ + MyTime.myTime = t - - ## - # Equivalent to time.time() - # - # @return time.time() if setTime was called with None - + @staticmethod def time(): + """Decorate time.time() for the purpose of testing mocking + + @return time.time() if setTime was called with None + """ + if MyTime.myTime is None: return time.time() else: return MyTime.myTime - - ## - # Equivalent to time.gmtime() - # - # @return time.gmtime() if setTime was called with None - + @staticmethod def gmtime(): + """Decorate time.gmtime() for the purpose of testing mocking + + @return time.gmtime() if setTime was called with None + """ if MyTime.myTime is None: return time.gmtime() else: @@ -73,6 +75,10 @@ class MyTime: @staticmethod def now(): + """Decorate datetime.now() for the purpose of testing mocking + + @return datetime.now() if setTime was called with None + """ if MyTime.myTime is None: return datetime.datetime.now() else: @@ -80,6 +86,10 @@ class MyTime: @staticmethod def localtime(x=None): + """Decorate time.localtime() for the purpose of testing mocking + + @return time.localtime() if setTime was called with None + """ if MyTime.myTime is None or x is not None: return time.localtime(x) else: From 6293e448895cc342ef83f005a92f304a5d783c3d Mon Sep 17 00:00:00 2001 From: Florian Pelgrim Date: Wed, 13 Aug 2014 17:34:02 +0200 Subject: [PATCH 28/36] Added myself into THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 0433f7ed..61e99466 100644 --- a/THANKS +++ b/THANKS @@ -26,6 +26,7 @@ Christian Rauch Christophe Carles Christoph Haas Christos Psonis +craneworks Cyril Jaquier Daniel B. Cid Daniel B. From 3dabd5fc833e2546d4715366126ac600a2088614 Mon Sep 17 00:00:00 2001 From: Florian Pelgrim Date: Sat, 25 Oct 2014 10:38:18 -0400 Subject: [PATCH 29/36] DOC: documentation about available vagrantfile setup manually picked up from a commit within https://github.com/fail2ban/fail2ban/pull/786 --- DEVELOP | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DEVELOP b/DEVELOP index f1426561..1384a6ac 100644 --- a/DEVELOP +++ b/DEVELOP @@ -81,6 +81,18 @@ some quick commands:: status test +Testing with vagrant +-------------------- + +Testing can now be done inside a vagrant VM. Vagrantfile provided in +source code repository established two VMs: + +- VM "secure" which can be used for testing fail2ban code. +- VM "attacker" which hcan be used to perform attack against our "secure" VM. + +Both VMs are sharing the 192.168.200/24 network. If you are using this network +take a look into the Vagrantfile and change the IP. + Coding Standards ================ From 3dac76559853b25ce308488dd1eba6eb5d2ab6b9 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Oct 2014 17:20:01 +0200 Subject: [PATCH 30/36] ConfigReader.touch renamed into protected _create_unshared --- fail2ban/client/configreader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d9bfb271..b6c39628 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -77,7 +77,7 @@ class ConfigReader(): """ # already shared ? if not self._cfg: - self.touch(name) + self._create_unshared(name) # performance feature - read once if using shared config reader: if once and self._cfg.read_cfg_files is not None: return self._cfg.read_cfg_files @@ -90,7 +90,7 @@ class ConfigReader(): self._cfg.read_cfg_files = ret return ret - def touch(self, name=''): + def _create_unshared(self, name=''): """ Allocates and share a config file by it name. Automatically allocates unshared or reuses shared handle by given 'name' and @@ -268,7 +268,7 @@ class DefinitionInitConfigReader(ConfigReader): # needed for fail2ban-regex that doesn't need fancy directories def readexplicit(self): if not self._cfg: - self.touch(self._file) + self._create_unshared(self._file) return SafeConfigParserWithIncludes.read(self._cfg, self._file) def getOptions(self, pOpts): From b60e2bf42f888f7a5421731a12742c0b8c74ccb9 Mon Sep 17 00:00:00 2001 From: pacop Date: Sat, 25 Oct 2014 18:17:57 +0200 Subject: [PATCH 31/36] Add portsentry to changelog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index ad92a55e..3e9d99b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -81,6 +81,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger not affect SYSLOG output * Log unhandled exceptions * cyrus-imap: catch "user not found" attempts + * Add support for Portsentry ver. 0.9.0 (2014/03/14) - beta ---------- From 9e8e4dde69c4c5df5cfd681d59e12c816ecd0afa Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 21:27:51 -0400 Subject: [PATCH 32/36] Populated MANIFEST with more entries which were preiously missed or duplicated. Sorted within each "section" --- MANIFEST | 348 ++++++++++++++++++++++++++++------------------------ MANIFEST.in | 2 +- RELEASE | 4 + 3 files changed, 195 insertions(+), 159 deletions(-) diff --git a/MANIFEST b/MANIFEST index 92edcca8..eeb13eef 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,11 +1,14 @@ -README.md -README.Solaris -ChangeLog -TODO -THANKS +CONTRIBUTING.md COPYING +ChangeLog DEVELOP FILTERS +README.Solaris +README.md +RELEASE +THANKS +TODO +Vagrantfile fail2ban-2to3 fail2ban-testcases-all fail2ban-testcases-all-python3 @@ -14,109 +17,108 @@ bin/fail2ban-server bin/fail2ban-testcases bin/fail2ban-regex doc/run-rootless.txt -fail2ban/client/configreader.py -fail2ban/client/configparserinc.py -fail2ban/client/jailreader.py -fail2ban/client/fail2banreader.py -fail2ban/client/jailsreader.py -fail2ban/client/beautifier.py -fail2ban/client/filterreader.py -fail2ban/client/actionreader.py fail2ban/client/__init__.py +fail2ban/client/actionreader.py +fail2ban/client/beautifier.py +fail2ban/client/configparserinc.py +fail2ban/client/configreader.py fail2ban/client/configurator.py fail2ban/client/csocket.py -fail2ban/server/asyncserver.py -fail2ban/server/database.py -fail2ban/server/filter.py -fail2ban/server/filterpyinotify.py -fail2ban/server/filtergamin.py -fail2ban/server/filterpoll.py -fail2ban/server/filtersystemd.py -fail2ban/server/iso8601.py -fail2ban/server/server.py +fail2ban/client/fail2banreader.py +fail2ban/client/filterreader.py +fail2ban/client/jailreader.py +fail2ban/client/jailsreader.py +fail2ban/server/__init__.py +fail2ban/server/action.py fail2ban/server/actions.py +fail2ban/server/asyncserver.py +fail2ban/server/banmanager.py +fail2ban/server/database.py +fail2ban/server/datedetector.py +fail2ban/server/datetemplate.py fail2ban/server/faildata.py fail2ban/server/failmanager.py -fail2ban/server/datedetector.py -fail2ban/server/jailthread.py -fail2ban/server/transmitter.py -fail2ban/server/action.py -fail2ban/server/ticket.py +fail2ban/server/failregex.py +fail2ban/server/filter.py +fail2ban/server/filtergamin.py +fail2ban/server/filterpoll.py +fail2ban/server/filterpyinotify.py +fail2ban/server/filtersystemd.py +fail2ban/server/iso8601.py fail2ban/server/jail.py fail2ban/server/jails.py -fail2ban/server/__init__.py -fail2ban/server/banmanager.py -fail2ban/server/datetemplate.py +fail2ban/server/jailthread.py fail2ban/server/mytime.py -fail2ban/server/failregex.py -fail2ban/server/database.py -fail2ban/tests/banmanagertestcase.py -fail2ban/tests/failmanagertestcase.py -fail2ban/tests/clientreadertestcase.py -fail2ban/tests/filtertestcase.py +fail2ban/server/server.py +fail2ban/server/strptime.py +fail2ban/server/ticket.py +fail2ban/server/transmitter.py fail2ban/tests/__init__.py -fail2ban/tests/dummyjail.py -fail2ban/tests/samplestestcase.py -fail2ban/tests/datedetectortestcase.py +fail2ban/tests/action_d/__init__.py +fail2ban/tests/action_d/test_badips.py +fail2ban/tests/action_d/test_smtp.py +fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py -fail2ban/tests/servertestcase.py -fail2ban/tests/sockettestcase.py -fail2ban/tests/utils.py -fail2ban/tests/misctestcase.py -fail2ban/tests/databasetestcase.py -fail2ban/tests/config/jail.conf +fail2ban/tests/banmanagertestcase.py +fail2ban/tests/clientreadertestcase.py +fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/simple.conf +fail2ban/tests/config/jail.conf fail2ban/tests/config/paths-common.conf +fail2ban/tests/config/paths-debian.conf fail2ban/tests/config/paths-freebsd.conf fail2ban/tests/config/paths-osx.conf -fail2ban/tests/config/paths-debian.conf -fail2ban/tests/config/filter.d/simple.conf -fail2ban/tests/config/action.d/brokenaction.conf -fail2ban/tests/files/config/apache-auth/digest/.htaccess -fail2ban/tests/files/config/apache-auth/digest/.htpasswd -fail2ban/tests/files/config/apache-auth/digest_time/.htaccess -fail2ban/tests/files/config/apache-auth/digest_time/.htpasswd +fail2ban/tests/databasetestcase.py +fail2ban/tests/datedetectortestcase.py +fail2ban/tests/dummyjail.py +fail2ban/tests/failmanagertestcase.py +fail2ban/tests/files/action.d/action.py +fail2ban/tests/files/action.d/action_errors.py +fail2ban/tests/files/action.d/action_modifyainfo.py +fail2ban/tests/files/action.d/action_noAction.py +fail2ban/tests/files/action.d/action_nomethod.py +fail2ban/tests/files/config/apache-auth/README fail2ban/tests/files/config/apache-auth/basic/authz_owner/.htaccess -fail2ban/tests/files/config/apache-auth/basic/authz_owner/cant_get_me.html fail2ban/tests/files/config/apache-auth/basic/authz_owner/.htpasswd +fail2ban/tests/files/config/apache-auth/basic/authz_owner/cant_get_me.html fail2ban/tests/files/config/apache-auth/basic/file/.htaccess fail2ban/tests/files/config/apache-auth/basic/file/.htpasswd fail2ban/tests/files/config/apache-auth/digest.py -fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htaccess -fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htpasswd +fail2ban/tests/files/config/apache-auth/digest/.htaccess +fail2ban/tests/files/config/apache-auth/digest/.htpasswd fail2ban/tests/files/config/apache-auth/digest_anon/.htaccess fail2ban/tests/files/config/apache-auth/digest_anon/.htpasswd -fail2ban/tests/files/config/apache-auth/README +fail2ban/tests/files/config/apache-auth/digest_time/.htaccess +fail2ban/tests/files/config/apache-auth/digest_time/.htpasswd +fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htaccess +fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htpasswd fail2ban/tests/files/config/apache-auth/noentry/.htaccess fail2ban/tests/files/database_v1.db -fail2ban/tests/files/ignorecommand.py fail2ban/tests/files/filter.d/substition.conf fail2ban/tests/files/filter.d/testcase-common.conf fail2ban/tests/files/filter.d/testcase01.conf -fail2ban/tests/files/testcase01.log -fail2ban/tests/files/testcase02.log -fail2ban/tests/files/testcase03.log -fail2ban/tests/files/testcase04.log -fail2ban/tests/files/testcase-usedns.log -fail2ban/tests/files/testcase-journal.log -fail2ban/tests/files/testcase-multiline.log -fail2ban/tests/files/logs/bsd/syslog-plain.txt -fail2ban/tests/files/logs/bsd/syslog-v.txt -fail2ban/tests/files/logs/bsd/syslog-vv.txt +fail2ban/tests/files/ignorecommand.py fail2ban/tests/files/logs/3proxy fail2ban/tests/files/logs/apache-auth fail2ban/tests/files/logs/apache-badbots fail2ban/tests/files/logs/apache-botscripts +fail2ban/tests/files/logs/apache-botsearch fail2ban/tests/files/logs/apache-modsecurity fail2ban/tests/files/logs/apache-nohome fail2ban/tests/files/logs/apache-noscript fail2ban/tests/files/logs/apache-overflows +fail2ban/tests/files/logs/apache-shellshock fail2ban/tests/files/logs/assp fail2ban/tests/files/logs/asterisk +fail2ban/tests/files/logs/bsd/syslog-plain.txt +fail2ban/tests/files/logs/bsd/syslog-v.txt +fail2ban/tests/files/logs/bsd/syslog-vv.txt fail2ban/tests/files/logs/counter-strike fail2ban/tests/files/logs/courier-auth fail2ban/tests/files/logs/courier-smtp fail2ban/tests/files/logs/cyrus-imap +fail2ban/tests/files/logs/directadmin fail2ban/tests/files/logs/dovecot fail2ban/tests/files/logs/dropbear fail2ban/tests/files/logs/ejabberd-auth @@ -126,42 +128,60 @@ fail2ban/tests/files/logs/freeswitch fail2ban/tests/files/logs/groupoffice fail2ban/tests/files/logs/gssftpd fail2ban/tests/files/logs/guacamole +fail2ban/tests/files/logs/horde fail2ban/tests/files/logs/kerio fail2ban/tests/files/logs/lighttpd-auth +fail2ban/tests/files/logs/monit fail2ban/tests/files/logs/mysqld-auth fail2ban/tests/files/logs/nagios -fail2ban/tests/files/logs/nsd -fail2ban/tests/files/logs/perdition -fail2ban/tests/files/logs/php-url-fopen -fail2ban/tests/files/logs/postfix-sasl fail2ban/tests/files/logs/named-refused fail2ban/tests/files/logs/nginx-http-auth +fail2ban/tests/files/logs/nsd +fail2ban/tests/files/logs/openwebmail +fail2ban/tests/files/logs/oracleims fail2ban/tests/files/logs/pam-generic +fail2ban/tests/files/logs/perdition +fail2ban/tests/files/logs/php-url-fopen +fail2ban/tests/files/logs/portsentry fail2ban/tests/files/logs/postfix +fail2ban/tests/files/logs/postfix-sasl fail2ban/tests/files/logs/proftpd fail2ban/tests/files/logs/pure-ftpd fail2ban/tests/files/logs/qmail fail2ban/tests/files/logs/recidive fail2ban/tests/files/logs/roundcube-auth fail2ban/tests/files/logs/selinux-ssh +fail2ban/tests/files/logs/sendmail-auth +fail2ban/tests/files/logs/sendmail-reject fail2ban/tests/files/logs/sendmail-spam fail2ban/tests/files/logs/sieve -fail2ban/tests/files/logs/squid -fail2ban/tests/files/logs/stunnel -fail2ban/tests/files/logs/suhosin fail2ban/tests/files/logs/sogo-auth fail2ban/tests/files/logs/solid-pop3d +fail2ban/tests/files/logs/squid +fail2ban/tests/files/logs/squirrelmail fail2ban/tests/files/logs/sshd fail2ban/tests/files/logs/sshd-ddos +fail2ban/tests/files/logs/stunnel +fail2ban/tests/files/logs/suhosin +fail2ban/tests/files/logs/tine20 +fail2ban/tests/files/logs/uwimap-auth fail2ban/tests/files/logs/vsftpd fail2ban/tests/files/logs/webmin-auth fail2ban/tests/files/logs/wuftpd -fail2ban/tests/files/logs/uwimap-auth fail2ban/tests/files/logs/xinetd-fail -fail2ban/tests/config/jail.conf -fail2ban/tests/config/fail2ban.conf -fail2ban/tests/config/filter.d/simple.conf -fail2ban/tests/config/action.d/brokenaction.conf +fail2ban/tests/files/testcase-journal.log +fail2ban/tests/files/testcase-multiline.log +fail2ban/tests/files/testcase-usedns.log +fail2ban/tests/files/testcase01.log +fail2ban/tests/files/testcase02.log +fail2ban/tests/files/testcase03.log +fail2ban/tests/files/testcase04.log +fail2ban/tests/filtertestcase.py +fail2ban/tests/misctestcase.py +fail2ban/tests/samplestestcase.py +fail2ban/tests/servertestcase.py +fail2ban/tests/sockettestcase.py +fail2ban/tests/utils.py setup.py setup.cfg fail2ban/__init__.py @@ -169,126 +189,136 @@ fail2ban/exceptions.py fail2ban/helpers.py fail2ban/version.py fail2ban/protocol.py -setup.py -setup.cfg kill-server -config/jail.conf +config/action.d/apf.conf +config/action.d/badips.conf +config/action.d/badips.py +config/action.d/blocklist_de.conf +config/action.d/bsd-ipfw.conf +config/action.d/cloudflare.conf +config/action.d/complain.conf +config/action.d/dshield.conf +config/action.d/dummy.conf +config/action.d/firewallcmd-ipset.conf +config/action.d/firewallcmd-new.conf +config/action.d/hostsdeny.conf +config/action.d/ipfilter.conf +config/action.d/ipfw.conf +config/action.d/iptables-allports.conf +config/action.d/iptables-common.conf +config/action.d/iptables-ipset-proto4.conf +config/action.d/iptables-ipset-proto6-allports.conf +config/action.d/iptables-ipset-proto6.conf +config/action.d/iptables-multiport-log.conf +config/action.d/iptables-multiport.conf +config/action.d/iptables-new.conf +config/action.d/iptables-xt_recent-echo.conf +config/action.d/iptables.conf +config/action.d/mail-buffered.conf +config/action.d/mail-whois-lines.conf +config/action.d/mail-whois.conf +config/action.d/mail.conf +config/action.d/mynetwatchman.conf +config/action.d/osx-afctl.conf +config/action.d/osx-ipfw.conf +config/action.d/pf.conf +config/action.d/route.conf +config/action.d/sendmail-buffered.conf +config/action.d/sendmail-common.conf +config/action.d/sendmail-whois-ipjailmatches.conf +config/action.d/sendmail-whois-ipmatches.conf +config/action.d/sendmail-whois-lines.conf +config/action.d/sendmail-whois-matches.conf +config/action.d/sendmail-whois.conf +config/action.d/sendmail.conf +config/action.d/shorewall.conf +config/action.d/smtp.py +config/action.d/symbiosis-blacklist-allports.conf +config/action.d/ufw.conf +config/action.d/xarf-login-attack.conf config/fail2ban.conf -config/filter.d/common.conf +config/filter.d/3proxy.conf config/filter.d/apache-auth.conf config/filter.d/apache-badbots.conf config/filter.d/apache-botsearch.conf +config/filter.d/apache-common.conf config/filter.d/apache-modsecurity.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf -config/filter.d/nginx-http-auth.conf +config/filter.d/apache-shellshock.conf +config/filter.d/assp.conf +config/filter.d/asterisk.conf +config/filter.d/common.conf config/filter.d/counter-strike.conf config/filter.d/courier-auth.conf config/filter.d/courier-smtp.conf config/filter.d/cyrus-imap.conf +config/filter.d/directadmin.conf +config/filter.d/dovecot.conf +config/filter.d/dropbear.conf config/filter.d/ejabberd-auth.conf +config/filter.d/exim-common.conf +config/filter.d/exim-spam.conf config/filter.d/exim.conf config/filter.d/freeswitch.conf +config/filter.d/groupoffice.conf config/filter.d/gssftpd.conf -config/filter.d/kerio.conf +config/filter.d/guacamole.conf config/filter.d/horde.conf -config/filter.d/suhosin.conf +config/filter.d/kerio.conf +config/filter.d/lighttpd-auth.conf +config/filter.d/monit.conf +config/filter.d/mysqld-auth.conf config/filter.d/nagios.conf config/filter.d/named-refused.conf +config/filter.d/nginx-http-auth.conf config/filter.d/nsd.conf config/filter.d/openwebmail.conf +config/filter.d/oracleims.conf config/filter.d/pam-generic.conf +config/filter.d/pam-generic.conf +config/filter.d/pam-generic.conf +config/filter.d/perdition.conf config/filter.d/php-url-fopen.conf +config/filter.d/php-url-fopen.conf +config/filter.d/php-url-fopen.conf +config/filter.d/portsentry.conf +config/filter.d/postfix-sasl.conf config/filter.d/postfix-sasl.conf -config/filter.d/pam-generic.conf -config/filter.d/php-url-fopen.conf config/filter.d/postfix-sasl.conf config/filter.d/postfix.conf config/filter.d/proftpd.conf config/filter.d/pure-ftpd.conf config/filter.d/qmail.conf -config/filter.d/pam-generic.conf -config/filter.d/php-url-fopen.conf -config/filter.d/postfix-sasl.conf +config/filter.d/recidive.conf +config/filter.d/roundcube-auth.conf +config/filter.d/selinux-common.conf +config/filter.d/selinux-ssh.conf config/filter.d/sendmail-auth.conf config/filter.d/sendmail-reject.conf +config/filter.d/sendmail-spam.conf config/filter.d/sieve.conf +config/filter.d/sogo-auth.conf config/filter.d/solid-pop3d.conf config/filter.d/squid.conf -config/filter.d/sshd.conf +config/filter.d/squirrelmail.conf config/filter.d/sshd-ddos.conf +config/filter.d/sshd.conf config/filter.d/stunnel.conf +config/filter.d/suhosin.conf +config/filter.d/tine20.conf +config/filter.d/uwimap-auth.conf config/filter.d/vsftpd.conf config/filter.d/webmin-auth.conf config/filter.d/wuftpd.conf config/filter.d/xinetd-fail.conf -config/filter.d/asterisk.conf -config/filter.d/dovecot.conf -config/filter.d/dropbear.conf -config/filter.d/lighttpd-auth.conf -config/filter.d/recidive.conf -config/filter.d/roundcube-auth.conf -config/filter.d/assp.conf -config/filter.d/sogo-auth.conf -config/filter.d/mysqld-auth.conf -config/filter.d/selinux-common.conf -config/filter.d/selinux-ssh.conf -config/filter.d/3proxy.conf -config/filter.d/apache-common.conf -config/filter.d/exim-common.conf -config/filter.d/exim-spam.conf -config/filter.d/freeswitch.conf -config/filter.d/groupoffice.conf -config/filter.d/perdition.conf -config/filter.d/uwimap-auth.conf -config/filter.d/courier-auth.conf -config/filter.d/courier-smtp.conf -config/filter.d/ejabberd-auth.conf -config/filter.d/guacamole.conf -config/filter.d/sendmail-spam.conf -config/action.d/apf.conf -config/action.d/blocklist_de.conf -config/action.d/osx-afctl.conf -config/action.d/osx-ipfw.conf -config/action.d/sendmail-common.conf -config/action.d/badips.conf -config/action.d/bsd-ipfw.conf -config/action.d/dummy.conf -config/action.d/firewallcmd-new.conf -config/action.d/firewallcmd-ipset.conf -config/action.d/iptables-ipset-proto6-allports.conf -config/action.d/iptables-common.conf -config/action.d/iptables-ipset-proto4.conf -config/action.d/iptables-ipset-proto6.conf -config/action.d/iptables-xt_recent-echo.conf -config/action.d/route.conf -config/action.d/complain.conf -config/action.d/dshield.conf -config/action.d/hostsdeny.conf -config/action.d/ipfw.conf -config/action.d/ipfilter.conf -config/action.d/iptables.conf -config/action.d/iptables-allports.conf -config/action.d/iptables-multiport.conf -config/action.d/iptables-multiport-log.conf -config/action.d/iptables-new.conf -config/action.d/mail.conf -config/action.d/mail-buffered.conf -config/action.d/mail-whois.conf -config/action.d/mail-whois-lines.conf -config/action.d/mynetwatchman.conf -config/action.d/pf.conf -config/action.d/sendmail.conf -config/action.d/sendmail-buffered.conf -config/action.d/sendmail-whois-ipmatches.conf -config/action.d/sendmail-whois.conf -config/action.d/sendmail-whois-lines.conf -config/action.d/shorewall.conf -config/action.d/xarf-login-attack.conf -config/action.d/ufw.conf -config/fail2ban.conf -doc/run-rootless.txt +config/jail.conf +config/paths-common.conf +config/paths-debian.conf +config/paths-fedora.conf +config/paths-freebsd.conf +config/paths-osx.conf man/fail2ban-client.1 man/fail2ban.1 man/jail.conf.5 @@ -306,6 +336,8 @@ files/solaris-fail2ban.xml files/solaris-svc-fail2ban files/suse-initd files/fail2ban-logrotate +files/fail2ban.upstart +files/logwatch/fail2ban files/cacti/fail2ban_stats.sh files/cacti/cacti_host_template_fail2ban.xml files/cacti/README diff --git a/MANIFEST.in b/MANIFEST.in index a0d9c2a3..9473d3a6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include ChangeLog COPYING DEVELOP FILTERS README.* THANKS TODO +include ChangeLog COPYING DEVELOP FILTERS README.* THANKS TODO CONTRIBUTING* Vagrantfile graft doc graft files recursive-include config *.conf *.py diff --git a/RELEASE b/RELEASE index 9002b75e..6cc60400 100644 --- a/RELEASE +++ b/RELEASE @@ -47,6 +47,10 @@ Preparation * Ensure the MANIFEST is complete + ad-hoc bash script to run in a clean clone: + + find -type f | grep -v -e '\.git' -e '/doc/' -e '\.travis' -e MANIFEST | sed -e 's,^\./,,g' | while read f; do grep -ne "^$f\$" MANIFEST >/dev/null || echo "$f" ; done + * Run:: python setup.py sdist From 987356d6c0cf9ed14ca683341b3fdc6c6041e87d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 21:38:15 -0400 Subject: [PATCH 33/36] Changes for the 0.9.1 release versioning --- ChangeLog | 4 ++-- README.md | 2 +- RELEASE | 2 +- fail2ban/version.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3e9d99b4..cc3999ce 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,10 +4,10 @@ |_| \__,_|_|_/___|_.__/\__,_|_||_| ================================================================================ -Fail2Ban (version 0.9.0.dev) 2014/xx/xx +Fail2Ban (version 0.9.1) 2014/10/29 ================================================================================ -ver. 0.9.1 (2014/xx/xx) - better, faster, stronger +ver. 0.9.1 (2014/10/29) - better, faster, stronger ---------- - Refactoring (IMPORTANT -- Please review your setup and configuration): diff --git a/README.md b/README.md index 2679fe53..54d5e3bb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.0 2014/03/14 + v0.9.1 2014/10/29 ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/RELEASE b/RELEASE index 6cc60400..9ce304c0 100644 --- a/RELEASE +++ b/RELEASE @@ -185,7 +185,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.1 (2014/XX/XXX) - wanna-be-released + ver. 0.9.2 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: diff --git a/fail2ban/version.py b/fail2ban/version.py index 914cfe29..f7b776cd 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2014 Yaroslav Halchenko, 2013-2013 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.0.dev" +version = "0.9.1" From 564eb3389bd226c16e3f3f4e6f0873f9a24489fb Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 21:45:43 -0400 Subject: [PATCH 34/36] ENH: fail early in generate-man + provide PYTHONPATH upstairs --- man/generate-man | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/man/generate-man b/man/generate-man index 0907e738..f18c3604 100755 --- a/man/generate-man +++ b/man/generate-man @@ -1,5 +1,9 @@ #!/bin/bash +set -eu + +export PYTHONPATH=.. + # fail2ban-client echo -n "Generating fail2ban-client " help2man --section=1 --no-info --include=fail2ban-client.h2m --output fail2ban-client.1 ../bin/fail2ban-client From d0a5fe620f87c3a7b57c874d0280353faba1d369 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 21:47:48 -0400 Subject: [PATCH 35/36] Refreshed manpages --- man/fail2ban-client.1 | 52 ++++++++++++++++++------------------------- man/fail2ban-regex.1 | 21 +++++++++-------- man/fail2ban-server.1 | 8 +++---- 3 files changed, 38 insertions(+), 43 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index e2df68ed..c760f8ed 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.41.2. -.TH FAIL2BAN-CLIENT "1" "March 2014" "fail2ban-client v0.9.0" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. +.TH FAIL2BAN-CLIENT "1" "October 2014" "fail2ban-client v0.9.1" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client -[\fIOPTIONS\fR] \fI\fR +[\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.9.0 reads log file that contains password failure report +Fail2Ban v0.9.1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP @@ -35,10 +35,10 @@ decrease verbosity force execution of the server (remove socket file) .TP \fB\-b\fR -start the server in background mode (default) +start server in background (default) .TP \fB\-f\fR -start the server in foreground mode (note that the client forks once itself) +start server in foreground (note that the client forks once itself) .TP \fB\-h\fR, \fB\-\-help\fR display this help message @@ -217,9 +217,8 @@ for \fBset maxlines \fR sets the number of to buffer for regex search for -.IP -set addaction [ ] -.IP +.TP +\fBset addaction [ ]\fR adds a new action named for . Optionally for a Python based action, a and @@ -231,45 +230,38 @@ removes the action from .IP COMMAND ACTION CONFIGURATION -.IP -set action actionstart -.IP +.TP +\fBset action actionstart \fR sets the start command of the action for -.IP -set action actionstop sets the stop command of the -.IP +.TP +\fBset action actionstop sets the stop command of the\fR action for -.IP -set action actioncheck -.IP +.TP +\fBset action actioncheck \fR sets the check command of the action for .TP \fBset action actionban \fR sets the ban command of the action for -.IP -set action actionunban -.IP +.TP +\fBset action actionunban \fR sets the unban command of the action for -.IP -set action timeout -.IP +.TP +\fBset action timeout \fR sets as the command timeout in seconds for the action for .IP GENERAL ACTION CONFIGURATION -.IP -set action -.IP +.TP +\fBset action \fR sets the of for the action for -.IP -set action [ ] -.IP +.TP +\fBset action [ ]\fR calls the with for the action for diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index ec773b02..05379351 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,10 +1,10 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.41.2. -.TH FAIL2BAN-REGEX "1" "March 2014" "fail2ban-regex 0.9.0" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. +.TH FAIL2BAN-REGEX "1" "October 2014" "fail2ban-regex 0.9.1" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS .B fail2ban-regex -[\fIOPTIONS\fR] \fI \fR[\fIIGNOREREGEX\fR] +[\fI\,OPTIONS\/\fR] \fI\, \/\fR[\fI\,IGNOREREGEX\/\fR] .SH DESCRIPTION Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. @@ -16,7 +16,7 @@ string a string representing a log line .TP filename -path to a log file (\fI/var/log/auth.log\fP) +path to a log file (\fI\,/var/log/auth.log\/\fP) .TP "systemd\-journal" search systemd journal (systemd\-python required) @@ -42,20 +42,20 @@ show program's version number and exit \fB\-h\fR, \fB\-\-help\fR show this help message and exit .TP -\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fIDATEPATTERN\fR +\fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR set custom pattern used to match date/times .TP -\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fIENCODING\fR +\fB\-e\fR ENCODING, \fB\-\-encoding\fR=\fI\,ENCODING\/\fR File encoding. Default: system locale .TP -\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fIMAXLINES\fR +\fB\-L\fR MAXLINES, \fB\-\-maxlines\fR=\fI\,MAXLINES\/\fR maxlines for multi\-line regex .TP -\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fIJOURNALMATCH\fR +\fB\-m\fR JOURNALMATCH, \fB\-\-journalmatch\fR=\fI\,JOURNALMATCH\/\fR journalctl style matches overriding filter file. "systemd\-journal" only .TP -\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fILOG_LEVEL\fR +\fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR Log level for the Fail2Ban logger to use .TP \fB\-v\fR, \fB\-\-verbose\fR @@ -70,6 +70,9 @@ Do not print any missed lines \fB\-\-print\-no\-ignored\fR Do not print any ignored lines .TP +\fB\-\-print\-all\-matched\fR +Print all matched lines +.TP \fB\-\-print\-all\-missed\fR Print all missed lines, no matter how many .TP diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index b49c0173..6aa51053 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.41.2. -.TH FAIL2BAN-SERVER "1" "March 2014" "fail2ban-server v0.9.0" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. +.TH FAIL2BAN-SERVER "1" "October 2014" "fail2ban-server v0.9.1" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server -[\fIOPTIONS\fR] +[\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.9.0 reads log file that contains password failure report +Fail2Ban v0.9.1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with From a0115ee4582e0befbcd7664989b6ab87f8d226a2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 27 Oct 2014 21:49:40 -0400 Subject: [PATCH 36/36] ENH: additional versioning changes --- README.md | 4 ++-- RELEASE | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 54d5e3bb..6afb5457 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.0.tar.bz2 - cd fail2ban-0.9.0 + tar xvfj fail2ban-0.9.1.tar.bz2 + cd fail2ban-0.9.1 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index 9ce304c0..23bb8382 100644 --- a/RELEASE +++ b/RELEASE @@ -61,24 +61,24 @@ Preparation * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: - tar -C /tmp -jxf dist/fail2ban-0.9.0.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.9.1.tar.bz2 * clean up current direcory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.0/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.1/ * Only differences should be files that you don't want distributed. * Ensure the tests work from the tarball:: - cd /tmp/fail2ban-0.9.0/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases + cd /tmp/fail2ban-0.9.1/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases * Add/finalize the corresponding entry in the ChangeLog * To generate a list of committers use e.g.:: - git shortlog -sn 0.8.12.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.1.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date.