Merge remote-tracking branch 'remotes/upstream/master' into sebres:ban-time-incr

pull/716/head
sebres 2014-12-01 23:28:14 +01:00
commit 5ca275876b
9 changed files with 106 additions and 22 deletions

View File

@ -11,19 +11,26 @@ ver. 0.9.2 (2014/xx/xx) - increment ban time
---------- ----------
- Fixes: - Fixes:
* purge database will be executed now (within observer).
* database functionality extended with bad ips.
* restoring currently banned ip after service restart fixed
(now < timeofban + bantime), ignore old log failures (already banned)
* $ typo in jail.conf. Thanks Skibbi. Debian bug #767255 * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255
* grep'ing for IP in *mail-whois-lines.conf should now match also * grep'ing for IP in *mail-whois-lines.conf should now match also
at the begginning and EOL. Thanks Dean Lee at the begginning and EOL. Thanks Dean Lee
* jail.conf * jail.conf
- php-url-fopen: separate logpath entries by newline - php-url-fopen: separate logpath entries by newline
* purge database will be executed now (within observer). * failregex declared direct in jail was joined to single line (specifying of
* database functionality extended with bad ips. multiple expressions was not possible).
* restoring currently banned ip after service restart fixed
(now < timeofban + bantime), ignore old log failures (already banned)
- New Features: - New Features:
* increment ban time (+ observer) functionality introduced. * increment ban time (+ observer) functionality introduced.
Thanks Serg G. Brester (sebres) Thanks Serg G. Brester (sebres)
* New interpolation feature for config readers - `%(known/parameter)s`.
(means last known option with name `parameter`). This interpolation makes
possible to extend a stock filter or jail regexp in .local file
(opposite to simply set failregex/ignoreregex that overwrites it),
see gh-867.
- Enhancements: - Enhancements:
* Enable multiport for firewallcmd-new action. Closes gh-834 * Enable multiport for firewallcmd-new action. Closes gh-834

View File

@ -226,6 +226,13 @@ after = 1.conf
if isinstance(s, dict): if isinstance(s, dict):
s2 = alls.get(n) s2 = alls.get(n)
if isinstance(s2, dict): if isinstance(s2, dict):
# save previous known values, for possible using in local interpolations later:
sk = {}
for k, v in s2.iteritems():
if not k.startswith('known/'):
sk['known/'+k] = v
s2.update(sk)
# merge section
s2.update(s) s2.update(s)
else: else:
alls[n] = s.copy() alls[n] = s.copy()
@ -242,3 +249,12 @@ after = 1.conf
return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8') return SafeConfigParser.read(self, fileNamesFull, encoding='utf-8')
else: else:
return SafeConfigParser.read(self, fileNamesFull) return SafeConfigParser.read(self, fileNamesFull)
def merge_section(self, section, options, pref='known/'):
alls = self.get_sections()
sk = {}
for k, v in options.iteritems():
if pref == '' or not k.startswith(pref):
sk[pref+k] = v
alls[section].update(sk)

View File

@ -116,6 +116,10 @@ class ConfigReader():
return self._cfg.has_section(sec) return self._cfg.has_section(sec)
return False return False
def merge_section(self, *args, **kwargs):
if self._cfg is not None:
return self._cfg.merge_section(*args, **kwargs)
def options(self, *args): def options(self, *args):
if self._cfg is not None: if self._cfg is not None:
return self._cfg.options(*args) return self._cfg.options(*args)

View File

@ -47,12 +47,20 @@ class FilterReader(DefinitionInitConfigReader):
def getFile(self): def getFile(self):
return self.__file return self.__file
def convert(self): def getCombined(self):
stream = list()
combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items())) combinedopts = dict(list(self._opts.items()) + list(self._initOpts.items()))
if not len(combinedopts):
return {};
opts = CommandAction.substituteRecursiveTags(combinedopts) opts = CommandAction.substituteRecursiveTags(combinedopts)
if not opts: if not opts:
raise ValueError('recursive tag definitions unable to be resolved') raise ValueError('recursive tag definitions unable to be resolved')
return opts;
def convert(self):
stream = list()
opts = self.getCombined()
if not len(opts):
return stream;
for opt, value in opts.iteritems(): for opt, value in opts.iteritems():
if opt == "failregex": if opt == "failregex":
for regex in value.split('\n'): for regex in value.split('\n'):

View File

@ -87,6 +87,8 @@ class JailReader(ConfigReader):
return pathList return pathList
def getOptions(self): def getOptions(self):
opts1st = [["bool", "enabled", False],
["string", "filter", ""]]
opts = [["bool", "enabled", False], opts = [["bool", "enabled", False],
["string", "logpath", None], ["string", "logpath", None],
["string", "logencoding", None], ["string", "logencoding", None],
@ -108,7 +110,9 @@ class JailReader(ConfigReader):
["string", "ignoreip", None], ["string", "ignoreip", None],
["string", "filter", ""], ["string", "filter", ""],
["string", "action", ""]] ["string", "action", ""]]
self.__opts = ConfigReader.getOptions(self, self.__name, opts)
# Read first options only needed for merge defaults ('known/...' from filter):
self.__opts = ConfigReader.getOptions(self, self.__name, opts1st)
if not self.__opts: if not self.__opts:
return False return False
@ -120,15 +124,25 @@ class JailReader(ConfigReader):
self.__filter = FilterReader( self.__filter = FilterReader(
filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
ret = self.__filter.read() ret = self.__filter.read()
if ret: # merge options from filter as 'known/...':
self.__filter.getOptions(self.__opts) self.__filter.getOptions(self.__opts)
else: ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/')
if not ret:
logSys.error("Unable to read the filter") logSys.error("Unable to read the filter")
return False return False
else: else:
self.__filter = None self.__filter = None
logSys.warning("No filter set for jail %s" % self.__name) logSys.warning("No filter set for jail %s" % self.__name)
# Read second all options (so variables like %(known/param) can be interpolated):
self.__opts = ConfigReader.getOptions(self, self.__name, opts)
if not self.__opts:
return False
# cumulate filter options again (ignore given in jail):
if self.__filter:
self.__filter.getOptions(self.__opts)
# Read action # Read action
for act in self.__opts["action"].split('\n'): for act in self.__opts["action"].split('\n'):
try: try:
@ -211,7 +225,10 @@ class JailReader(ConfigReader):
elif opt == "usedns": elif opt == "usedns":
stream.append(["set", self.__name, "usedns", self.__opts[opt]]) stream.append(["set", self.__name, "usedns", self.__opts[opt]])
elif opt == "failregex": elif opt == "failregex":
stream.append(["set", self.__name, "addfailregex", self.__opts[opt]]) for regex in self.__opts[opt].split('\n'):
# Do not send a command if the rule is empty.
if regex != '':
stream.append(["set", self.__name, "addfailregex", regex])
elif opt == "ignorecommand": elif opt == "ignorecommand":
stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]]) stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]])
elif opt == "ignoreregex": elif opt == "ignoreregex":

View File

@ -155,12 +155,16 @@ c = d ;in line comment
class JailReaderTest(LogCaptureTestCase): class JailReaderTest(LogCaptureTestCase):
def __init__(self, *args, **kwargs):
super(JailReaderTest, self).__init__(*args, **kwargs)
self.__share_cfg = {}
def testIncorrectJail(self): def testIncorrectJail(self):
jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR) jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg)
self.assertRaises(ValueError, jail.read) self.assertRaises(ValueError, jail.read)
def testJailActionEmpty(self): def testJailActionEmpty(self):
jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG) jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg)
self.assertTrue(jail.read()) self.assertTrue(jail.read())
self.assertTrue(jail.getOptions()) self.assertTrue(jail.getOptions())
self.assertTrue(jail.isEnabled()) self.assertTrue(jail.isEnabled())
@ -168,7 +172,7 @@ class JailReaderTest(LogCaptureTestCase):
self.assertTrue(self._is_logged('No actions were defined for emptyaction')) self.assertTrue(self._is_logged('No actions were defined for emptyaction'))
def testJailActionFilterMissing(self): def testJailActionFilterMissing(self):
jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG) jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg)
self.assertTrue(jail.read()) self.assertTrue(jail.read())
self.assertFalse(jail.getOptions()) self.assertFalse(jail.getOptions())
self.assertTrue(jail.isEnabled()) self.assertTrue(jail.isEnabled())
@ -176,7 +180,7 @@ class JailReaderTest(LogCaptureTestCase):
self.assertTrue(self._is_logged('Unable to read the filter')) self.assertTrue(self._is_logged('Unable to read the filter'))
def TODOtestJailActionBrokenDef(self): def TODOtestJailActionBrokenDef(self):
jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG) jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg)
self.assertTrue(jail.read()) self.assertTrue(jail.read())
self.assertFalse(jail.getOptions()) self.assertFalse(jail.getOptions())
self.assertTrue(jail.isEnabled()) self.assertTrue(jail.isEnabled())
@ -187,7 +191,7 @@ class JailReaderTest(LogCaptureTestCase):
if STOCK: if STOCK:
def testStockSSHJail(self): def testStockSSHJail(self):
jail = JailReader('sshd', basedir=CONFIG_DIR) # we are running tests from root project dir atm jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm
self.assertTrue(jail.read()) self.assertTrue(jail.read())
self.assertTrue(jail.getOptions()) self.assertTrue(jail.getOptions())
self.assertFalse(jail.isEnabled()) self.assertFalse(jail.isEnabled())
@ -411,13 +415,17 @@ class JailsReaderTestCache(LogCaptureTestCase):
class JailsReaderTest(LogCaptureTestCase): class JailsReaderTest(LogCaptureTestCase):
def __init__(self, *args, **kwargs):
super(JailsReaderTest, self).__init__(*args, **kwargs)
self.__share_cfg = {}
def testProvidingBadBasedir(self): def testProvidingBadBasedir(self):
if not os.path.exists('/XXX'): if not os.path.exists('/XXX'):
reader = JailsReader(basedir='/XXX') reader = JailsReader(basedir='/XXX')
self.assertRaises(ValueError, reader.read) self.assertRaises(ValueError, reader.read)
def testReadTestJailConf(self): def testReadTestJailConf(self):
jails = JailsReader(basedir=IMPERFECT_CONFIG) jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg)
self.assertTrue(jails.read()) self.assertTrue(jails.read())
self.assertFalse(jails.getOptions()) self.assertFalse(jails.getOptions())
self.assertRaises(ValueError, jails.convert) self.assertRaises(ValueError, jails.convert)
@ -425,6 +433,11 @@ class JailsReaderTest(LogCaptureTestCase):
self.maxDiff = None self.maxDiff = None
self.assertEqual(sorted(comm_commands), self.assertEqual(sorted(comm_commands),
sorted([['add', 'emptyaction', 'auto'], sorted([['add', 'emptyaction', 'auto'],
['add', 'test-known-interp', 'auto'],
['set', 'test-known-interp', 'addfailregex', 'failure test 1 (filter.d/test.conf) <HOST>'],
['set', 'test-known-interp', 'addfailregex', 'failure test 2 (filter.d/test.local) <HOST>'],
['set', 'test-known-interp', 'addfailregex', 'failure test 3 (jail.local) <HOST>'],
['start', 'test-known-interp'],
['add', 'missinglogfiles', 'auto'], ['add', 'missinglogfiles', 'auto'],
['set', 'missinglogfiles', 'addfailregex', '<IP>'], ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
['add', 'brokenaction', 'auto'], ['add', 'brokenaction', 'auto'],
@ -447,7 +460,7 @@ class JailsReaderTest(LogCaptureTestCase):
if STOCK: if STOCK:
def testReadStockJailConf(self): def testReadStockJailConf(self):
jails = JailsReader(basedir=CONFIG_DIR) # we are running tests from root project dir atm jails = JailsReader(basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm
self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.read()) # opens fine
self.assertTrue(jails.getOptions()) # reads fine self.assertTrue(jails.getOptions()) # reads fine
comm_commands = jails.convert() comm_commands = jails.convert()
@ -508,7 +521,7 @@ class JailsReaderTest(LogCaptureTestCase):
# Verify that all filters found under config/ have a jail # Verify that all filters found under config/ have a jail
def testReadStockJailFilterComplete(self): def testReadStockJailFilterComplete(self):
jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg)
self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.read()) # opens fine
self.assertTrue(jails.getOptions()) # reads fine self.assertTrue(jails.getOptions()) # reads fine
# grab all filter names # grab all filter names
@ -525,7 +538,7 @@ class JailsReaderTest(LogCaptureTestCase):
def testReadStockJailConfForceEnabled(self): def testReadStockJailConfForceEnabled(self):
# more of a smoke test to make sure that no obvious surprises # more of a smoke test to make sure that no obvious surprises
# on users' systems when enabling shipped jails # on users' systems when enabling shipped jails
jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) # we are running tests from root project dir atm jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=self.__share_cfg) # we are running tests from root project dir atm
self.assertTrue(jails.read()) # opens fine self.assertTrue(jails.read()) # opens fine
self.assertTrue(jails.getOptions()) # reads fine self.assertTrue(jails.getOptions()) # reads fine
comm_commands = jails.convert(allow_no_files=True) comm_commands = jails.convert(allow_no_files=True)
@ -620,7 +633,7 @@ action = testaction1[actname=test1]
filter = testfilter1 filter = testfilter1
""") """)
jailfd.close() jailfd.close()
jails = JailsReader(basedir=basedir) jails = JailsReader(basedir=basedir, share_config=self.__share_cfg)
self.assertTrue(jails.read()) self.assertTrue(jails.read())
self.assertTrue(jails.getOptions()) self.assertTrue(jails.getOptions())
comm_commands = jails.convert(allow_no_files=True) comm_commands = jails.convert(allow_no_files=True)

View File

@ -0,0 +1,6 @@
#[INCLUDES]
#before = common.conf
[Definition]
failregex = failure test 1 (filter.d/test.conf) <HOST>

View File

@ -0,0 +1,7 @@
#[INCLUDES]
#before = common.conf
[Definition]
failregex = %(known/failregex)s
failure test 2 (filter.d/test.local) <HOST>

View File

@ -13,6 +13,12 @@ failregex = <IP>
ignoreregex = ignoreregex =
ignoreip = ignoreip =
[test-known-interp]
enabled = true
filter = test
failregex = %(known/failregex)s
failure test 3 (jail.local) <HOST>
[missinglogfiles] [missinglogfiles]
enabled = true enabled = true
logpath = /weapons/of/mass/destruction logpath = /weapons/of/mass/destruction