diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b7da271b..04502504 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -29,24 +29,12 @@ import os from ConfigParser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel -from ..helpers import getLogger, substituteRecursiveTags +from ..helpers import getLogger, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) -# if sys.version_info >= (3,5): -# def _merge_dicts(x, y): -# return {**x, **y} -# else: -def _merge_dicts(x, y): - r = x - if y: - r = x.copy() - r.update(y) - return r - - class ConfigReader(): """Generic config reader class. diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 911abd28..782513a7 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -49,14 +49,14 @@ from ..version import version from .jailreader import JailReader from .filterreader import FilterReader from ..server.filter import Filter, FileContainer -from ..server.failregex import RegexException +from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, PREFER_ENC # Gets the instance of the logger. logSys = getLogger("fail2ban") -def debuggexURL(sample, regex): - q = urllib.urlencode({ 're': regex.replace('', '(?&.ipv4)'), +def debuggexURL(sample, regex, useDns="yes"): + q = urllib.urlencode({ 're': Regex._resolveHostTag(regex, useDns=useDns), 'str': sample, 'flavor': 'python' }) return 'https://www.debuggex.com/?' + q @@ -198,15 +198,17 @@ class RegexStat(object): class LineStats(object): """Just a convenience container for stats """ - def __init__(self): + def __init__(self, opts): self.tested = self.matched = 0 self.matched_lines = [] self.missed = 0 self.missed_lines = [] - self.missed_lines_timeextracted = [] self.ignored = 0 self.ignored_lines = [] - self.ignored_lines_timeextracted = [] + if opts.debuggex: + self.matched_lines_timeextracted = [] + self.missed_lines_timeextracted = [] + self.ignored_lines_timeextracted = [] def __str__(self): return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self @@ -230,7 +232,7 @@ class Fail2banRegex(object): self._ignoreregex = list() self._failregex = list() self._time_elapsed = None - self._line_stats = LineStats() + self._line_stats = LineStats(opts) if opts.maxlines: self.setMaxLines(opts.maxlines) @@ -414,9 +416,10 @@ class Fail2banRegex(object): try: self._line_stats.missed_lines.pop( self._line_stats.missed_lines.index("".join(bufLine))) - self._line_stats.missed_lines_timeextracted.pop( - self._line_stats.missed_lines_timeextracted.index( - "".join(bufLine[::2]))) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.pop( + self._line_stats.missed_lines_timeextracted.index( + "".join(bufLine[::2]))) except ValueError: pass else: @@ -443,19 +446,23 @@ class Fail2banRegex(object): self._line_stats.ignored += 1 if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1): self._line_stats.ignored_lines.append(line) - self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) + if self._debuggex: + self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) if len(ret) > 0: assert(not is_ignored) self._line_stats.matched += 1 if self._print_all_matched: self._line_stats.matched_lines.append(line) + if self._debuggex: + self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) else: if not is_ignored: self._line_stats.missed += 1 if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): self._line_stats.missed_lines.append(line) - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 self._time_elapsed = time.time() - t0 @@ -478,7 +485,7 @@ class Fail2banRegex(object): for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + - debuggexURL(self.encode_line(a[0]), a[1].getFailRegex()), ans) + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), self._opts.usedns), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 2407df1f..556ca173 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -169,6 +169,36 @@ def splitwords(s): return [] return filter(bool, map(str.strip, re.split('[ ,\n]+', s))) +if sys.version_info >= (3,5): + eval(compile(r'''if 1: + def _merge_dicts(x, y): + """Helper to merge dicts. + """ + if y: + return {**x, **y} + return x + + def _merge_copy_dicts(x, y): + """Helper to merge dicts to guarantee a copy result (r is never x). + """ + return {**x, **y} + ''', __file__, 'exec')) +else: + def _merge_dicts(x, y): + """Helper to merge dicts. + """ + r = x + if y: + r = x.copy() + r.update(y) + return r + def _merge_copy_dicts(x, y): + """Helper to merge dicts to guarantee a copy result (r is never x). + """ + r = x.copy() + if y: + r.update(y) + return r # # Following "uni_decode" function unified python independent any to string converting @@ -240,6 +270,7 @@ def substituteRecursiveTags(inptags, conditional='', # init: ignore = set(ignore) done = set() + noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) while True: repFlag = False @@ -247,6 +278,8 @@ def substituteRecursiveTags(inptags, conditional='', for tag in tags.iterkeys(): # ignore escaped or already done (or in ignore list): if tag in ignore or tag in done: continue + # ignore replacing callable items from calling map - should be converted on demand only (by get): + if noRecRepl and callable(tags.getRawItem(tag)): continue value = orgval = str(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) @@ -281,6 +314,8 @@ def substituteRecursiveTags(inptags, conditional='', # constructs like . m = tre_search(value, m.end()) continue + # if calling map - be sure we've string: + if noRecRepl: repl = str(repl) value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 075c5c72..b0bcdf92 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -36,7 +36,7 @@ from .failregex import mapTag2Opt from .ipdns import asip from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT +from ..helpers import getLogger, _merge_copy_dicts, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT # Gets the instance of the logger. logSys = getLogger(__name__) @@ -98,6 +98,13 @@ class CallingMap(MutableMapping, object): except: return dict(self.data, **self.storage) + def getRawItem(self, key): + try: + value = self.storage[key] + except KeyError: + value = self.data[key] + return value + def __getitem__(self, key): try: value = self.storage[key] @@ -141,7 +148,7 @@ class CallingMap(MutableMapping, object): return len(self.data) def copy(self): # pargma: no cover - return self.__class__(self.data.copy()) + return self.__class__(_merge_copy_dicts(self.data, self.storage)) class ActionBase(object): @@ -446,7 +453,7 @@ class CommandAction(ActionBase): return value @classmethod - def replaceTag(cls, query, aInfo, conditional='', cache=None): + def replaceTag(cls, query, aInfo, conditional='', cache=None, substRec=True): """Replaces tags in `query` with property values. Parameters @@ -471,23 +478,29 @@ class CommandAction(ActionBase): except KeyError: pass - # first try get cached tags dictionary: - subInfo = csubkey = None - if cache is not None: - csubkey = ('subst-tags', id(aInfo), conditional) - try: - subInfo = cache[csubkey] - except KeyError: - pass - # interpolation of dictionary: - if subInfo is None: - subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) - # cache if possible: - if csubkey is not None: - cache[csubkey] = subInfo + # **Important**: don't replace if calling map - contains dynamic values only, + # no recursive tags, otherwise may be vulnerable on foreign user-input: + noRecRepl = isinstance(aInfo, CallingMap) + if noRecRepl: + subInfo = aInfo + else: + # substitute tags recursive (and cache if possible), + # first try get cached tags dictionary: + subInfo = csubkey = None + if cache is not None: + csubkey = ('subst-tags', id(aInfo), conditional) + try: + subInfo = cache[csubkey] + except KeyError: + pass + # interpolation of dictionary: + if subInfo is None: + subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) + # cache if possible: + if csubkey is not None: + cache[csubkey] = subInfo # substitution callable, used by interpolation of each tag - repeatSubst = {0: 0} def substVal(m): tag = m.group(1) # tagname from match value = None @@ -503,18 +516,17 @@ class CommandAction(ActionBase): # That one needs to be escaped since its content is # out of our control value = cls.escapeTag(value) - # possible contains tags: - if '<' in value: - repeatSubst[0] = 1 + # replacement for tag: return value # interpolation of query: count = MAX_TAG_REPLACE_COUNT + 1 while True: - repeatSubst[0] = 0 value = TAG_CRE.sub(substVal, query) + # **Important**: no recursive replacement for tags from calling map (properties only): + if noRecRepl: break # possible recursion ? - if not repeatSubst or value == query: break + if value == query or '<' not in value: break query = value count -= 1 if count <= 0: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 8834ab8d..70562baf 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -157,6 +157,37 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(substituteRecursiveTags({'A': 'A HOST> B IP C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}), {'A': 'A 1.2.3.4 B IPV4 C', 'PREF': 'V4', 'IPV4HOST': '1.2.3.4'}) + def testSubstRec_DontTouchUnusedCallable(self): + cm = CallingMap({ + 'A':0, + 'B':lambda self: '', + 'C':'', + 'D':'' + }) + # + # should raise no exceptions: + substituteRecursiveTags(cm) + # add exception tag: + cm['C'] = lambda self,i=0: 5 // int(self['A']) # raise error by access + # test direct get of callable (should raise an error): + self.assertRaises(ZeroDivisionError, lambda: cm['C']) + # should raise no exceptions (tag "C" still unused): + substituteRecursiveTags(cm) + # add reference to "broken" tag: + cm['D'] = 'test=' + # should raise an exception (BOOM by replacement of tag "D" recursive): + self.assertRaises(ZeroDivisionError, lambda: substituteRecursiveTags(cm)) + # + # should raise no exceptions: + self.assertEqual(self.__action.replaceTag('test=', cm), "test=0") + # **Important**: recursive replacement of dynamic data from calling map should be prohibited, + # otherwise may be vulnerable on foreign user-input: + self.assertEqual(self.__action.replaceTag('test=----', cm), "test=0----0") + # should raise an exception (BOOM by replacement of tag "C"): + self.assertRaises(ZeroDivisionError, lambda: self.__action.replaceTag('test=', cm)) + # should raise no exceptions (replaces tag "D" only): + self.assertEqual(self.__action.replaceTag('', cm), "test=") + def testReplaceTag(self): aInfo = { 'HOST': "192.0.2.0", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 1bac3a5f..0cd0e303 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -286,11 +286,12 @@ class Fail2banRegexTest(LogCaptureTestCase): "-l", "notice", # put down log-level, because of too many debug-messages "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", - Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD, + r"llinco[^\\]" ) self.assertTrue(fail2banRegex.start(args)) self.assertLogged('Error decoding line') - self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + self.assertLogged('Lines: 4 lines, 1 ignored, 2 matched, 1 missed') self.assertLogged('https://') diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index aa0676ec..23c52a43 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -269,7 +269,9 @@ def initTests(opts): # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName - for i in xrange(255): + # increase max count and max time (too many entries, long time testing): + c.setOptions(maxCount=10000, maxTime=5*60) + for i in xrange(256): c.set('192.0.2.%s' % i, None) c.set('198.51.100.%s' % i, None) c.set('203.0.113.%s' % i, None)