Merge pull request #1669 from sebres/0.10-recognize-restored-tickets

Recognize state of restored tickets
pull/1597/merge
Serg G. Brester 8 years ago committed by GitHub
commit 5bfdd521f0

@ -165,6 +165,31 @@ fail2ban-client set loglevel INFO
- faster match and fewer searching of appropriate templates
(DateDetector.matchTime calls rarer DateTemplate.matchDate now);
- several standard filters extended with exact prefixed or anchored date templates;
* Added possibility to recognize restored state of the tickets (see gh-1669).
New option `norestored` introduced, to ignore restored tickets (after restart).
To avoid execution of ban/unban for the restored tickets, `norestored = true`
could be added in definition section of action.
For conditional usage in the shell-based actions an interpolation `<restored>`
could be used also. E. g. it is enough to add following script-piece at begin
of `actionban` (or `actionunban`) to prevent execution:
`if [ '<restored>' = '1' ]; then exit 0; fi;`
Several actions extended now using `norestored` option:
- complain.conf
- dshield.conf
- mail-buffered.conf
- mail-whois-lines.conf
- mail-whois.conf
- mail.conf
- sendmail-buffered.conf
- sendmail-geoip-lines.conf
- sendmail-whois-ipjailmatches.conf
- sendmail-whois-ipmatches.conf
- sendmail-whois-lines.conf
- sendmail-whois-matches.conf
- sendmail-whois.conf
- sendmail.conf
- smtp.py
- xarf-login-attack.conf
* fail2ban-testcases:
- `assertLogged` extended with parameter wait (to wait up to specified timeout,
before we throw assert exception) + test cases rewritten using that

@ -34,6 +34,9 @@ before = helpers-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -28,6 +28,9 @@
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -7,6 +7,9 @@
_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
# Used for actions, that should not by executed if ticket was restored:
_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
[Init]
greplimit = tail -n <grepmax>
grepmax = 1000

@ -6,6 +6,9 @@
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -11,6 +11,9 @@ before = mail-whois-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD
@ -52,6 +55,7 @@ _ban_mail_content = ( printf %%b "Hi,\n
printf %%b "\n
Regards,\n
Fail2Ban" )
actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from `uname -n`" <dest>
# Option: actionunban

@ -10,6 +10,9 @@ before = mail-whois-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -6,6 +6,9 @@
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionstart
# Notes.: command executed once at the start of Fail2Ban.
# Values: CMD

@ -11,6 +11,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: Command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -11,6 +11,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -10,6 +10,9 @@ before = sendmail-common.conf
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
# Option: actionban
# Notes.: command executed when banning an IP. Take care that the
# command is executed with Fail2Ban user rights.

@ -126,6 +126,9 @@ class SMTPAction(ActionBase):
bantime = self._jail.actions.getBanTime,
)
# bypass ban/unban for restored tickets
self.norestored = 1
def _sendMessage(self, subject, text):
"""Sends message based on arguments and instance's properties.
@ -211,6 +214,8 @@ class SMTPAction(ActionBase):
Dictionary which includes information in relation to
the ban.
"""
if aInfo.get('restored'):
return
aInfo.update(self.message_values)
message = "".join([
messages['ban']['head'],

@ -32,6 +32,9 @@
[Definition]
# bypass ban/unban for restored tickets
norestored = 1
actionstart =
actionstop =

@ -43,10 +43,15 @@ class ActionReader(DefinitionInitConfigReader):
"actionrepair": ["string", None],
"actionban": ["string", None],
"actionunban": ["string", None],
"norestored": ["string", None],
}
def __init__(self, file_, jailName, initOpts, **kwargs):
self._name = initOpts.get("actname", file_)
actname = initOpts.get("actname")
if actname is None:
actname = file_
initOpts["actname"] = actname
self._name = actname
DefinitionInitConfigReader.__init__(
self, file_, jailName, initOpts, **kwargs)
@ -64,16 +69,22 @@ class ActionReader(DefinitionInitConfigReader):
return self._name
def convert(self):
opts = self.getCombined(ignore=('timeout', 'bantime'))
# type-convert only after combined (otherwise boolean converting prevents substitution):
if opts.get('norestored'):
opts['norestored'] = self._convert_to_boolean(opts['norestored'])
# stream-convert:
head = ["set", self._jailName]
stream = list()
stream.append(head + ["addaction", self._name])
multi = []
for opt, optval in self._opts.iteritems():
for opt, optval in opts.iteritems():
if opt in self._configOpts:
multi.append([opt, optval])
if self._initOpts:
for opt, optval in self._initOpts.iteritems():
multi.append([opt, optval])
if opt not in self._configOpts:
multi.append([opt, optval])
if len(multi) > 1:
stream.append(["multi-set", self._jailName, "action", self._name, multi])
elif len(multi):

@ -30,6 +30,7 @@ from ConfigParser import NoOptionError, NoSectionError
from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
from ..helpers import getLogger
from ..server.action import CommandAction
# Gets the instance of the logger.
logSys = getLogger(__name__)
@ -319,6 +320,28 @@ class DefinitionInitConfigReader(ConfigReader):
self._initOpts['known/'+opt] = v
if not opt in self._initOpts:
self._initOpts[opt] = v
def _convert_to_boolean(self, value):
return value.lower() in ("1", "yes", "true", "on")
def getCombined(self, ignore=()):
combinedopts = self._opts
ignore = set(ignore).copy()
if self._initOpts:
combinedopts = _merge_dicts(self._opts, self._initOpts)
if not len(combinedopts):
return {}
# ignore conditional options:
for n in combinedopts:
cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
if cond:
n, cond = cond.groups()
ignore.add(n)
# substiture options already specified direct:
opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore)
if not opts:
raise ValueError('recursive tag definitions unable to be resolved')
return opts
def convert(self):
raise NotImplementedError

@ -27,8 +27,7 @@ __license__ = "GPL"
import os
import shlex
from .configreader import DefinitionInitConfigReader, _merge_dicts
from ..server.action import CommandAction
from .configreader import DefinitionInitConfigReader
from ..helpers import getLogger
# Gets the instance of the logger.
@ -52,17 +51,6 @@ class FilterReader(DefinitionInitConfigReader):
def getFile(self):
return self.__file
def getCombined(self):
combinedopts = self._opts
if self._initOpts:
combinedopts = _merge_dicts(self._opts, self._initOpts)
if not len(combinedopts):
return {}
opts = CommandAction.substituteRecursiveTags(combinedopts)
if not opts:
raise ValueError('recursive tag definitions unable to be resolved')
return opts
def convert(self):
stream = list()
opts = self.getCombined()
@ -70,6 +58,7 @@ class FilterReader(DefinitionInitConfigReader):
return stream
for opt, value in opts.iteritems():
if opt in ("failregex", "ignoreregex"):
if value is None: continue
multi = []
for regex in value.split('\n'):
# Do not send a command if the rule is empty.
@ -87,6 +76,7 @@ class FilterReader(DefinitionInitConfigReader):
stream.append(["set", self._jailName, "datepattern", value])
# Do not send a command if the match is empty.
elif opt == 'journalmatch':
if value is None: continue
for match in value.split("\n"):
if match == '': continue
stream.append(

@ -136,7 +136,8 @@ class JailReader(ConfigReader):
if not filterName:
raise JailDefError("Invalid filter definition %r" % flt)
self.__filter = FilterReader(
filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
filterName, self.__name, filterOpt,
share_config=self.share_config, basedir=self.getBaseDir())
ret = self.__filter.read()
# merge options from filter as 'known/...':
self.__filter.getOptions(self.__opts)

@ -219,9 +219,9 @@ class CommandAction(ActionBase):
self.timeout = 60
## Command executed in order to initialize the system.
self.actionstart = ''
## Command executed when an IP address gets banned.
## Command executed when ticket gets banned.
self.actionban = ''
## Command executed when an IP address gets removed.
## Command executed when ticket gets removed.
self.actionunban = ''
## Command executed in order to check requirements.
self.actioncheck = ''
@ -365,7 +365,7 @@ class CommandAction(ActionBase):
return self._executeOperation('<actionreload>', 'reloading')
@classmethod
def substituteRecursiveTags(cls, inptags, conditional=''):
def substituteRecursiveTags(cls, inptags, conditional='', ignore=()):
"""Sort out tag definitions within other tags.
Since v.0.9.2 supports embedded interpolation (see test cases for examples).
@ -387,21 +387,26 @@ class CommandAction(ActionBase):
# copy return tags dict to prevent modifying of inptags:
tags = inptags.copy()
t = TAG_CRE
ignore = set(ignore)
done = cls._escapedTags.copy() | ignore
# repeat substitution while embedded-recursive (repFlag is True)
done = cls._escapedTags.copy()
while True:
repFlag = False
# substitute each value:
for tag in tags.iterkeys():
# ignore escaped or already done:
# ignore escaped or already done (or in ignore list):
if tag in done: continue
value = str(tags[tag])
value = orgval = str(tags[tag])
# search and replace all tags within value, that can be interpolated using other tags:
m = t.search(value)
refCounts = {}
#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
while m:
found_tag = m.group(1)
# don't replace tags that should be currently ignored (pre-replacement):
if found_tag in ignore:
m = t.search(value, m.end())
continue
#logSys.log(5, 'found: %s' % found_tag)
if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
# recursive definitions are bad
@ -429,7 +434,7 @@ class CommandAction(ActionBase):
m = t.search(value, m.start())
#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
# was substituted?
if tags[tag] != value:
if orgval != value:
# check still contains any tag - should be repeated (possible embedded-recursive substitution):
if t.search(value):
repFlag = True

@ -348,6 +348,8 @@ class Actions(JailThread, Mapping):
aInfo["failures"] = bTicket.getAttempt()
aInfo["time"] = bTicket.getTime()
aInfo["matches"] = "\n".join(bTicket.getMatches())
# to bypass actions, that should not be executed for restored tickets
aInfo["restored"] = 1 if ticket.restored else 0
if self._jail.database is not None:
mi4ip = lambda overalljails=False, self=self, \
mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
@ -361,6 +363,8 @@ class Actions(JailThread, Mapping):
logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
for name, action in self._actions.iteritems():
try:
if ticket.restored and getattr(action, 'norestored', False):
continue
action.ban(aInfo.copy())
except Exception as e:
logSys.error(
@ -449,10 +453,14 @@ class Actions(JailThread, Mapping):
aInfo["failures"] = ticket.getAttempt()
aInfo["time"] = ticket.getTime()
aInfo["matches"] = "".join(ticket.getMatches())
# to bypass actions, that should not be executed for restored tickets
aInfo["restored"] = 1 if ticket.restored else 0
if actions is None:
logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
for name, action in unbactions.iteritems():
try:
if ticket.restored and getattr(action, 'norestored', False):
continue
logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"])
action.unban(aInfo.copy())
except Exception as e:

@ -94,16 +94,21 @@ class SMTPActionTest(unittest.TestCase):
"Subject: [Fail2Ban] %s: stopped" %
self.jail.name in self.smtpd.data)
def testBan(self):
def _testBan(self, restored=False):
aInfo = {
'ip': "127.0.0.2",
'failures': 3,
'matches': "Test fail 1\n",
'ipjailmatches': "Test fail 1\nTest Fail2\n",
'ipmatches': "Test fail 1\nTest Fail2\nTest Fail3\n",
}
}
if restored:
aInfo['restored'] = 1
self.action.ban(aInfo)
if restored: # no mail, should raises attribute error:
self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
return
self.assertEqual(self.smtpd.mailfrom, "fail2ban")
self.assertEqual(self.smtpd.rcpttos, ["root"])
subject = "Subject: [Fail2Ban] %s: banned %s" % (
@ -123,6 +128,12 @@ class SMTPActionTest(unittest.TestCase):
self.action.matches = "ipmatches"
self.action.ban(aInfo)
self.assertIn(aInfo['ipmatches'], self.smtpd.data)
def testBan(self):
self._testBan()
def testNOPByRestored(self):
self._testBan(restored=True)
def testOptions(self):
self.action.start()

@ -347,7 +347,7 @@ class FilterReaderTest(unittest.TestCase):
['set', 'testcase01', 'addjournalmatch',
"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
['set', 'testcase01', 'maxlines', "1"], # Last for overide test
['set', 'testcase01', 'maxlines', 1], # Last for overide test
]
filterReader = FilterReader("testcase01", "testcase01", {})
filterReader.setBaseDir(TEST_FILES_DIR)
@ -517,12 +517,10 @@ class JailsReaderTest(LogCaptureTestCase):
['add', 'brokenaction', 'auto'],
['set', 'brokenaction', 'addfailregex', '<IP>'],
['set', 'brokenaction', 'addaction', 'brokenaction'],
['set',
'brokenaction',
'action',
'brokenaction',
'actionban',
'hit with big stick <ip>'],
['multi-set', 'brokenaction', 'action', 'brokenaction', [
['actionban', 'hit with big stick <ip>'],
['actname', 'brokenaction']
]],
['add', 'parse_to_end_of_jail.conf', 'auto'],
['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
['start', 'emptyaction'],
@ -548,7 +546,10 @@ class JailsReaderTest(LogCaptureTestCase):
actionName = os.path.basename(actionConfig).replace('.conf', '')
actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR)
self.assertTrue(actionReader.read())
actionReader.getOptions({}) # populate _opts
try:
actionReader.getOptions({}) # populate _opts
except Exception as e: # pragma: no cover
self.fail("action %r\n%s: %s" % (actionName, type(e).__name__, e))
if not actionName.endswith('-common'):
self.assertIn('Definition', actionReader.sections(),
msg="Action file %r is lacking [Definition] section" % actionConfig)

@ -755,12 +755,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
os.remove(fn)
return
_write_file(fn, "w",
"[DEFAULT]",
"_exec_once = 0",
"",
"[Definition]",
"actionstart = echo '[<name>] %s: ** start'" % actname, start,
"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
"actionban = echo '[<name>] %s: ++ ban <ip>'" % actname, ban,
"actionunban = echo '[<name>] %s: -- unban <ip>'" % actname, unban,
"actionstop = echo '[<name>] %s: __ stop'" % actname, stop,
"norestored = %(_exec_once)s",
"restore = ",
"actionstart = echo '[%(name)s] %(actname)s: ** start'", start,
"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
"actionban = echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s'", ban,
"actionunban = echo '[%(name)s] %(actname)s: -- unban <ip>'", unban,
"actionstop = echo '[%(name)s] %(actname)s: __ stop'", stop,
)
if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
_out_file(fn)
@ -777,17 +782,26 @@ class Fail2banServerTest(Fail2banClientServerBase):
"",
"[test-jail1]", "backend = " + backend, "filter =",
"action = ",
" test-action1[name='%(__name__)s']" if 1 in actions else "",
" test-action2[name='%(__name__)s']" if 2 in actions else "",
" test-action1[name='%(__name__)s']" \
if 1 in actions else "",
" test-action2[name='%(__name__)s', restore='restored: <restored>']" \
if 2 in actions else "",
" test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
if 3 in actions else "",
"logpath = " + test1log,
" " + test2log if 2 in enabled else "",
" " + test3log if 2 in enabled else "",
"failregex = ^\s*failure (401|403) from <HOST>",
" ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
" ^\s*error (401|403) from <HOST>" \
if 2 in enabled else "",
"enabled = true" if 1 in enabled else "",
"",
"[test-jail2]", "backend = " + backend, "filter =",
"action =",
"action = ",
" test-action2[name='%(__name__)s', restore='restored: <restored>']" \
if 2 in actions else "",
" test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
if 3 in actions else "",
"logpath = " + test2log,
"enabled = true" if 2 in enabled else "",
)
@ -798,7 +812,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
_write_action_cfg(actname="test-action1")
_write_action_cfg(actname="test-action2")
_write_jail_cfg(enabled=[1], actions=[1,2])
_write_jail_cfg(enabled=[1], actions=[1,2,3])
# append one wrong configured jail:
_write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]",
"", "filter = broken-jail-filter", "enabled = true")
@ -821,6 +835,11 @@ class Fail2banServerTest(Fail2banClientServerBase):
self.assertLogged(
"stdout: '[test-jail1] test-action1: ** start'",
"stdout: '[test-jail1] test-action2: ** start'", all=True)
# test restored is 0 (both actions available):
self.assertLogged(
"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
"stdout: '[test-jail1] test-action3: ++ ban 192.0.2.1 restored: 0'",
all=True, wait=MID_WAITTIME)
# broken jail was logged (in client and server log):
self.assertLogged(
@ -882,10 +901,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
self.assertNotLogged(
"stdout: '[test-jail1] test-action1: -- unban 192.0.2.1'")
# don't need both actions anymore:
# don't need action1 anymore:
_write_action_cfg(actname="test-action1", allow=False)
_write_action_cfg(actname="test-action2", allow=False)
_write_jail_cfg(actions=[])
# leave action2 just to test restored interpolation:
_write_jail_cfg(actions=[2,3])
# write new failures:
self.pruneLog("[test-phase 2b]")
@ -913,7 +932,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
"[test-jail2] Found 192.0.2.2",
"[test-jail2] Ban 192.0.2.2",
"[test-jail2] Found 192.0.2.3",
"[test-jail2] Ban 192.0.2.3", all=True)
"[test-jail2] Ban 192.0.2.3",
all=True)
# rotate logs:
_write_file(test1log, "w+")
@ -936,6 +956,20 @@ class Fail2banServerTest(Fail2banClientServerBase):
"[test-jail2] Restore Ban 192.0.2.4",
"[test-jail2] Restore Ban 192.0.2.8", all=True
)
# test restored is 1 (only test-action2):
self.assertLogged(
"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
all=True, wait=MID_WAITTIME)
# test test-action3 not executed at all (norestored check):
self.assertNotLogged(
"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.4 restored: 1'",
"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'",
all=True)
# don't need actions anymore:
_write_action_cfg(actname="test-action2", allow=False)
_write_jail_cfg(actions=[])
# restart jail with unban all:
self.pruneLog("[test-phase 2d]")

Loading…
Cancel
Save