Merge branch '0.10' into 0.11

# Conflicts:
#	fail2ban/tests/fail2banclienttestcase.py
pull/1976/merge
sebres 7 years ago
commit 5cc0abbb02

@ -53,6 +53,9 @@ ver. 0.11.0-dev-0 (2017/??/??) - development nightly edition
see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944); see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944);
### New Features ### New Features
* New Actions:
- `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in
nginx-location with map-file);
### Enhancements ### Enhancements
* action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once. * action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once.

@ -0,0 +1,108 @@
# Fail2Ban configuration file for black-listing via nginx
#
# Author: Serg G. Brester (aka sebres)
#
# To use 'nginx-block-map' action you should define some special blocks in your nginx configuration,
# and use it hereafter in your locations (to notify fail2ban by failure, resp. nginx by ban).
#
# Example (argument "token_id" resp. cookie "session_id" used here as unique identifier for user):
#
# http {
# ...
# # maps to check user is blacklisted (banned in f2b):
# #map $arg_token_id $blck_lst_tok { include blacklisted-tokens.map; }
# map $cookie_session_id $blck_lst_ses { include blacklisted-sessions.map; }
# ...
# # special log-format to notify fail2ban about failures:
# log_format f2b_session_errors '$msec failure "$cookie_session_id" - $remote_addr - $remote_user '
# ;# '"$request" $status $bytes_sent '
# # '"$http_referer" "$http_user_agent"';
#
# # location checking blacklisted values:
# location ... {
# # check banned sessionid:
# if ($blck_lst_ses != "") {
# try_files "" @f2b-banned;
# }
# ...
# # notify fail2ban about a failure inside nginx:
# error_page 401 = @notify-f2b;
# ...
# }
# ...
# # location for return with "403 Forbidden" if banned:
# location @f2b-banned {
# default_type text/html;
# return 403 "<br/><center>
# <b style=\"color:red; font-size:18pt; border:2pt solid black; padding:5pt;\">
# You are banned!</b></center>";
# }
# ...
# # location to notify fail2ban about a failure inside nginx:
# location @notify-f2b {
# access_log /var/log/nginx/f2b-auth-errors.log f2b_session_errors;
# }
# }
# ...
#
# Note that quote-character (and possibly other special characters) are not allowed currently as session-id.
# Thus please add any session-id validation rule in your locations (or in the corresponding backend-service),
# like in example below:
#
# location ... {
# if ($cookie_session_id !~ "^[\w\-]+$") {
# return 403 "Wrong session-id"
# }
# ...
# }
#
# The parameters for jail corresponding log-format (f2b_session_errors):
#
# [nginx-blck-lst]
# filter =
# datepattern = ^Epoch
# failregex = ^ failure "<F-ID>[^"]+</F-ID>" - <ADDR>
# usedns = no
#
# The same log-file can be used for IP-related jail (additionally to session-related, to ban very bad IPs):
#
# [nginx-blck-ip]
# maxretry = 100
# filter =
# datepattern = ^Epoch
# failregex = ^ failure "[^"]+" - <ADDR>
# usedns = no
#
[Definition]
# path to configuration of nginx (used to target nginx-instance in multi-instance system,
# and as path for the blacklisted map):
srv_cfg_path = /etc/nginx/
# cmd-line arguments to supply to test/reload nginx:
#srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf
srv_cmd = nginx
# first test configuration is correct, hereafter send reload signal:
blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then
%(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi;
fi;
# map-file for nginx, can be redefined using `action = nginx-block-map[blck_lst_file="/path/file.map"]`:
blck_lst_file = %(srv_cfg_path)s/blacklisted-sessions.map
# Action definition:
actionstart_on_demand = false
actionstart = touch '%(blck_lst_file)s'
actionflush = truncate -s 0 '%(blck_lst_file)s'; %(blck_lst_reload)s
actionstop = %(actionflush)s
actioncheck =
actionban = echo "\\\\<fid> 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s
actionunban = id=$(echo "<fid>" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/$id 1;/d" %(blck_lst_file)s; %(blck_lst_reload)s

@ -145,11 +145,12 @@ fail2banserver.PRODUCTION = False
def _out_file(fn, handle=logSys.debug): def _out_file(fn, handle=logSys.debug):
"""Helper which outputs content of the file at HEAVYDEBUG loglevels""" """Helper which outputs content of the file at HEAVYDEBUG loglevels"""
handle('---- ' + fn + ' ----') if (handle != logSys.debug or logSys.getEffectiveLevel() <= logging.DEBUG):
for line in fileinput.input(fn): handle('---- ' + fn + ' ----')
line = line.rstrip('\n') for line in fileinput.input(fn):
handle(line) line = line.rstrip('\n')
handle('-'*30) handle(line)
handle('-'*30)
def _write_file(fn, mode, *lines): def _write_file(fn, mode, *lines):
@ -157,8 +158,19 @@ def _write_file(fn, mode, *lines):
f.write('\n'.join(lines)) f.write('\n'.join(lines))
f.close() f.close()
def _read_file(fn):
f = None
try:
f = open(fn)
return f.read()
finally:
if f is not None:
f.close()
def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
def _start_params(tmp, use_stock=False, use_stock_cfg=None,
logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None
):
cfg = pjoin(tmp, "config") cfg = pjoin(tmp, "config")
if db == 'auto': if db == 'auto':
db = pjoin(tmp, "f2b-db.sqlite3") db = pjoin(tmp, "f2b-db.sqlite3")
@ -168,8 +180,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
"""Filters list of 'files' to contain only directories (under dir)""" """Filters list of 'files' to contain only directories (under dir)"""
return [f for f in files if isdir(pjoin(dir, f))] return [f for f in files if isdir(pjoin(dir, f))]
shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs)
os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "action.d")), pjoin(cfg, "action.d")) use_stock_cfg = ('action.d', 'filter.d')
os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "filter.d")), pjoin(cfg, "filter.d"))
# replace fail2ban params (database with memory): # replace fail2ban params (database with memory):
r = re.compile(r'^dbfile\s*=') r = re.compile(r'^dbfile\s*=')
for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True): for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True):
@ -200,13 +211,21 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
"", "",
) )
_write_file(pjoin(cfg, "jail.conf"), "w", _write_file(pjoin(cfg, "jail.conf"), "w",
"[INCLUDES]", "", *((
"[DEFAULT]", "", "[INCLUDES]", "",
"", "[DEFAULT]", "tmp = " + tmp, "",
)+jails)
) )
if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover
_out_file(pjoin(cfg, "fail2ban.conf")) _out_file(pjoin(cfg, "fail2ban.conf"))
_out_file(pjoin(cfg, "jail.conf")) _out_file(pjoin(cfg, "jail.conf"))
# link stock actions and filters:
if use_stock_cfg and STOCK:
for n in use_stock_cfg:
os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n))
if create_before_start:
for n in create_before_start:
_write_file(n % {'tmp': tmp}, 'w', '')
# parameters (sock/pid and config, increase verbosity, set log, etc.): # parameters (sock/pid and config, increase verbosity, set log, etc.):
vvv, llev = (), "INFO" vvv, llev = (), "INFO"
if unittest.F2B.log_level < logging.INFO: # pragma: no cover if unittest.F2B.log_level < logging.INFO: # pragma: no cover
@ -221,17 +240,13 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"):
) )
def _get_pid_from_file(pidfile): def _get_pid_from_file(pidfile):
f = pid = None pid = None
try: try:
f = open(pidfile) pid = _read_file(pidfile)
pid = f.read()
pid = re.match(r'\S+', pid).group() pid = re.match(r'\S+', pid).group()
return int(pid) return int(pid)
except Exception as e: # pragma: no cover except Exception as e: # pragma: no cover
logSys.debug(e) logSys.debug(e)
finally:
if f is not None:
f.close()
return pid return pid
def _kill_srv(pidfile): def _kill_srv(pidfile):
@ -327,12 +342,16 @@ def with_foreground_server_thread(startextra={}):
finally: finally:
DefLogSys.info('=== within server: end. ===') DefLogSys.info('=== within server: end. ===')
self.pruneLog() self.pruneLog()
# stop: # if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state):
self.execSuccess(startparams, "stop") if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")):
# wait for end: Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME)
Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) # stop (if still running):
self.assertTrue(phase.get('end', None)) if not phase.get('end', None):
self.assertLogged("Shutdown successful", "Exiting Fail2ban") self.execSuccess(startparams, "stop")
# wait for end:
Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME)
self.assertTrue(phase.get('end', None))
self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True)
finally: finally:
if th: if th:
# we start client/server directly in current process (new thread), # we start client/server directly in current process (new thread),
@ -1190,6 +1209,77 @@ class Fail2banServerTest(Fail2banClientServerBase):
"Jail 'test-jail1' stopped", "Jail 'test-jail1' stopped",
"Jail 'test-jail1' started", all=True) "Jail 'test-jail1' started", all=True)
# test action.d/nginx-block-map.conf --
@with_foreground_server_thread(startextra={
# create log-file (avoid "not found" errors):
'create_before_start': ('%(tmp)s/blck-failures.log',),
# we need action.d/nginx-block-map.conf:
'use_stock_cfg': ('action.d',),
# jail-config:
'jails': (
'[nginx-blck-lst]',
'backend = polling',
'usedns = no',
'logpath = %(tmp)s/blck-failures.log',
'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]',
'filter =',
'datepattern = ^Epoch',
'failregex = ^ failure "<F-ID>[^"]+</F-ID>" - <ADDR>',
'maxretry = 1', # ban by first failure
'enabled = true',
)
})
def testServerActions_NginxBlockMap(self, tmp, startparams):
cfg = pjoin(tmp, "config")
lgfn = '%(tmp)s/blck-failures.log' % {'tmp': tmp}
mpfn = '%(tmp)s/blck-lst.map' % {'tmp': tmp}
# ban sessions (write log like nginx does it with f2b_session_errors log-format):
_write_file(lgfn, "w+",
str(int(MyTime.time())) + ' failure "125-000-001" - 192.0.2.1',
str(int(MyTime.time())) + ' failure "125-000-002" - 192.0.2.1',
str(int(MyTime.time())) + ' failure "125-000-003" - 192.0.2.1',
str(int(MyTime.time())) + ' failure "125-000-004" - 192.0.2.1',
str(int(MyTime.time())) + ' failure "125-000-005" - 192.0.2.1',
)
# check all sessions are banned (and blacklisted in map-file):
self.assertLogged(
"[nginx-blck-lst] Ban 125-000-001",
"[nginx-blck-lst] Ban 125-000-002",
"[nginx-blck-lst] Ban 125-000-003",
"[nginx-blck-lst] Ban 125-000-004",
"[nginx-blck-lst] Ban 125-000-005",
"Banned 5 / 5, 5 ticket(s)",
all=True, wait=MID_WAITTIME
)
_out_file(mpfn)
mp = _read_file(mpfn)
self.assertIn('\\125-000-001 1;\n', mp)
self.assertIn('\\125-000-002 1;\n', mp)
self.assertIn('\\125-000-003 1;\n', mp)
self.assertIn('\\125-000-004 1;\n', mp)
self.assertIn('\\125-000-005 1;\n', mp)
# unban 1, 2 and 5:
self.execSuccess(startparams, 'unban', '125-000-001', '125-000-002', '125-000-005')
_out_file(mpfn)
# check really unbanned but other sessions are still present (blacklisted in map-file):
mp = _read_file(mpfn)
self.assertNotIn('\\125-000-001 1;\n', mp)
self.assertNotIn('\\125-000-002 1;\n', mp)
self.assertNotIn('\\125-000-005 1;\n', mp)
self.assertIn('\\125-000-003 1;\n', mp)
self.assertIn('\\125-000-004 1;\n', mp)
# stop server and wait for end:
self.execSuccess(startparams, 'stop')
self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True, wait=MID_WAITTIME)
# check flushed (all sessions were deleted from map-file):
self.assertLogged("[nginx-blck-lst] Flush ticket(s) with nginx-block-map")
_out_file(mpfn)
mp = _read_file(mpfn)
self.assertEqual(mp, '')
@with_foreground_server_thread() @with_foreground_server_thread()
def testServerObserver(self, tmp, startparams): def testServerObserver(self, tmp, startparams):
cfg = pjoin(tmp, "config") cfg = pjoin(tmp, "config")
@ -1292,3 +1382,4 @@ class Fail2banServerTest(Fail2banClientServerBase):
self.assertLogged( self.assertLogged(
"stdout: '[test-jail1] test-action2: ++ prolong 192.0.2.11 -c 2 -t 600 : ", "stdout: '[test-jail1] test-action2: ++ prolong 192.0.2.11 -c 2 -t 600 : ",
all=True, wait=MID_WAITTIME) all=True, wait=MID_WAITTIME)

Loading…
Cancel
Save