From 76f28658835aac152d4f850a7f43930b871e3b4c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Nov 2017 13:42:41 +0100 Subject: [PATCH 1/4] implemented new action "action.d/nginx-block-map.conf", used in order to ban not IP-related tickets via nginx (session blacklisting in nginx-location with map-file); --- config/action.d/nginx-block-map.conf | 108 ++++++++++++++++++ fail2ban/tests/fail2banclienttestcase.py | 133 +++++++++++++++++++---- 2 files changed, 218 insertions(+), 23 deletions(-) create mode 100644 config/action.d/nginx-block-map.conf diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf new file mode 100644 index 00000000..33c15f9c --- /dev/null +++ b/config/action.d/nginx-block-map.conf @@ -0,0 +1,108 @@ +# Fail2Ban configuration file for black-listing via nginx +# +# Author: Serg G. Brester (aka sebres) +# +# To use 'nginx-block-map' action you should define some special blocks in your nginx configuration, +# and use it hereafter in your locations (to notify fail2ban by failure, resp. nginx by ban). +# +# Example (argument "token_id" resp. cookie "session_id" used here as unique identifier for user): +# +# http { +# ... +# # maps to check user is blacklisted (banned in f2b): +# #map $arg_token_id $blck_lst_tok { include blacklisted-tokens.map; } +# map $cookie_session_id $blck_lst_ses { include blacklisted-sessions.map; } +# ... +# # special log-format to notify fail2ban about failures: +# log_format f2b_session_errors '$msec failure "$cookie_session_id" - $remote_addr - $remote_user ' +# ;# '"$request" $status $bytes_sent ' +# # '"$http_referer" "$http_user_agent"'; +# +# # location checking blacklisted values: +# location ... { +# # check banned sessionid: +# if ($blck_lst_ses != "") { +# try_files "" @f2b-banned; +# } +# ... +# # notify fail2ban about a failure inside nginx: +# error_page 401 = @notify-f2b; +# ... +# } +# ... +# # location for return with "403 Forbidden" if banned: +# location @f2b-banned { +# default_type text/html; +# return 403 "
+# +# You are banned!
"; +# } +# ... +# # location to notify fail2ban about a failure inside nginx: +# location @notify-f2b { +# access_log /var/log/nginx/f2b-auth-errors.log f2b_session_errors; +# } +# } +# ... +# +# Note that quote-character (and possibly other special characters) are not allowed currently as session-id. +# Thus please add any session-id validation rule in your locations (or in the corresponding backend-service), +# like in example below: +# +# location ... { +# if ($cookie_session_id !~ "^[\w\-]+$") { +# return 403 "Wrong session-id" +# } +# ... +# } +# +# The parameters for jail corresponding log-format (f2b_session_errors): +# +# [nginx-blck-lst] +# filter = +# datepattern = ^Epoch +# failregex = ^ failure "[^"]+" - +# usedns = no +# +# The same log-file can be used for IP-related jail (additionally to session-related, to ban very bad IPs): +# +# [nginx-blck-ip] +# maxretry = 100 +# filter = +# datepattern = ^Epoch +# failregex = ^ failure "[^"]+" - +# usedns = no +# + +[Definition] + +# path to configuration of nginx (used to target nginx-instance in multi-instance system, +# and as path for the blacklisted map): +srv_cfg_path = /etc/nginx/ + +# cmd-line arguments to supply to test/reload nginx: +#srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf +srv_cmd = nginx + +# first test configuration is correct, hereafter send reload signal: +blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then + %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi; + fi; + +# map-file for nginx, can be redefined using `action = nginx-block-map[blck_lst_file="/path/file.map"]`: +blck_lst_file = %(srv_cfg_path)s/blacklisted-sessions.map + +# Action definition: + +actionstart_on_demand = false +actionstart = touch '%(blck_lst_file)s' + +actionflush = truncate -s 0 '%(blck_lst_file)s'; %(blck_lst_reload)s + +actionstop = %(actionflush)s + +actioncheck = + +actionban = echo "\\\\ 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s + +actionunban = id=$(echo "" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/$id 1;/d" %(blck_lst_file)s; %(blck_lst_reload)s diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 083ce815..befef926 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -115,11 +115,12 @@ fail2banserver.PRODUCTION = False def _out_file(fn, handle=logSys.debug): """Helper which outputs content of the file at HEAVYDEBUG loglevels""" - handle('---- ' + fn + ' ----') - for line in fileinput.input(fn): - line = line.rstrip('\n') - handle(line) - handle('-'*30) + if (handle != logSys.debug or logSys.getEffectiveLevel() <= logging.DEBUG): + handle('---- ' + fn + ' ----') + for line in fileinput.input(fn): + line = line.rstrip('\n') + handle(line) + handle('-'*30) def _write_file(fn, mode, *lines): @@ -127,8 +128,19 @@ def _write_file(fn, mode, *lines): f.write('\n'.join(lines)) f.close() +def _read_file(fn): + f = None + try: + f = open(fn) + return f.read() + finally: + if f is not None: + f.close() + -def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"): +def _start_params(tmp, use_stock=False, use_stock_cfg=None, + logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None +): cfg = pjoin(tmp, "config") if db == 'auto': db = pjoin(tmp, "f2b-db.sqlite3") @@ -138,8 +150,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"): """Filters list of 'files' to contain only directories (under dir)""" return [f for f in files if isdir(pjoin(dir, f))] shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) - os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "action.d")), pjoin(cfg, "action.d")) - os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, "filter.d")), pjoin(cfg, "filter.d")) + use_stock_cfg = ('action.d', 'filter.d') # replace fail2ban params (database with memory): r = re.compile(r'^dbfile\s*=') for line in fileinput.input(pjoin(cfg, "fail2ban.conf"), inplace=True): @@ -170,13 +181,21 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"): "", ) _write_file(pjoin(cfg, "jail.conf"), "w", - "[INCLUDES]", "", - "[DEFAULT]", "", - "", + *(( + "[INCLUDES]", "", + "[DEFAULT]", "tmp = " + tmp, "", + )+jails) ) if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(pjoin(cfg, "fail2ban.conf")) _out_file(pjoin(cfg, "jail.conf")) + # link stock actions and filters: + if use_stock_cfg and STOCK: + for n in use_stock_cfg: + os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n)) + if create_before_start: + for n in create_before_start: + _write_file(n % {'tmp': tmp}, 'w', '') # parameters (sock/pid and config, increase verbosity, set log, etc.): vvv, llev = (), "INFO" if unittest.F2B.log_level < logging.INFO: # pragma: no cover @@ -191,17 +210,13 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null", db=":memory:"): ) def _get_pid_from_file(pidfile): - f = pid = None + pid = None try: - f = open(pidfile) - pid = f.read() + pid = _read_file(pidfile) pid = re.match(r'\S+', pid).group() return int(pid) except Exception as e: # pragma: no cover logSys.debug(e) - finally: - if f is not None: - f.close() return pid def _kill_srv(pidfile): @@ -297,12 +312,13 @@ def with_foreground_server_thread(startextra={}): finally: DefLogSys.info('=== within server: end. ===') self.pruneLog() - # stop: - self.execSuccess(startparams, "stop") - # wait for end: - Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('end', None)) - self.assertLogged("Shutdown successful", "Exiting Fail2ban") + # stop (if still running): + if not phase.get('end', None): + self.execSuccess(startparams, "stop") + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True) finally: if th: # we start client/server directly in current process (new thread), @@ -1155,3 +1171,74 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", all=True) + + # test action.d/nginx-block-map.conf -- + @with_foreground_server_thread(startextra={ + # create log-file (avoid "not found" errors): + 'create_before_start': ('%(tmp)s/blck-failures.log',), + # we need action.d/nginx-block-map.conf: + 'use_stock_cfg': ('action.d',), + # jail-config: + 'jails': ( + '[nginx-blck-lst]', + 'backend = polling', + 'usedns = no', + 'logpath = %(tmp)s/blck-failures.log', + 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + 'filter =', + 'datepattern = ^Epoch', + 'failregex = ^ failure "[^"]+" - ', + 'maxretry = 1', # ban by first failure + 'enabled = true', + ) + }) + def testServerActions_NginxBlockMap(self, tmp, startparams): + cfg = pjoin(tmp, "config") + lgfn = '%(tmp)s/blck-failures.log' % {'tmp': tmp} + mpfn = '%(tmp)s/blck-lst.map' % {'tmp': tmp} + # ban sessions (write log like nginx does it with f2b_session_errors log-format): + _write_file(lgfn, "w+", + str(int(MyTime.time())) + ' failure "125-000-001" - 192.0.2.1', + str(int(MyTime.time())) + ' failure "125-000-002" - 192.0.2.1', + str(int(MyTime.time())) + ' failure "125-000-003" - 192.0.2.1', + str(int(MyTime.time())) + ' failure "125-000-004" - 192.0.2.1', + str(int(MyTime.time())) + ' failure "125-000-005" - 192.0.2.1', + ) + # check all sessions are banned (and blacklisted in map-file): + self.assertLogged( + "[nginx-blck-lst] Ban 125-000-001", + "[nginx-blck-lst] Ban 125-000-002", + "[nginx-blck-lst] Ban 125-000-003", + "[nginx-blck-lst] Ban 125-000-004", + "[nginx-blck-lst] Ban 125-000-005", + "Banned 5 / 5, 5 ticket(s)", + all=True, wait=MID_WAITTIME + ) + _out_file(mpfn) + mp = _read_file(mpfn) + self.assertIn('\\125-000-001 1;\n', mp) + self.assertIn('\\125-000-002 1;\n', mp) + self.assertIn('\\125-000-003 1;\n', mp) + self.assertIn('\\125-000-004 1;\n', mp) + self.assertIn('\\125-000-005 1;\n', mp) + + # unban 1, 2 and 5: + self.execSuccess(startparams, 'unban', '125-000-001', '125-000-002', '125-000-005') + _out_file(mpfn) + # check really unbanned but other sessions are still present (blacklisted in map-file): + mp = _read_file(mpfn) + self.assertNotIn('\\125-000-001 1;\n', mp) + self.assertNotIn('\\125-000-002 1;\n', mp) + self.assertNotIn('\\125-000-005 1;\n', mp) + self.assertIn('\\125-000-003 1;\n', mp) + self.assertIn('\\125-000-004 1;\n', mp) + + # stop server and wait for end: + self.execSuccess(startparams, 'stop') + self.assertLogged("Shutdown successful", "Exiting Fail2ban", all=True, wait=MID_WAITTIME) + + # check flushed (all sessions were deleted from map-file): + self.assertLogged("[nginx-blck-lst] Flush ticket(s) with nginx-block-map") + _out_file(mpfn) + mp = _read_file(mpfn) + self.assertEqual(mp, '') From b62ab2d51e7fd27584dbbae3dae38139fc6861c1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Nov 2017 13:46:57 +0100 Subject: [PATCH 2/4] ChangeLog updated --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9abafd79..4a4eee9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -53,6 +53,9 @@ ver. 0.10.2-dev-1 (2017/??/??) - development edition see "ssherr.c" for all possible SSH_ERR_..._ALG_MATCH errors (gh-1943, gh-1944); ### New Features +* New Actions: + - `action.d/nginx-block-map.conf` - in order to ban not IP-related tickets via nginx (session blacklisting in + nginx-location with map-file); ### Enhancements * action.d/pf.conf: extended with bulk-unban, command `actionflush` in order to flush all bans at once. From 55c2a9968a7a4afab43de393f3ce294fe92135ac Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Nov 2017 16:13:37 +0100 Subject: [PATCH 3/4] remove lacking [Init] section check ([Init] section not necessary anymore for actions also); fix sporadic error by shutdown server in with_foreground_server_thread decorator (if shutdown too fast, but end-phase still does not reached the tester-thread); --- fail2ban/tests/clientreadertestcase.py | 2 -- fail2ban/tests/fail2banclienttestcase.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 4b2ec0e5..fdef3735 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -623,8 +623,6 @@ class JailsReaderTest(LogCaptureTestCase): #print('****', actionName, opts.get('actionstart', '')) self.assertIn('f2b-TEST', opts.get('actionstart', ''), msg="Action file %r: interpolation of actionstart does not contains jail-name 'f2b-TEST'" % actionConfig) - self.assertIn('Init', actionReader.sections(), - msg="Action file %r is lacking [Init] section" % actionConfig) def testReadStockJailConf(self): jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index befef926..78a09e81 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -312,6 +312,9 @@ def with_foreground_server_thread(startextra={}): finally: DefLogSys.info('=== within server: end. ===') self.pruneLog() + # if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state): + if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")): + Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME) # stop (if still running): if not phase.get('end', None): self.execSuccess(startparams, "stop") From fbf89e8cdd1229e7207ecb3ec522761423d31e5f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 28 Nov 2017 16:32:16 +0100 Subject: [PATCH 4/4] typo in indent (spaces to tabs) --- fail2ban/tests/fail2banclienttestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 78a09e81..11b67fb2 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -1185,7 +1185,7 @@ class Fail2banServerTest(Fail2banClientServerBase): 'jails': ( '[nginx-blck-lst]', 'backend = polling', - 'usedns = no', + 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', 'filter =',