From 55e107310fd0697d583aa8971e2b1db48e49c5ea Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sat, 7 Jan 2017 14:24:54 +0200 Subject: [PATCH 01/20] Added config for AbuseIPDB, ony tested on Ubuntu 16.04 --- THANKS | 2 + config/action.d/abuseipdb.conf | 74 ++++++++++++++++++++++++++++++++++ config/jail.conf | 21 +++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 config/action.d/abuseipdb.conf diff --git a/THANKS b/THANKS index 8f746f29..7537cb92 100644 --- a/THANKS +++ b/THANKS @@ -16,6 +16,7 @@ Alexander Koeppe (IPv6 support) Alexandre Perrin (kAworu) Amir Caspi Amy +Andrew James Collett (ajcollett) Andrew St. Jean Andrey G. Grozin Andy Fragen @@ -111,6 +112,7 @@ Sean DuBois Sebastian Arcus Serg G. Brester Sergey Safarov +Shaun C. Sireyessire silviogarbes Stefan Tatschner diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf new file mode 100644 index 00000000..7bd3f68a --- /dev/null +++ b/config/action.d/abuseipdb.conf @@ -0,0 +1,74 @@ +# Fail2ban configuration file +# +# Action to report IP address to abuseipdb.com +# You must sign up to obtain an API key from abuseipdb.com. +# +# IMPORTANT: +# +# Reporting an IP of abuse is a serious complaint. Make sure that it is +# serious. Fail2ban developers and network owners recommend you only use this +# action for: +# * The recidive where the IP has been banned multiple times +# * Where maxretry has been set quite high, beyond the normal user typing +# password incorrectly. +# * For filters that have a low likelihood of receiving human errors +# +# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# Added to fail2ban by Andrew James Collett (ajcollett) + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# +# ** IMPORTANT! ** +# +# By default, this posts directly to AbuseIPDB's API, unfortunately +# this results in a lot of backslashes/escapes appearing in the +# reports. This also may include info like your hostname. +# If you have your own web server with PHP available, you can +# use my (Shaun's) helper PHP script by commenting out the first #actionban +# line below, uncommenting the second one, and pointing the URL at +# wherever you install the helper script. For the PHP helper script, see +# +# +# --ciphers ecdhe_ecdsa_aes_256_sha is used to workaround a +# "NSS error -12286" from curl as it attempts to connect using +# SSLv3. See https://www.centos.org/forums/viewtopic.php?t=52732 +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = curl --fail --ciphers ecdhe_ecdsa_aes_256_sha --data 'key=' --data-urlencode 'comment=' --data 'ip=' --data 'category=' "https://www.abuseipdb.com/report/json" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] +# Option: abuseipdb_apikey +# Notes Your API key from abuseipdb.com +# Values: STRING Default: None +abuseipdb_apikey = diff --git a/config/jail.conf b/config/jail.conf index b7c927e2..f5d24621 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API -# +# # See the IMPORTANT note in action.d/blocklist_de.conf for when to # use this action. Create a file jail.d/blocklist_de.local containing # [Init] @@ -206,6 +206,23 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag # Report ban via badips.com (uses action.d/badips.conf for reporting only) # action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +# Actions to report to abuseipdb.com via API. +# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# If you want cleaner reports that ensure no user data see the helper script at the above IP +# NOTE: These reports may include sensitive Info. + +# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf +action_abuseipdb_fraud = abuseipdb[abuseipdb_category="3"] +action_abuseipdb_ddos = abuseipdb[abuseipdb_category="4"] +action_abuseipdb_proxy = abuseipdb[abuseipdb_category="9"] +action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"] +action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"] +action_abuseipdb_blogspam = abuseipdb[abuseipdb_category="12"] +action_abuseipdb_portscan = abuseipdb[abuseipdb_category="14"] +action_abuseipdb_hack = abuseipdb[abuseipdb_category="15"] +action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"] +action_abuseipdb_spoofing = abuseipdb[abuseipdb_category="17"] +action_abuseipdb_sshbrute = abuseipdb[abuseipdb_category="18"] # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -330,7 +347,7 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s -# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` # and define `limit_req` and `limit_req_zone` as described in nginx documentation # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html # or for example see in 'config/filter.d/nginx-limit-req.conf' From 1c41390f7caed9a9e6712c83d80774bf7bf1c14f Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:26:11 +0200 Subject: [PATCH 02/20] Restructured the way the catagories work. Jail.conf is cleaner and abuseipdb.conf is more flexible. --- config/action.d/abuseipdb.conf | 19 +++++++++++++++++++ config/jail.conf | 28 ++++++++++++---------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index 7bd3f68a..3bcfb065 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -16,6 +16,23 @@ # Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) +## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] +# ID Title Description +# 3 Fraud Orders +# 4 DDoS Attack +# 9 Open Proxy +# 10 Web Spam +# 11 Email Spam +# 14 Port Scan +# 18 Brute-Force +# 19 Bad Web Bot +# 20 Exploited Host +# 21 Web App Attack +# 22 SSH Secure Shell (SSH) abuse. Use this category in combination with more specific categories. +# 23 IoT Targeted +# See https://abuseipdb.com/categories for more descriptions + [Definition] # Option: actionstart @@ -71,4 +88,6 @@ actionunban = # Option: abuseipdb_apikey # Notes Your API key from abuseipdb.com # Values: STRING Default: None +# Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. +# You will need to set the catagory in the action call. abuseipdb_apikey = diff --git a/config/jail.conf b/config/jail.conf index f5d24621..7ba343fb 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -206,23 +206,19 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag # Report ban via badips.com (uses action.d/badips.conf for reporting only) # action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] -# Actions to report to abuseipdb.com via API. -# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban -# If you want cleaner reports that ensure no user data see the helper script at the above IP +# Report ban via abuseipdb.com. +# +# See action.d/abuseipdb.conf and +# https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# # NOTE: These reports may include sensitive Info. - -# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf -action_abuseipdb_fraud = abuseipdb[abuseipdb_category="3"] -action_abuseipdb_ddos = abuseipdb[abuseipdb_category="4"] -action_abuseipdb_proxy = abuseipdb[abuseipdb_category="9"] -action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"] -action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"] -action_abuseipdb_blogspam = abuseipdb[abuseipdb_category="12"] -action_abuseipdb_portscan = abuseipdb[abuseipdb_category="14"] -action_abuseipdb_hack = abuseipdb[abuseipdb_category="15"] -action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"] -action_abuseipdb_spoofing = abuseipdb[abuseipdb_category="17"] -action_abuseipdb_sshbrute = abuseipdb[abuseipdb_category="18"] +# If you want cleaner reports that ensure no user data see the helper script at the above website. +# +# IMPORTANT: This action relies on a api_key being added to the above action conf, +# and the appropriate catagories set. +# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] +# See action.d/abuseipdb.conf for catagories +action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local From b35391e768c4a476cc176959bad1571ccfd40708 Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:30:00 +0200 Subject: [PATCH 03/20] Update jail.conf Fixing spacing --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7ba343fb..09675904 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API -# +# # See the IMPORTANT note in action.d/blocklist_de.conf for when to # use this action. Create a file jail.d/blocklist_de.local containing # [Init] @@ -330,7 +330,7 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 - + [openhab-auth] filter = openhab From 10d61e077942c25ac5a026a6c6da2b3d2bb6ee3f Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:39:12 +0200 Subject: [PATCH 04/20] Fixed the spaces again --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 09675904..31f0c2be 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -330,7 +330,7 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 - + [openhab-auth] filter = openhab From 3991f51f30f10a1086f0999089aa25c8fedb653d Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:45:35 +0200 Subject: [PATCH 05/20] Update jail.conf Sigh, added a space back that I somehow missed in Vim, despite it being a rebase... --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 31f0c2be..560136a6 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -343,7 +343,7 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s -# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` # and define `limit_req` and `limit_req_zone` as described in nginx documentation # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html # or for example see in 'config/filter.d/nginx-limit-req.conf' From 18d09b6d8ef6c72b83fd99f81832ffe2e7c2ce8d Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:50:58 +0200 Subject: [PATCH 06/20] Updated changelog. --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7964a9ba..5f5496c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -83,6 +83,10 @@ TODO: implementing of options resp. other tasks from PR #1346 if configuration is clean (fails by wrong configured jails if option `-t` specified) * New command action parameter `actionrepair` - command executed in order to restore sane environment in error case of `actioncheck`. +* Reporting via abuseipdb.com: + - Bans can now be reported to abuseipdb + - Catagories must be set in the config + - Relevant log lines included in report ### Enhancements * Huge increasing of fail2ban performance and especially test-cases performance (see gh-1109) From a0bb51ef92864ec504903d75e814c14f242204cf Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Mar 2017 16:34:03 +0100 Subject: [PATCH 07/20] New tag '' introduced: can be used in actions to retrieve the host name (dns) from the IP address --- fail2ban/server/actions.py | 1 + fail2ban/server/ipdns.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 6b793b8f..e0719cde 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -291,6 +291,7 @@ class Actions(JailThread, Mapping): AI_DICT = { "ip": lambda self: self.__ticket.getIP(), "ip-rev": lambda self: self['ip'].getPTR(''), + "ip-host": lambda self: self['ip'].getHost(), "fid": lambda self: self.__ticket.getID(), "failures": lambda self: self.__ticket.getAttempt(), "time": lambda self: self.__ticket.getTime(), diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 757cceba..8990618a 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -376,6 +376,11 @@ class IPAddr(object): return "%s.%s" % (".".join(reversed(exploded_ip)), suffix) + def getHost(self): + """Return the host name (DNS) of the provided IP address object + """ + return DNSUtils.ipToName(self.ntoa) + @property def isIPv4(self): """Either the IP object is of address family AF_INET From 59cf7611298b570cea657a9b1f30741d910cf5a6 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Mar 2017 16:50:21 +0100 Subject: [PATCH 08/20] Real action info instead of calling map in test cases, covering of the new tag ''; dns lookup: pre-caching within test cases - prevent slow dns-resolving and failures if no-network, of if some IP addresses will be changed later --- fail2ban/tests/servertestcase.py | 14 +++++++++----- fail2ban/tests/utils.py | 3 +++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 464295fd..461c6b2f 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -38,7 +38,9 @@ from ..server.server import Server from ..server.ipdns import IPAddr from ..server.jail import Jail from ..server.jailthread import JailThread +from ..server.ticket import BanTicket from ..server.utils import Utils +from .dummyjail import DummyJail from .utils import LogCaptureTestCase from ..helpers import getLogger, PREFER_ENC from .. import version @@ -1677,7 +1679,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): # complain -- ('j-complain-abuse', 'complain[' - 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s",' + + 'name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd="mail -s Hostname: - ",' + # test reverse ip: 'debug=1,' + # 2 logs to test grep from multiple logs: @@ -1692,14 +1694,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', # both abuse mails should be separated with space: - 'mail -s Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', + 'mail -s Hostname: test-host - Abuse from 87.142.124.10 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), 'ip6-ban': ( # test reverse ip: 'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org', 'Lines containing failures of 2001:db8::1 (max 2)', # both abuse mails should be separated with space: - 'mail -s Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', + 'mail -s Hostname: test-host - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), }), ) @@ -1723,6 +1725,7 @@ class ServerConfigReaderTests(LogCaptureTestCase): ipv4 = IPAddr('87.142.124.10') ipv6 = IPAddr('2001:db8::1'); + dmyjail = DummyJail() for jail, act, tests in testJailsActions: # print(jail, jails[jail]) for a in jails[jail].actions: @@ -1736,7 +1739,8 @@ class ServerConfigReaderTests(LogCaptureTestCase): for (test, ip) in (('ip4-ban', ipv4), ('ip6-ban', ipv6)): if not tests.get(test): continue self.pruneLog('# === %s ===' % test) - ticket = _actions.CallingMap({ - 'ip': ip, 'ip-rev': lambda self: self['ip'].getPTR(''), 'failures': 100,}) + ticket = BanTicket(ip) + ticket.setAttempt(100) + ticket = _actions.Actions.ActionInfo(ticket, dmyjail) action.ban(ticket) self.assertLogged(*tests[test], all=True) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 78a42d09..7fba73c9 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -273,6 +273,9 @@ def initTests(opts): c.set('192.0.2.%s' % i, None) c.set('198.51.100.%s' % i, None) c.set('203.0.113.%s' % i, None) + c.set('2001:db8::%s' %i, 'test-host') + # some legal ips used in our test cases (prevent slow dns-resolving and failures if will be changed later): + c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover # precache all wrong dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp From 6a2c95da9542c6ad2f26eda6b1d3fd131524ec0c Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Mar 2017 16:45:04 +0100 Subject: [PATCH 09/20] `action.d/sendmail-geoip-lines.conf` fixed using new tag `` (dns-cache and without external command execution); changelog updated; --- ChangeLog | 8 +++++++- config/action.d/sendmail-geoip-lines.conf | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 83d61a55..d2284473 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,8 @@ TODO: implementing of options resp. other tasks from PR #1346 - [grave] injection on user name to host fixed * `action.d/complain.conf` - fixed using new tag `` (sh/dash compliant now) +* `action.d/sendmail-geoip-lines.conf` + - fixed using new tag `` (without external command execution) ### New Features * New Actions: @@ -44,7 +46,11 @@ TODO: implementing of options resp. other tasks from PR #1346 to re.sub with callable) * substituteRecursiveTags optimization + moved in helpers facilities (because currently used commonly in server and in client) -* Provides new tag `` for PTR reversed representation of IP address +* New tags (usable in actions): + - `` - failure identifier (if raw resp. failures without IP address) + - `` - PTR reversed representation of IP address + - `` - host name of the IP address + - `` - interpolates to the corresponding filter group capture `...` ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index 34c3aedd..decf2c05 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -36,7 +36,7 @@ actionban = ( printf %%b "Subject: [Fail2Ban] : banned from `uname -n http://whois.domaintools.com/\n\n Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "" | cut -d':' -f2-` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "" | cut -d':' -f2-` - hostname: `host -t A 2>&1`\n\n + hostname: \n\n Lines containing failures of \n"; %(_grep_logs)s; printf %%b "\n From 62fa02241f3c2c159d2d17ad8f1bf8ab96881d3a Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 9 Mar 2017 13:31:40 +0100 Subject: [PATCH 10/20] Update jail.conf --- config/jail.conf | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 560136a6..75a824e2 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -206,18 +206,11 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag # Report ban via badips.com (uses action.d/badips.conf for reporting only) # action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] + # Report ban via abuseipdb.com. # -# See action.d/abuseipdb.conf and -# https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# See action.d/abuseipdb.conf for usage example and details. # -# NOTE: These reports may include sensitive Info. -# If you want cleaner reports that ensure no user data see the helper script at the above website. -# -# IMPORTANT: This action relies on a api_key being added to the above action conf, -# and the appropriate catagories set. -# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] -# See action.d/abuseipdb.conf for catagories action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the From b1f5ac948431cfac4089c18df016f6cd02976fa9 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 9 Mar 2017 13:33:11 +0100 Subject: [PATCH 11/20] Update abuseipdb.conf --- config/action.d/abuseipdb.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index 3bcfb065..15e41fbe 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -3,6 +3,9 @@ # Action to report IP address to abuseipdb.com # You must sign up to obtain an API key from abuseipdb.com. # +# NOTE: These reports may include sensitive Info. +# If you want cleaner reports that ensure no user data see the helper script at the below website. +# # IMPORTANT: # # Reporting an IP of abuse is a serious complaint. Make sure that it is @@ -13,6 +16,15 @@ # password incorrectly. # * For filters that have a low likelihood of receiving human errors # +# This action relies on a api_key being added to the above action conf, +# and the appropriate categories set. +# +# Example, for ssh bruteforce (in section [sshd] of `jail.local`): +# action = %(known/action)s +# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# +# See below for catagories. +# # Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) From 8768776d68d4271bb166af36870d12eab2bbca9c Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Mar 2017 16:13:45 +0100 Subject: [PATCH 12/20] filter.d/cyrus-imap.conf: fixed `failregex` - accept entries without login-info resp. hostname before IP address --- config/filter.d/cyrus-imap.conf | 2 +- fail2ban/tests/files/logs/cyrus-imap | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/cyrus-imap.conf b/config/filter.d/cyrus-imap.conf index 73764d9d..31dfda60 100644 --- a/config/filter.d/cyrus-imap.conf +++ b/config/filter.d/cyrus-imap.conf @@ -13,7 +13,7 @@ before = common.conf _daemon = (?:cyrus/)?(?:imap(d|s)?|pop3(d|s)?) -failregex = ^%(__prefix_line)sbadlogin: \S+ ?\[\] \S+ .*?\[?SASL\(-13\): (authentication failure|user not found): .*\]?$ +failregex = ^%(__prefix_line)sbadlogin: [^\[]*\[\] \S+ .*?\[?SASL\(-13\): (authentication failure|user not found): .*\]?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/cyrus-imap b/fail2ban/tests/files/logs/cyrus-imap index f1edff06..c0ec54cb 100644 --- a/fail2ban/tests/files/logs/cyrus-imap +++ b/fail2ban/tests/files/logs/cyrus-imap @@ -16,3 +16,6 @@ Dec 30 16:03:27 somehost imapd[2517]: badlogin: local-somehost[1.2.3.4] OTP [SAS Jul 17 22:55:56 derry cyrus/imaps[7568]: badlogin: serafinat.xxxxxx [1.2.3.4] plain [SASL(-13): user not found: user: pressy@derry property: cmusaslsecretPLAIN not found in sasldb] # failJSON: { "time": "2005-07-18T16:46:42", "match": true , "host": "1.2.3.4" } Jul 18 16:46:42 derry cyrus/imaps[27449]: badlogin: serafinat.xxxxxx [1.2.3.4] PLAIN [SASL(-13): user not found: Password verification failed] + +# failJSON: { "time": "2005-03-08T05:25:21", "match": true , "host": "192.0.2.4", "desc": "entry without loginname/hostname before IP" } +Mar 8 05:25:21 host imap[22130]: badlogin: [192.0.2.4] plain [SASL(-13): authentication failure: Password verification failed] \ No newline at end of file From 0f8cb1749fdfe4e6a81e07a9cead1beefcd0de5b Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 9 Mar 2017 16:15:45 +0100 Subject: [PATCH 13/20] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2a7e6638..e6114caa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,8 @@ releases. and suffix (logged from several ssh versions), according to gh-1206; * filter.d/suhosin.conf - greedy catch-all before `` fixed (potential vulnerability) +* filter.d/cyrus-imap.conf + - accept entries without login-info resp. hostname before IP address (gh-1707) * Filter tests extended with check of all config-regexp, that contains greedy catch-all before ``, that is hard-anchored at end or precise sub expression after `` From a683e88a74d1b7fb6e7235d86674b30bacab2c4e Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Mar 2017 20:39:09 +0100 Subject: [PATCH 14/20] samples test case factory extended with filter options - dict in JSON to control filter options (e. g. mode, etc.): # filterOptions: {"mode": "aggressive"} --- fail2ban/tests/samplestestcase.py | 50 +++++++++++++++++++------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index b59ccd98..96e0b21c 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -49,12 +49,7 @@ class FilterSamplesRegex(unittest.TestCase): def setUp(self): """Call before every test case.""" super(FilterSamplesRegex, self).setUp() - self.filter = Filter(None) - self.filter.returnRawHost = True - self.filter.checkAllRegex = True - self.filter.checkFindTime = False - self.filter.active = True - + self.filter = None setUpMyTime() def tearDown(self): @@ -83,11 +78,15 @@ class FilterSamplesRegex(unittest.TestCase): RE_WRONG_GREED.search('non-greedy .+? test' + RE_HOST + ' test vary catch-all .* anchored$')) -def testSampleRegexsFactory(name, basedir): - def testFilter(self): - + def _readFilter(self, name, basedir, opts=None): + self.filter = Filter(None) + self.filter.returnRawHost = True + self.filter.checkAllRegex = True + self.filter.checkFindTime = False + self.filter.active = True + if opts is None: opts = dict() # Check filter exists - filterConf = FilterReader(name, "jail", {}, + filterConf = FilterReader(name, "jail", opts, basedir=basedir, share_config=unittest.F2B.share_config) self.assertEqual(filterConf.getFile(), name) self.assertEqual(filterConf.getJailName(), "jail") @@ -113,6 +112,17 @@ def testSampleRegexsFactory(name, basedir): elif opt[2] == "datepattern": self.filter.setDatePattern(optval) + # test regexp contains greedy catch-all before , that is + # not hard-anchored at end or has not precise sub expression after : + for fr in self.filter.getFailRegex(): + if RE_WRONG_GREED.search(fr): # pragma: no cover + raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before , " + "that is not hard-anchored at end or has not precise sub expression after :\n%s" % + (name, str(fr).replace(RE_HOST, ''))) + +def testSampleRegexsFactory(name, basedir): + def testFilter(self): + self.assertTrue( os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)), "No sample log file available for '%s' filter" % name) @@ -125,22 +135,21 @@ def testSampleRegexsFactory(name, basedir): logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", filename)) - # test regexp contains greedy catch-all before , that is - # not hard-anchored at end or has not precise sub expression after : - for fr in self.filter.getFailRegex(): - if RE_WRONG_GREED.search(fr): # pragma: no cover - raise AssertionError("Following regexp of \"%s\" contains greedy catch-all before , " - "that is not hard-anchored at end or has not precise sub expression after :\n%s" % - (name, str(fr).replace(RE_HOST, ''))) - for line in logFile: - jsonREMatch = re.match("^# ?(failJSON|addFILE):(.+)$", line) + jsonREMatch = re.match("^#+ ?(failJSON|filterOptions|addFILE):(.+)$", line) if jsonREMatch: try: faildata = json.loads(jsonREMatch.group(2)) + # filterOptions - dict in JSON to control filter options (e. g. mode, etc.): + if jsonREMatch.group(1) == 'filterOptions': + self.filter = None + self._readFilter(name, basedir, opts=faildata) + continue + # addFILE - filename to "include" test-files should be additionally parsed: if jsonREMatch.group(1) == 'addFILE': filenames.append(faildata) continue + # failJSON - faildata contains info of the failure to check it. except ValueError as e: raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) @@ -150,6 +159,9 @@ def testSampleRegexsFactory(name, basedir): else: faildata = {} + if self.filter is None: + self._readFilter(name, basedir, opts=None) + try: ret = self.filter.processLine(line) if not ret: From 7e442c5b2744fed7771d5cec440e91d85072048d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Mar 2017 20:43:53 +0100 Subject: [PATCH 15/20] filter.d/sendmail-reject.conf: - rewritten using `prefregex` and used MLFID-related multi-line parsing (by using tag `` instead of buffering with `maxlines`); - optional parameter `mode` introduced: normal (default), extra or aggressive (see sendmail-reject for regex details); test cases extended --- config/filter.d/sendmail-reject.conf | 47 +++++++++++++++-------- fail2ban/tests/files/logs/sendmail-reject | 27 ++++++++----- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index 2f8fd882..0793a99b 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,30 +21,45 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) -failregex = ^%(__prefix_line)s\w{14}: ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[\]( \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^%(__prefix_line)sruleset=check_relay, arg1=(?P\S+), arg2=, relay=((?P=dom) )?\[(\d+\.){3}\d+\]( \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^%(__prefix_line)s\w{14}: rejecting commands from (\S* )?\[\] due to pre-greeting traffic after \d+ seconds$ - ^%(__prefix_line)s\w{14}: (\S+ )?\[\]: ((?i)expn|vrfy) \S+ \[rejected\]$ - ^(?P<__prefix>%(__prefix_line)s\w+: )<[^@]+@[^>]+>\.\.\. No such user here$^(?P=__prefix)from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[\]$ +prefregex = ^%(__prefix_line)s(?:\w{14}: )?.+$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay, arg1=(?P\S+), arg2=, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ + ^<[^@]+@[^>]+>\.\.\. No such user here$ + ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[\]$ -ignoreregex = +mdre-normal = +mdre-extra = ^(?:\S+ )?\[\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to M(?:TA|SP)(?:-\w+)?$ -[Init] +mdre-aggressive = %(mdre-extra)s + +failregex = %(cmnfailre)s + > + +# Parameter "mode": normal (default), extra or aggressive +# Usage example (for jail.local): +# [sendmail-reject] +# filter = sendmail-reject[mode=extra] +# +mode = normal + +ignoreregex = -# "maxlines" is number of log lines to buffer for multi-line regex searches -maxlines = 10 # DEV NOTES: # -# Regarding the last multiline regex: +# Regarding the multiline regex: # -# There can be a nunber of non-related lines between the first and second part -# of this regex maxlines of 10 is quite generious. Only one of the -# "No such user" lines needs to be matched before the line with the HOST. +# "No such user" lines generate a failure and needs to be matched together with +# another line with the HOST, therefore no-failure line was added as regex, that +# contains HOST (see line with tag ). # -# Note the capture __prefix, includes both the __prefix_lines (which includes -# the sendmail PID), but also the \w+ which the the sendmail assigned mail ID. +# Note the capture , includes both the __prefix_lines (which includes +# the sendmail PID), but also the `\w{14}` which the the sendmail assigned +# mail ID (todo: check this is necessary, possible obsolete). # -# Author: Daniel Black and Fabian Wenk +# Author: Daniel Black, Fabian Wenk and Sergey Brester aka sebres. +# Rewritten using prefregex by Serg G. Brester. diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index 70d4dde6..44f8eb92 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -1,3 +1,5 @@ +# normal mode # filterOptions: {"mode": "normal"} + # failJSON: { "time": "2005-02-25T03:01:10", "match": true , "host": "128.68.136.133" } Feb 25 03:01:10 kismet sm-acceptingconnections[27713]: s1P819mk027713: ruleset=check_rcpt, arg1=, relay=128-68-136-133.broadband.corbina.ru [128.68.136.133], reject=550 5.7.1 ... Relaying denied. Proper authentication required. @@ -69,20 +71,27 @@ Feb 22 14:02:44 batman sm-mta[4030]: s1MD2hsd004030: rrcs-24-73-201-194.se.biz.r # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026250: ... No such user here # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026251: ... No such user here # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: ... No such user here # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: ... No such user here + # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here # failJSON: { "time": "2004-11-03T11:35:30", "match": true , "host": "95.32.23.163" } Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: from=, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163] # failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026254: ... No such user here -# Different mail ID shouldn't match -# failJSON: { "match": false } -Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026255: from=, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163] +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026252: ... No such user here +# failJSON: { "match": false, "desc": "Different mail ID shouldn't match" } +Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026255: from=, size=0, class=0, nrcpts=0, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=163.23.32.95.dsl-dynamic.vsi.ru [95.32.23.163] + +# filterOptions: {"mode": "extra"} + +# failJSON: { "time": "2005-03-06T16:55:28", "match": true , "host": "192.0.2.194", "desc": "wrong resp. non RFC compiant (ddos prelude?), MTA-mode" } +Mar 6 16:55:28 s192-168-0-1 sm-mta[20949]: v26LtRA0020949: some-host-24.example.org [192.0.2.194] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-07T15:04:37", "match": true , "host": "192.0.2.195", "desc": "wrong resp. non RFC compiant (ddos prelude?), MSP-mode, (may be forged)" } +Mar 7 15:04:37 s192-168-0-1 sm-mta[18624]: v27K4Vj8018624: some-host-24.example.org [192.0.2.195] (may be forged) did not issue MAIL/EXPN/VRFY/ETRN during connection to MSP-v4 From 0c1707afdabe4f4df60b7817f82ab9c88c629d8e Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Mar 2017 22:08:03 +0100 Subject: [PATCH 16/20] filter.d/sshd.conf: - optional parameter `mode` rewritten: normal (default), ddos, extra or aggressive (combines all), see sshd for regex details); test cases reformatted (since "filterOptions", we don't need multiple test log-files anymore); --- MANIFEST | 2 - config/filter.d/sshd-aggressive.conf | 11 --- config/filter.d/sshd-ddos.conf | 17 ---- config/filter.d/sshd.conf | 92 +++++++++++-------- config/jail.conf | 22 ++--- .../filter.d/zzz-sshd-obsolete-multiline.conf | 34 ++++--- fail2ban/tests/files/logs/sshd | 46 ++++++++++ fail2ban/tests/files/logs/sshd-aggressive | 3 - fail2ban/tests/files/logs/sshd-ddos | 41 --------- 9 files changed, 132 insertions(+), 136 deletions(-) delete mode 100644 config/filter.d/sshd-aggressive.conf delete mode 100644 config/filter.d/sshd-ddos.conf delete mode 100644 fail2ban/tests/files/logs/sshd-aggressive delete mode 100644 fail2ban/tests/files/logs/sshd-ddos diff --git a/MANIFEST b/MANIFEST index f03f185b..b726a111 100644 --- a/MANIFEST +++ b/MANIFEST @@ -138,7 +138,6 @@ config/filter.d/solid-pop3d.conf config/filter.d/squid.conf config/filter.d/squirrelmail.conf config/filter.d/sshd.conf -config/filter.d/sshd-ddos.conf config/filter.d/stunnel.conf config/filter.d/suhosin.conf config/filter.d/tine20.conf @@ -327,7 +326,6 @@ fail2ban/tests/files/logs/solid-pop3d fail2ban/tests/files/logs/squid fail2ban/tests/files/logs/squirrelmail fail2ban/tests/files/logs/sshd -fail2ban/tests/files/logs/sshd-ddos fail2ban/tests/files/logs/stunnel fail2ban/tests/files/logs/suhosin fail2ban/tests/files/logs/tine20 diff --git a/config/filter.d/sshd-aggressive.conf b/config/filter.d/sshd-aggressive.conf deleted file mode 100644 index 98175cbe..00000000 --- a/config/filter.d/sshd-aggressive.conf +++ /dev/null @@ -1,11 +0,0 @@ -# Fail2Ban aggressive ssh filter for at attempted exploit -# -# Includes failregex of both sshd and sshd-ddos filters -# -[INCLUDES] - -before = sshd.conf - -[Definition] - -mode = %(aggressive)s diff --git a/config/filter.d/sshd-ddos.conf b/config/filter.d/sshd-ddos.conf deleted file mode 100644 index 69b42069..00000000 --- a/config/filter.d/sshd-ddos.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Fail2Ban ssh filter for at attempted exploit -# -# The regex here also relates to a exploit: -# -# http://www.securityfocus.com/bid/17958/exploit -# The example code here shows the pushing of the exploit straight after -# reading the server version. This is where the client version string normally -# pushed. As such the server will read this unparsible information as -# "Did not receive identification string". - -[INCLUDES] - -before = sshd.conf - -[Definition] - -mode = %(ddos)s diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 922ea193..8163aa03 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -24,45 +24,59 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: \[preauth\])?\s* __on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)? -prefregex = ^%(__prefix_line)s%(__pref)s.+$ - -mode = %(normal)s - -normal = ^[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ - ^User not known to the underlying authentication module for .* from \s*%(__suff)s$ - ^Failed \S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^ROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ - ^[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ - ^User .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ - ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ - ^User .+ from not allowed because not in any group\s*%(__suff)s$ - ^refused connect from \S+ \(\)\s*%(__suff)s$ - ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ - ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ - ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ - ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ - ^User .+ not allowed because account is locked%(__suff)s - ^Disconnecting: Too many authentication failures for .+?%(__suff)s - ^Received disconnect from : 11: - ^Connection closed by %(__suff)s$ - -ddos = ^Did not receive identification string from %(__suff)s$ - ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ - ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. - ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$ - ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: - ^Read from socket failed: Connection reset by peer \[preauth\] - -common = ^Connection from - -aggressive = %(normal)s - %(ddos)s - [Definition] -failregex = %(mode)s - %(common)s +prefregex = ^%(__prefix_line)s%(__pref)s.+$ + +cmnfailre = ^[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ + ^User not known to the underlying authentication module for .* from \s*%(__suff)s$ + ^Failed \S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^ROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ + ^[iI](?:llegal|nvalid) user .*? from %(__on_port_opt)s\s*$ + ^User .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ + ^User .+ from not allowed because listed in DenyUsers\s*%(__suff)s$ + ^User .+ from not allowed because not in any group\s*%(__suff)s$ + ^refused connect from \S+ \(\)\s*%(__suff)s$ + ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ + ^User .+ from not allowed because a group is listed in DenyGroups\s*%(__suff)s$ + ^User .+ from not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$ + ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*%(__suff)s$ + ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)? \[preauth\]$ + ^User .+ not allowed because account is locked%(__suff)s + ^Disconnecting: Too many authentication failures for .+?%(__suff)s + ^Received disconnect from : 11: + ^Connection closed by %(__suff)s$ + +mdre-normal = + +mdre-ddos = ^Did not receive identification string from %(__suff)s$ + ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: + ^Read from socket failed: Connection reset by peer \[preauth\] + +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ + ^Unable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. + ^Unable to negotiate a (?:cipher|key exchange method)%(__suff)s$ + +mdre-aggressive = %(mdre-ddos)s + %(mdre-extra)s + +cfooterre = ^Connection from + +failregex = %(cmnfailre)s + > + %(cfooterre)s + +# Parameter "mode": normal (default), ddos, extra or aggressive (combines all) +# Usage example (for jail.local): +# [sshd] +# mode = extra +# # or another jail (rewrite filter parameters of jail): +# [sshd-aggressive] +# filter = sshd[mode=aggressive] +# +mode = normal + +#filter = sshd[mode=aggressive] ignoreregex = @@ -79,5 +93,5 @@ datepattern = {^LN-BEG} # and later catch-all's could contain user-provided input, which need to be greedily # matched away first. # -# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black - +# Author: Cyril Jaquier, Yaroslav Halchenko, Petr Voralek, Daniel Black and Sergey Brester aka sebres +# Rewritten using prefregex (and introduced "mode" parameter) by Serg G. Brester. diff --git a/config/jail.conf b/config/jail.conf index 3e917b34..c5440b71 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -229,17 +229,11 @@ action = %(action_)s [sshd] -# To use more aggressive sshd filter (inclusive sshd-ddos failregex): -#filter = sshd-aggressive -port = ssh -logpath = %(sshd_log)s -backend = %(sshd_backend)s - - -[sshd-ddos] -# This jail corresponds to the standard configuration in Fail2ban. -# The mail-whois action send a notification e-mail with a whois request -# in the body. +# To use more aggressive sshd modes set filter parameter "mode" in jail.local: +# normal (default), ddos, extra or aggressive (combines all). +# See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details. +mode = normal +filter = sshd[mode=%(mode)s] port = ssh logpath = %(sshd_log)s backend = %(sshd_backend)s @@ -555,7 +549,11 @@ backend = %(syslog_backend)s [sendmail-reject] - +# To use more aggressive modes set filter parameter "mode" in jail.local: +# normal (default), extra or aggressive +# See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details. +mode = normal +filter = sendmail-reject[mode=%(mode)s] port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 729fecd7..4f28e60f 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -23,9 +23,9 @@ __prefix_line_sl = %(__prefix_line)s%(__pref)s __prefix_line_ml1 = (?P<__prefix>%(__prefix_line)s)%(__pref)s __prefix_line_ml2 = %(__suff)s$^(?P=__prefix)%(__pref)s -mode = %(normal)s +[Definition] -normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ +cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sROOT LOGIN REFUSED.* FROM \s*%(__suff)s$ @@ -43,18 +43,30 @@ normal = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* ^%(__prefix_line_ml1)sDisconnecting: Too many authentication failures for .+?%(__prefix_line_ml2)sConnection closed by %(__suff)s$ ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sDisconnecting: Too many authentication failures for .+%(__suff)s$ -ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ - ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ - ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. - ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$ - ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ +mdre-normal = -aggressive = %(normal)s - %(ddos)s +mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__suff)s$ + ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -[Definition] +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$ + ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching (?:cipher|key exchange method) found. + ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a (?:cipher|key exchange method)%(__suff)s$ -failregex = %(mode)s +mdre-aggressive = %(mdre-ddos)s + %(mdre-extra)s + +failregex = %(cmnfailre)s + > + +# Parameter "mode": normal (default), ddos, extra or aggressive (combines all) +# Usage example (for jail.local): +# [sshd] +# mode = extra +# # or another jail (rewrite filter parameters of jail): +# [sshd-aggressive] +# filter = sshd[mode=aggressive] +# +mode = normal ignoreregex = diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e2c5f537..b53b3d96 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -183,3 +183,49 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root + +# filterOptions: {"mode": "ddos"} + +# http://forums.powervps.com/showthread.php?t=1667 +# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" } +Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 + +# gh-864(1): +# failJSON: { "match": false } +Nov 24 23:46:39 host sshd[32686]: SSH: Server;Ltype: Version;Remote: 127.0.0.1-1780;Protocol: 2.0;Client: libssh2_1.4.3 +# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (1)" } +Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# gh-864(2): +# failJSON: { "match": false } +Nov 24 23:46:40 host sshd[32686]: SSH: Server;Ltype: Kex;Remote: 127.0.0.1-1780;Enc: aes128-ctr;MAC: hmac-sha1;Comp: none [preauth] +# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (2)" } +Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# gh-864(3): +# failJSON: { "match": false } +Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-1780;Name: root [preauth] +# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" } +Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# filterOptions: {"mode": "extra"} + +# several other cases from gh-864: +# failJSON: { "time": "2004-11-25T01:34:12", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } +Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:13", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } +Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } +Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] + +# gh-1545: +# failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } +Nov 26 13:03:29 srv sshd[45]: Unable to negotiate with 192.0.2.1 port 55419: no matching cipher found. Their offer: aes256-cbc,rijndael-cbc@lysator.liu.se,aes192-cbc,aes128-cbc,arcfour128,arcfour,3des-cbc,none [preauth] + +# gh-1117: +# failJSON: { "time": "2004-11-26T13:03:30", "match": true , "host": "192.0.2.2", "desc": "No matching key exchange method" } +Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching key exchange method found. Their offer: diffie-hellman-group1-sha1 +# failJSON: { "match": false } +Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 +# failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } +Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file diff --git a/fail2ban/tests/files/logs/sshd-aggressive b/fail2ban/tests/files/logs/sshd-aggressive deleted file mode 100644 index 5b4d3a12..00000000 --- a/fail2ban/tests/files/logs/sshd-aggressive +++ /dev/null @@ -1,3 +0,0 @@ -# sshd-aggressive includes sshd and sshd-ddos failregex's: -# addFILE: "sshd" -# addFILE: "sshd-ddos" \ No newline at end of file diff --git a/fail2ban/tests/files/logs/sshd-ddos b/fail2ban/tests/files/logs/sshd-ddos deleted file mode 100644 index f0a787a1..00000000 --- a/fail2ban/tests/files/logs/sshd-ddos +++ /dev/null @@ -1,41 +0,0 @@ -# http://forums.powervps.com/showthread.php?t=1667 -# failJSON: { "time": "2005-06-07T01:10:56", "match": true , "host": "69.61.56.114" } -Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 - -# gh-864(1): -# failJSON: { "match": false } -Nov 24 23:46:39 host sshd[32686]: SSH: Server;Ltype: Version;Remote: 127.0.0.1-1780;Protocol: 2.0;Client: libssh2_1.4.3 -# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (1)" } -Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] - -# gh-864(2): -# failJSON: { "match": false } -Nov 24 23:46:40 host sshd[32686]: SSH: Server;Ltype: Kex;Remote: 127.0.0.1-1780;Enc: aes128-ctr;MAC: hmac-sha1;Comp: none [preauth] -# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (2)" } -Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] - -# gh-864(3): -# failJSON: { "match": false } -Nov 24 23:46:41 host sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-1780;Name: root [preauth] -# failJSON: { "time": "2004-11-24T23:46:43", "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" } -Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] - -# several other cases from gh-864: -# failJSON: { "time": "2004-11-25T01:34:12", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } -Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] -# failJSON: { "time": "2004-11-25T01:35:13", "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } -Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] -# failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } -Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] - -# gh-1545: -# failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } -Nov 26 13:03:29 srv sshd[45]: Unable to negotiate with 192.0.2.1 port 55419: no matching cipher found. Their offer: aes256-cbc,rijndael-cbc@lysator.liu.se,aes192-cbc,aes128-cbc,arcfour128,arcfour,3des-cbc,none [preauth] - -# gh-1117: -# failJSON: { "time": "2004-11-26T13:03:30", "match": true , "host": "192.0.2.2", "desc": "No matching key exchange method" } -Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching key exchange method found. Their offer: diffie-hellman-group1-sha1 -# failJSON: { "match": false } -Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 -# failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } -Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] \ No newline at end of file From 8af7a73bfc0e0588766dfd7a74f3c5249fa8d767 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 10 Mar 2017 22:14:39 +0100 Subject: [PATCH 17/20] update ChangeLog --- ChangeLog | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ChangeLog b/ChangeLog index 71e46daa..5ca9fa07 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,14 @@ TODO: implementing of options resp. other tasks from PR #1346 ### Fixes * `filter.d/pam-generic.conf`: - [grave] injection on user name to host fixed +* `filter.d/sshd.conf`: + - rewritten using `prefregex` and used MLFID-related multi-line parsing + (by using tag `` instead of buffering with `maxlines`); + - optional parameter `mode` rewritten: normal (default), ddos, extra or aggressive (combines all), + see sshd for regex details) +* filter.d/sendmail-reject.conf: + - rewritten using `prefregex` and used MLFID-related multi-line parsing; + - optional parameter `mode` introduced: normal (default), extra or aggressive * `action.d/complain.conf` - fixed using new tag `` (sh/dash compliant now) * `action.d/sendmail-geoip-lines.conf` @@ -51,6 +59,9 @@ TODO: implementing of options resp. other tasks from PR #1346 - `` - PTR reversed representation of IP address - `` - host name of the IP address - `` - interpolates to the corresponding filter group capture `...` +* Samples test case factory extended with filter options - dict in JSON to control + filter options (e. g. mode, etc.): + # filterOptions: {"mode": "aggressive"} ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc From 6a26602ba870e9acf367b9714d23efbb0fccbd93 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 11 Mar 2017 00:06:29 +0100 Subject: [PATCH 18/20] allow to use filter options by fail2ban-regex, example: fail2ban-regex text.log "sshd[mode=aggressive]" --- fail2ban/client/fail2banregex.py | 63 +++++++++++++++++++------ fail2ban/client/jailreader.py | 2 +- fail2ban/tests/fail2banregextestcase.py | 19 ++++++-- 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index e2f1f67c..539383f6 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -120,6 +120,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues version="%prog " + version) p.add_options([ + Option("-c", "--config", default='/etc/fail2ban', + help="set alternate config directory"), Option("-d", "--datepattern", help="set custom pattern used to match date/times"), Option("-e", "--encoding", default=PREFER_ENC, @@ -271,24 +273,55 @@ class Fail2banRegex(object): def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) regex = regextype + 'regex' - if regextype == 'fail' and (os.path.isfile(value) or os.path.isfile(value + '.conf')): - if os.path.basename(os.path.dirname(value)) == 'filter.d': + # try to check - we've case filter?[options...]?: + basedir = self._opts.config + fltFile = None + fltOpt = {} + if regextype == 'fail': + fltName, fltOpt = JailReader.extractOptions(value) + if fltName is not None: + if "." in fltName[~5:]: + tryNames = (fltName,) + else: + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + # if it is filter file: + if fltFile is not None: + if (basedir == self._opts.config + or os.path.basename(basedir) == 'filter.d' + or ("." not in fltName[~5:] and "/" not in fltName) + ): ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): - basedir = os.path.dirname(os.path.dirname(value)) - value = os.path.splitext(os.path.basename(value))[0] - output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) ) - reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir) - if not reader.read(): # pragma: no cover - output( "ERROR: failed to load filter %s" % value ) - return False - else: # pragma: no cover + if os.path.basename(basedir) == 'filter.d': + basedir = os.path.dirname(basedir) + fltName = os.path.splitext(os.path.basename(fltName))[0] + output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) + else: + ## foreign file - readexplicit this file and includes if possible: + output( "Use %11s file : %s" % (regex, fltName) ) + basedir = None + if fltOpt: + output( "Use filter options : %r" % fltOpt ) + reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) + if basedir is not None: # pragma: no cover + ret = reader.read() + else: ## foreign file - readexplicit this file and includes if possible: - output( "Use %11s file : %s" % (regex, value) ) - reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config) reader.setBaseDir(None) - if not reader.readexplicit(): - output( "ERROR: failed to read %s" % value ) - return False + ret = reader.readexplicit() + if not ret: + output( "ERROR: failed to load filter %s" % value ) + return False reader.getOptions(None) readercommands = reader.convert() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index f4adadbf..2bef2c4f 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -43,7 +43,7 @@ logSys = getLogger(__name__) class JailReader(ConfigReader): # regex, to extract list of options: - optionCRE = re.compile(r"^([\w\-_\.]+)(?:\[(.*)\])?\s*$", re.DOTALL) + optionCRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL) # regex, to iterate over single option in option list, syntax: # `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']' # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index af7af1bc..19006831 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -209,7 +209,8 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages "-v", "--verbose-date", "--print-all-matched", - Fail2banRegexTest.FILENAME_SSHD, Fail2banRegexTest.FILTER_SSHD + "-c", CONFIG_DIR, + Fail2banRegexTest.FILENAME_SSHD, "sshd" ) self.assertTrue(fail2banRegex.start(args)) # test failure line and not-failure lines both presents: @@ -220,7 +221,8 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages "--print-all-matched", - Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_SSHD + "-c", CONFIG_DIR, + Fail2banRegexTest.FILENAME_ZZZ_SSHD, "sshd.conf[mode=normal]" ) self.assertTrue(fail2banRegex.start(args)) # test failure line and all not-failure lines presents: @@ -234,7 +236,8 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages "--print-all-matched", "--print-all-missed", - Fail2banRegexTest.FILENAME_ZZZ_SSHD, Fail2banRegexTest.FILTER_ZZZ_SSHD + "-c", os.path.dirname(Fail2banRegexTest.FILTER_ZZZ_SSHD), + Fail2banRegexTest.FILENAME_ZZZ_SSHD, os.path.basename(Fail2banRegexTest.FILTER_ZZZ_SSHD) ) self.assertTrue(fail2banRegex.start(args)) # test "failure" line presents (2nd part only, because multiline fewer precise): @@ -245,10 +248,18 @@ class Fail2banRegexTest(LogCaptureTestCase): # by the way test of ignoreregex (specified in filter file)... (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages - Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN + Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN+"[mode=test]" ) self.assertTrue(fail2banRegex.start(args)) + def testWrongFilterFile(self): + # use test log as filter file to cover eror cases... + (opts, args, fail2banRegex) = _Fail2banRegex( + "-l", "notice", # put down log-level, because of too many debug-messages + Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILENAME_ZZZ_GEN + ) + self.assertFalse(fail2banRegex.start(args)) + def _reset(self): # reset global warn-counter: from ..server.filter import _decode_line_warn From eb3623e90cf1db8031cb89b40f7fdb8fee05b9f5 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 12 Mar 2017 19:04:45 +0100 Subject: [PATCH 19/20] configreader.py: correct reading real relative path (starting with "./"); fail2ban-regex: catch read exceptions by wrong config files (raise exception in verbose mode only); --- fail2ban/client/configreader.py | 4 ++++ fail2ban/client/fail2banregex.py | 17 +++++++++++------ fail2ban/tests/fail2banregextestcase.py | 1 - 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 2caa97ce..b7da271b 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -176,6 +176,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): if not os.path.exists(self._basedir): raise ValueError("Base configuration directory %s does not exist " % self._basedir) + if filename.startswith("./"): # pragma: no cover + filename = os.path.abspath(filename) basename = os.path.join(self._basedir, filename) logSys.debug("Reading configs for %s under %s " , filename, self._basedir) config_files = [ basename + ".conf" ] @@ -277,6 +279,8 @@ class DefinitionInitConfigReader(ConfigReader): def __init__(self, file_, jailName, initOpts, **kwargs): ConfigReader.__init__(self, **kwargs) + if file_.startswith("./"): # pragma: no cover + file_ = os.path.abspath(file_) self.setFile(file_) self.setJailName(jailName) self._initOpts = initOpts diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 539383f6..911abd28 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -313,12 +313,17 @@ class Fail2banRegex(object): if fltOpt: output( "Use filter options : %r" % fltOpt ) reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) - if basedir is not None: # pragma: no cover - ret = reader.read() - else: - ## foreign file - readexplicit this file and includes if possible: - reader.setBaseDir(None) - ret = reader.readexplicit() + ret = None + try: + if basedir is not None: + ret = reader.read() + else: + ## foreign file - readexplicit this file and includes if possible: + reader.setBaseDir(None) + ret = reader.readexplicit() + except Exception as e: + output("Wrong config file: %s" % (str(e),)) + if self._verbose: raise(e) if not ret: output( "ERROR: failed to load filter %s" % value ) return False diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 19006831..1bac3a5f 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -255,7 +255,6 @@ class Fail2banRegexTest(LogCaptureTestCase): def testWrongFilterFile(self): # use test log as filter file to cover eror cases... (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILENAME_ZZZ_GEN ) self.assertFalse(fail2banRegex.start(args)) From 30b53bb2ce0c4e7e0a0af9689c297f0428747877 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 13 Mar 2017 02:07:14 +0100 Subject: [PATCH 20/20] update ChangeLog and man/fail2ban-regex.1 --- ChangeLog | 2 ++ man/fail2ban-regex.1 | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5ca9fa07..5c1ba70e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,8 @@ TODO: implementing of options resp. other tasks from PR #1346 - `` - PTR reversed representation of IP address - `` - host name of the IP address - `` - interpolates to the corresponding filter group capture `...` +* Allow to use filter options by `fail2ban-regex`, example: + fail2ban-regex text.log "sshd[mode=aggressive]" * Samples test case factory extended with filter options - dict in JSON to control filter options (e. g. mode, etc.): # filterOptions: {"mode": "aggressive"} diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 44e13c86..3f31a376 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -27,6 +27,9 @@ a string representing a 'failregex' .TP filename path to a filter file (filter.d/sshd.conf) +.TP +filtername[option=value, ..., option=value] +short path to a filter relative filter.d in configuration base (sshd[mode=aggressive]) .SS "IGNOREREGEX:" .TP string @@ -42,6 +45,9 @@ show program's version number and exit \fB\-h\fR, \fB\-\-help\fR show this help message and exit .TP +\fB\-c\fR CONFIGBASE, \fB\-\-config\fR=\fI\,CONFIGBASE\/\fR +set alternate config base directory (default /etc/fail2ban) +.TP \fB\-d\fR DATEPATTERN, \fB\-\-datepattern\fR=\fI\,DATEPATTERN\/\fR set custom pattern used to match date/times .TP