From 7805f9972d361eeb9ded6eabd69d84a7618c0fe7 Mon Sep 17 00:00:00 2001 From: Aaron Lindsay Date: Sat, 15 Oct 2016 12:02:45 -0400 Subject: [PATCH 001/147] filter.d/sshd.conf: Match 'Invalid user' with 'port \d*' --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index eeb1518e..9fff72ac 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -22,7 +22,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ ^%(__prefix_line)sFailed \S+ for .*? from (?: port \d*)?(?: ssh\d*)?(: (ruser .*|(\S+ ID \S+ \(serial \d+\) CA )?\S+ %(__md5hex)s(, client user ".*", client host ".*")?))?\s*$ ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ - ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from \s*$ + ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from (?: port \d*)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7baf4be7..be5896cc 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -162,3 +162,6 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root +# Match invalid user messages with port at the end +# failJSON: {"time": "2004-10-15T11:35:28", "match": true , "host": "1.2.3.4", "desc": "Invalid user root" } +Oct 15 11:35:28 somehost sshd[7024]: Invalid user root from 1.2.3.4 port 37220 From 8c407665119e37dd3f27433bc20bac813c9d3c05 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 10 Nov 2016 06:29:42 +0100 Subject: [PATCH 002/147] Add Mongodb-auth filter and jail --- config/filter.d/mongodb-auth.conf | 28 ++++++++++++++++++++++++++++ config/jail.conf | 12 ++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 config/filter.d/mongodb-auth.conf diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf new file mode 100644 index 00000000..77443f9c --- /dev/null +++ b/config/filter.d/mongodb-auth.conf @@ -0,0 +1,28 @@ +# Fail2Ban filter for unsuccesfull MongoDB authentication attempts +# +# + +[Init] +maxlines = 10 + +[Definition] +failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]*\s+\[conn(?P=__connid)\] end connection + +ignoreregex = + + +# DEV Notes: +# +# Log example: +# 2016-10-20T09:54:27.108+0200 [initandlisten] connection accepted from 127.0.0.1:53276 #1 (1 connection now open) +# 2016-10-20T09:54:27.109+0200 [conn1] authenticate db: test { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# 2016-10-20T09:54:27.110+0200 [conn1] Failed to authenticate root@test with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@test +# 2016-11-09T09:54:27.894+0100 [conn1] end connection 127.0.0.1:53276 (0 connections now open) +# 2016-11-09T11:55:58.890+0100 [initandlisten] connection accepted from 127.0.0.1:54266 #1510 (1 connection now open) +# 2016-11-09T11:55:58.892+0100 [conn1510] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# 2016-11-09T11:55:58.892+0100 [conn1510] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# 2016-11-09T11:55:58.894+0100 [conn1510] end connection 127.0.0.1:54266 (0 connections now open) +# +# Authors: Alexander Finkhäuser and sebres +# + diff --git a/config/jail.conf b/config/jail.conf index 41d8d3b8..50ecbd72 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,6 +731,18 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +# Log wrong MongoDB auth +# Logfile /var/log/mongodb/mongodb.log +# add setting in /etc/mongodb.conf +# logpath=/var/log/mongodb/mongodb.log +# +# and use of the authentication +# auth = true +[mongodb-auth] +port = 27017 +logpath = /var/log/mongodb/mongodb.log + + # Jail for more extended banning of persistent abusers # !!! WARNINGS !!! # 1. Make sure that your loglevel specified in fail2ban.conf/.local From 8ac28e5dcb73a70cc5caa8ff8df60ed5b901a9d6 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 10 Nov 2016 13:09:32 +0100 Subject: [PATCH 003/147] Make changes and add test file --- config/filter.d/mongodb-auth.conf | 7 +++++++ config/jail.conf | 8 +------- fail2ban/tests/files/logs/mongodb-auth | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 fail2ban/tests/files/logs/mongodb-auth diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf index 77443f9c..43462e73 100644 --- a/config/filter.d/mongodb-auth.conf +++ b/config/filter.d/mongodb-auth.conf @@ -1,5 +1,12 @@ # Fail2Ban filter for unsuccesfull MongoDB authentication attempts # +# Logfile /var/log/mongodb/mongodb.log +# +# add setting in /etc/mongodb.conf +# logpath=/var/log/mongodb/mongodb.log +# +# and use of the authentication +# auth = true # [Init] diff --git a/config/jail.conf b/config/jail.conf index 50ecbd72..0778a318 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,13 +731,7 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s -# Log wrong MongoDB auth -# Logfile /var/log/mongodb/mongodb.log -# add setting in /etc/mongodb.conf -# logpath=/var/log/mongodb/mongodb.log -# -# and use of the authentication -# auth = true +# Log wrong MongoDB auth (for details how ... see filter ...) [mongodb-auth] port = 27017 logpath = /var/log/mongodb/mongodb.log diff --git a/fail2ban/tests/files/logs/mongodb-auth b/fail2ban/tests/files/logs/mongodb-auth new file mode 100644 index 00000000..a9c7b99b --- /dev/null +++ b/fail2ban/tests/files/logs/mongodb-auth @@ -0,0 +1,24 @@ +# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.168.1.35" } +2016-11-20T00:04:00.110+0200 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +2016-11-20T00:04:00.111+0200 [conn1] end connection 192.168.1.35:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.171" } +2016-11-20T00:24:00.110+0200 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +2016-11-20T00:24:00.111+0200 [conn5] end connection 220.95.238.171:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.176" } +2016-11-20T00:24:00.110+0200 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +2016-11-20T00:24:00.111+0200 [conn334] end connection 220.95.238.176:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "167.96.268.1" } +2016-11-20T00:24:00.110+0200 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +2016-11-20T00:24:00.111+0200 [conn56] end connection 167.96.268.1:53276 (0 connections now open) + +# failJSON: { "time": "2016-11-20T00:24:00", "match": false , "host": "127.0.0.1" } +2016-11-10T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) +2016-11-10T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) + +# failJSON: { "time": "2016-11-10T13:07:49", "match": false , "host": "177.13.20.178" } +2016-11-10T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +2016-11-10T13:07:49.834+0100 [conn2271] end connection 177.13.20.178:60268 (3 connections now open) + From b5433f48b7b0c7673b43c7956e0b74737b9116ca Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 11 Nov 2016 11:04:10 +0100 Subject: [PATCH 004/147] amend after code review of merge gh-1581 --- ChangeLog | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 10 ++++------ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1ed40dca..ca7864fc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ releases. - recognized "Failed publickey for" (gh-1477); - optimized failregex to match all of "Failed any-method for ... from " (gh-1479) - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) + - optional port part after host (see gh-1533, gh-1581) ### New Features diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index bdc1851e..35cd8754 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -22,7 +22,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ - ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from (?: port \d*)?\s*$ + ^%(__prefix_line)s[iI](?:llegal|nvalid) user .*? from (?: port \d+)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 7ceb899c..0800f86b 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -17,8 +17,10 @@ Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 Jan 5 01:31:41 www sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 #4 -# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "211.114.51.213" } -Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 211.114.51.213 +# failJSON: { "time": "2005-07-20T14:42:11", "match": true , "host": "192.0.2.1", "desc": "Invalid user" } +Jul 20 14:42:11 localhost sshd[22708]: Invalid user ftp from 192.0.2.1 +# failJSON: { "time": "2005-07-20T14:42:12", "match": true , "host": "192.0.2.2", "desc": "Invalid user with port" } +Jul 20 14:42:12 localhost sshd[22708]: Invalid user ftp from 192.0.2.2 port 37220 #5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch # yoh: added ':' after [sshd] since the case without is not really common any more @@ -167,7 +169,3 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems # failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "222.186.21.217", "desc": "Authentication for user failed" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=222.186.21.217 user=root - -# Match invalid user messages with port at the end -# failJSON: {"time": "2004-10-15T11:35:28", "match": true , "host": "1.2.3.4", "desc": "Invalid user root" } -Oct 15 11:35:28 somehost sshd[7024]: Invalid user root from 1.2.3.4 port 37220 From f46ada023ee35ebccc38aef2b57992288a276c05 Mon Sep 17 00:00:00 2001 From: Johannes Weberhofer Date: Thu, 10 Mar 2016 15:30:46 +0100 Subject: [PATCH 005/147] Use Fedora's backend-settings for openSUSE Those settings are ok for newer openSUSE versions --- config/paths-opensuse.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/paths-opensuse.conf b/config/paths-opensuse.conf index 0d6ad522..227a5e98 100644 --- a/config/paths-opensuse.conf +++ b/config/paths-opensuse.conf @@ -36,3 +36,15 @@ mysql_log = /var/log/mysql/mysqld.log roundcube_errors_log = /srv/www/roundcubemail/logs/errors solidpop3d_log = %(syslog_mail)s + +# These services will log to the journal via syslog, so use the journal by +# default. +syslog_backend = systemd +sshd_backend = systemd +dropbear_backend = systemd +proftpd_backend = systemd +pureftpd_backend = systemd +wuftpd_backend = systemd +postfix_backend = systemd +dovecot_backend = systemd +mysql_backend = systemd From 4f5389fee5ae97630a6f68d2823378011e66345d Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Thu, 24 Nov 2016 19:30:10 +0100 Subject: [PATCH 006/147] Update jail.conf --- config/jail.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 41d8d3b8..7200f4eb 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -810,8 +810,9 @@ maxretry = 1 [pass2allow-ftp] # this pass2allow example allows FTP traffic after successful HTTP authentication port = ftp,ftp-data,ftps,ftps-data -# knocking_url variable must be overridden to some secret value in filter.d/apache-pass.local -filter = apache-pass +# knocking_url variable must be overridden to some secret value in jail.local +knocking_url = /knocking/ +filter = apache-pass[knocking_url="%(knocking_url)s"] # access log of the website with HTTP auth logpath = %(apache_access_log)s blocktype = RETURN From 5678d08a79e8cfcba15389cd8e1bbfb10bb9ff68 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 26 Nov 2016 16:50:37 +0100 Subject: [PATCH 007/147] filter.d/dovecot.conf update: - fixes failregex, that ignores failures through some irrelevant info (closes #1623); - ignores whole additionally irrelevant info in anchored regex before fixed failure data `\((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\)` - review, IPv6 compatibility fix, non-capturing groups --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/dovecot.conf | 11 ++++++----- fail2ban/tests/files/logs/dovecot | 5 +++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca7864fc..ec8bdda3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,8 @@ releases. * Monit config: scripting is not supported in path (gh-1556) * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) +* `filter.d/dovecot.conf` + - Fixed failregex ignores failures through some not relevant info (gh-1623) * `filter.d/ignorecommands/apache-fakegooglebot` - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) diff --git a/THANKS b/THANKS index 64de43a4..7d9137d7 100644 --- a/THANKS +++ b/THANKS @@ -119,6 +119,7 @@ Thomas Mayer Tom Pike Tom Hendrikx Tomas Pihl +Thomas Skierlo (phaleas) Tony Lawrence Tomasz Ciolek Tyler diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 136a3947..a3dc60bb 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -9,11 +9,11 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) -failregex = ^%(__prefix_line)s(%(__pam_auth)s(\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ - ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=(, lip=(\d{1,3}\.){3}\d{1,3})?(, TLS( handshaking(: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ - ^%(__prefix_line)s(Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ - ^%(__prefix_line)s(auth|auth-worker\(\d+\)): (pam|passwd-file)\(\S+,\): unknown user\s*$ - ^%(__prefix_line)s(auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ +failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ + ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ + ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,\): unknown user\s*$ + ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ ignoreregex = @@ -30,3 +30,4 @@ journalmatch = _SYSTEMD_UNIT=dovecot.service # Author: Martin Waschbuesch # Daniel Black (rewrote with begin and end anchors) # Martin O'Neal (added LDAP authentication failure regex) +# Sergey G. Brester aka sebres (reviewed, optimized, IPv6-compatibility) diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 627b8dc8..987f89f1 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -73,3 +73,8 @@ Jul 02 13:49:32 hostname dovecot[442]: pop3-login: Disconnected (no auth attempt # failJSON: { "time": "2005-03-23T06:10:52", "match": true , "host": "52.37.139.121" } Mar 23 06:10:52 auth: Info: ldap(dog,52.37.139.121,): invalid credentials + +# failJSON: { "time": "2005-07-26T11:11:21", "match": true , "host": "192.0.2.1" } +Jul 26 11:11:21 hostname dovecot: imap-login: Disconnected: Too many invalid commands (tried to use disallowed plaintext auth): user=, rip=192.0.2.1, lip=192.168.1.1, session= +# failJSON: { "time": "2005-07-26T11:12:19", "match": true , "host": "192.0.2.2" } +Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid commands (auth failed, 1 attempts in 17 secs): user=, method=PLAIN, rip=192.0.2.2, lip=192.168.1.1, TLS, session= From 931eab84b51f988b45e51844833c4a0807c2b7d4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 11:28:27 +0100 Subject: [PATCH 008/147] `filter.d/apache-modsecurity.conf` - fixed for newer version (one space, closes gh-1626) reviewed and optimized: - non-greedy catch-all replaced for safer match - unneeded catch-all anchoring removed - non-capturing groups --- config/filter.d/apache-modsecurity.conf | 3 ++- fail2ban/tests/files/logs/apache-modsecurity | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/filter.d/apache-modsecurity.conf b/config/filter.d/apache-modsecurity.conf index ad7e9b24..13e9c5ea 100644 --- a/config/filter.d/apache-modsecurity.conf +++ b/config/filter.d/apache-modsecurity.conf @@ -10,9 +10,10 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s ModSecurity: (\[.*?\] )*Access denied with code [45]\d\d.*$ +failregex = ^%(_apache_error_client)s ModSecurity:\s+(?:\[(?:\w+ \"[^\"]*\"|[^\]]*)\]\s*)*Access denied with code [45]\d\d ignoreregex = # https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats # Author: Daniel Black +# Sergey G. Brester aka sebres (review, optimization) \ No newline at end of file diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index d46d8ab4..3ca2e074 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -1,5 +1,5 @@ # failJSON: { "time": "2013-12-23T13:12:31", "match": true , "host": "173.255.225.101" } [Mon Dec 23 13:12:31 2013] [error] [client 173.255.225.101] ModSecurity: [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_crs_21_protocol_anomalies.conf"] [line "47"] [id "960015"] [rev "1"] [msg "Request Missing an Accept Header"] [severity "NOTICE"] [ver "OWASP_CRS/2.2.8"] [maturity "9"] [accuracy "9"] [tag "OWASP_CRS/PROTOCOL_VIOLATION/MISSING_HEADER_ACCEPT"] [tag "WASCTC/WASC-21"][tag "OWASP_TOP_10/A7"] [tag "PCI/6.5.10"] Access denied with code 403 (phase 2). Operator EQ matched 0 at REQUEST_HEADERS. [hostname "www.mysite.net"] [uri "/"] [unique_id "Urf@f12qgHIAACrFOlgAAABA"] -# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69" } -[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] +# failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69", "desc": "additional entry (and exact one space)" } +[Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] From b8c41dcb491b6ed4ceb2e89127b6e1185e4ca344 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 11:31:51 +0100 Subject: [PATCH 009/147] ChangeLog update --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index ec8bdda3..29124261 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,9 @@ releases. * Fixed ambiguous wrong recognized date pattern resp. its optional parts (see gh-1512); * FIPS compliant, use sha1 instead of md5 if it not allowed (see gh-1540) * Monit config: scripting is not supported in path (gh-1556) +* `filter.d/apache-modsecurity.conf` + - Fixed for newer version (one space, gh-1626), optimized: non-greedy catch-all + replaced for safer match, unneeded catch-all anchoring removed, non-capturing * `filter.d/asterisk.conf` - Fixed to match different asterisk log prefix (source file: method:) * `filter.d/dovecot.conf` From 425170cef3d9c2f3ac98361d931111575a8a8fc2 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 18:37:12 +0100 Subject: [PATCH 010/147] code review, makes the test cases workable, added dev-notes --- config/filter.d/mongodb-auth.conf | 26 ++++++++++++---- fail2ban/tests/files/logs/mongodb-auth | 42 +++++++++++++++----------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/config/filter.d/mongodb-auth.conf b/config/filter.d/mongodb-auth.conf index 43462e73..66c27abb 100644 --- a/config/filter.d/mongodb-auth.conf +++ b/config/filter.d/mongodb-auth.conf @@ -9,17 +9,31 @@ # auth = true # -[Init] -maxlines = 10 - [Definition] -failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]*\s+\[conn(?P=__connid)\] end connection +#failregex = ^\s+\[initandlisten\] connection accepted from :\d+ \#(?P<__connid>\d+) \(1 connection now open\)\s+\[conn(?P=__connid)\] Failed to authenticate\s+ +failregex = ^\s+\[conn(?P<__connid>\d+)\] Failed to authenticate [^\n]+\s+\[conn(?P=__connid)\] end connection ignoreregex = +[Init] +maxlines = 10 + # DEV Notes: # +# Regarding the multiline regex: +# +# There can be a nunber of non-related lines between the first and second part +# of this regex maxlines of 10 is quite generious. +# +# Note the capture __connid, includes the connection ID, used in second part of regex. +# +# The first regex is commented out (but will match also), because it is better to use +# the host from "end connection" line (uncommented above): +# - it has the same prefix, searching begins directly with failure message +# (so faster, because ignores success connections at all) +# - it is not so vulnerable in case of possible race condition +# # Log example: # 2016-10-20T09:54:27.108+0200 [initandlisten] connection accepted from 127.0.0.1:53276 #1 (1 connection now open) # 2016-10-20T09:54:27.109+0200 [conn1] authenticate db: test { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } @@ -30,6 +44,6 @@ ignoreregex = # 2016-11-09T11:55:58.892+0100 [conn1510] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch # 2016-11-09T11:55:58.894+0100 [conn1510] end connection 127.0.0.1:54266 (0 connections now open) # -# Authors: Alexander Finkhäuser and sebres -# +# Authors: Alexander Finkhäuser +# Sergey G. Brester (sebres) diff --git a/fail2ban/tests/files/logs/mongodb-auth b/fail2ban/tests/files/logs/mongodb-auth index a9c7b99b..8a308892 100644 --- a/fail2ban/tests/files/logs/mongodb-auth +++ b/fail2ban/tests/files/logs/mongodb-auth @@ -1,24 +1,30 @@ -# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.168.1.35" } -2016-11-20T00:04:00.110+0200 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin -2016-11-20T00:04:00.111+0200 [conn1] end connection 192.168.1.35:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:04:00.110+0100 [conn1] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +# failJSON: { "time": "2016-11-20T00:04:00", "match": true , "host": "192.0.2.35" } +2016-11-20T00:04:00.111+0100 [conn1] end connection 192.0.2.35:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.171" } -2016-11-20T00:24:00.110+0200 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin -2016-11-20T00:24:00.111+0200 [conn5] end connection 220.95.238.171:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn5] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed UserNotFound Could not find user root@admin +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.171" } +2016-11-20T00:24:00.111+0100 [conn5] end connection 192.0.2.171:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "220.95.238.176" } -2016-11-20T00:24:00.110+0200 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch -2016-11-20T00:24:00.111+0200 [conn334] end connection 220.95.238.176:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn334] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.176" } +2016-11-20T00:24:00.111+0100 [conn334] end connection 192.0.2.176:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "167.96.268.1" } -2016-11-20T00:24:00.110+0200 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch -2016-11-20T00:24:00.111+0200 [conn56] end connection 167.96.268.1:53276 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T00:24:00.110+0100 [conn56] Failed to authenticate root@admin with mechanism MONGODB-CR: AuthenticationFailed key mismatch +# failJSON: { "time": "2016-11-20T00:24:00", "match": true , "host": "192.0.2.1" } +2016-11-20T00:24:00.111+0100 [conn56] end connection 192.0.2.1:53276 (0 connections now open) -# failJSON: { "time": "2016-11-20T00:24:00", "match": false , "host": "127.0.0.1" } -2016-11-10T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) -2016-11-10T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) +# failJSON: { "match": false } +2016-11-20T12:54:02.370+0100 [initandlisten] connection accepted from 127.0.0.1:58774 #2261 (1 connection now open) +# failJSON: { "match": false } +2016-11-20T12:54:02.370+0100 [conn2261] end connection 127.0.0.1:58774 (0 connections now open) -# failJSON: { "time": "2016-11-10T13:07:49", "match": false , "host": "177.13.20.178" } -2016-11-10T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } -2016-11-10T13:07:49.834+0100 [conn2271] end connection 177.13.20.178:60268 (3 connections now open) +# failJSON: { "match": false } +2016-11-20T13:07:49.781+0100 [conn2271] authenticate db: admin { authenticate: 1, nonce: "xxx", user: "root", key: "xxx" } +# failJSON: { "time": "2016-11-20T13:07:49", "match": false , "host": "192.0.2.178" } +2016-11-20T13:07:49.834+0100 [conn2271] end connection 192.0.2.178:60268 (3 connections now open) From 67c14afd8ee1c1cd3edc417474ab7afc65e1b393 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 28 Nov 2016 18:51:23 +0100 Subject: [PATCH 011/147] ChangeLog entry added + jail.conf review --- ChangeLog | 4 ++++ config/jail.conf | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2a6b5733..45d3ad99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,10 @@ releases. * New Actions: - `action.d/npf.conf` for NPF, the latest packet filter for NetBSD +* New Filters: + - `filter.d/mongodb-auth.conf` for MongoDB (document-oriented NoSQL database engine) + + ### Enhancements * DateTemplate regexp extended with the word-end boundary, additionally to word-start boundary diff --git a/config/jail.conf b/config/jail.conf index 0778a318..4d9f3345 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -731,8 +731,9 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s -# Log wrong MongoDB auth (for details how ... see filter ...) +# Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] +# change port when running with "--shardsvr" or "--configsvr" runtime operation port = 27017 logpath = /var/log/mongodb/mongodb.log From 556a9373ce42c934825c090370e1abe7e1670eb5 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 28 Nov 2016 23:40:33 +0100 Subject: [PATCH 012/147] Update ChangeLog --- ChangeLog | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index dc37e55c..e86fce18 100644 --- a/ChangeLog +++ b/ChangeLog @@ -49,14 +49,12 @@ releases. - eliminated possible complex injections (on user-name resp. auth-info, see gh-1479) - optional port part after host (see gh-1533, gh-1581) - ### New Features * New Actions: - `action.d/npf.conf` for NPF, the latest packet filter for NetBSD - * New Filters: - `filter.d/mongodb-auth.conf` for MongoDB (document-oriented NoSQL database engine) - + (gh-1586, gh-1606 and gh-1607) ### Enhancements * DateTemplate regexp extended with the word-end boundary, additionally to From e550850b9c3f04461624bf91ae438c920ac4d215 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:34:44 -0500 Subject: [PATCH 013/147] BF: added missing entires into MANIFEST --- MANIFEST | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/MANIFEST b/MANIFEST index e0d7398c..b12e3163 100644 --- a/MANIFEST +++ b/MANIFEST @@ -33,12 +33,14 @@ config/action.d/iptables-new.conf config/action.d/iptables-xt_recent-echo.conf config/action.d/mail-buffered.conf config/action.d/mail.conf +config/action.d/mail-whois-common.conf config/action.d/mail-whois.conf config/action.d/mail-whois-lines.conf config/action.d/mynetwatchman.conf config/action.d/nftables-allports.conf config/action.d/nftables-common.conf config/action.d/nftables-multiport.conf +config/action.d/npf.conf config/action.d/nsupdate.conf config/action.d/osx-afctl.conf config/action.d/osx-ipfw.conf @@ -54,6 +56,7 @@ config/action.d/sendmail-whois-ipmatches.conf config/action.d/sendmail-whois-lines.conf config/action.d/sendmail-whois-matches.conf config/action.d/shorewall.conf +config/action.d/shorewall-ipset-proto6.conf config/action.d/smtp.py config/action.d/symbiosis-blacklist-allports.conf config/action.d/ufw.conf @@ -69,6 +72,7 @@ config/filter.d/apache-modsecurity.conf config/filter.d/apache-nohome.conf config/filter.d/apache-noscript.conf config/filter.d/apache-overflows.conf +config/filter.d/apache-pass.conf config/filter.d/apache-shellshock.conf config/filter.d/assp.conf config/filter.d/asterisk.conf @@ -81,11 +85,13 @@ config/filter.d/cyrus-imap.conf config/filter.d/directadmin.conf config/filter.d/dovecot.conf config/filter.d/dropbear.conf +config/filter.d/drupal-auth.conf config/filter.d/ejabberd-auth.conf config/filter.d/exim-common.conf config/filter.d/exim.conf config/filter.d/exim-spam.conf config/filter.d/freeswitch.conf +config/filter.d/froxlor-auth.conf config/filter.d/groupoffice.conf config/filter.d/gssftpd.conf config/filter.d/guacamole.conf @@ -95,6 +101,7 @@ config/filter.d/ignorecommands config/filter.d/ignorecommands/apache-fakegooglebot config/filter.d/kerio.conf config/filter.d/lighttpd-auth.conf +config/filter.d/mongodb-auth.conf config/filter.d/monit.conf config/filter.d/murmur.conf config/filter.d/mysqld-auth.conf @@ -150,6 +157,7 @@ config/paths-opensuse.conf config/paths-osx.conf CONTRIBUTING.md COPYING +.coveragerc DEVELOP doc/run-rootless.txt fail2ban-2to3 @@ -206,6 +214,7 @@ fail2ban/tests/banmanagertestcase.py fail2ban/tests/clientreadertestcase.py fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/common.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local @@ -257,6 +266,7 @@ fail2ban/tests/files/logs/apache-modsecurity fail2ban/tests/files/logs/apache-nohome fail2ban/tests/files/logs/apache-noscript fail2ban/tests/files/logs/apache-overflows +fail2ban/tests/files/logs/apache-pass fail2ban/tests/files/logs/apache-shellshock fail2ban/tests/files/logs/assp fail2ban/tests/files/logs/asterisk @@ -270,10 +280,12 @@ fail2ban/tests/files/logs/cyrus-imap fail2ban/tests/files/logs/directadmin fail2ban/tests/files/logs/dovecot fail2ban/tests/files/logs/dropbear +fail2ban/tests/files/logs/drupal-auth fail2ban/tests/files/logs/ejabberd-auth fail2ban/tests/files/logs/exim fail2ban/tests/files/logs/exim-spam fail2ban/tests/files/logs/freeswitch +fail2ban/tests/files/logs/froxlor-auth fail2ban/tests/files/logs/groupoffice fail2ban/tests/files/logs/gssftpd fail2ban/tests/files/logs/guacamole @@ -281,6 +293,7 @@ fail2ban/tests/files/logs/haproxy-http-auth fail2ban/tests/files/logs/horde fail2ban/tests/files/logs/kerio fail2ban/tests/files/logs/lighttpd-auth +fail2ban/tests/files/logs/mongodb-auth fail2ban/tests/files/logs/monit fail2ban/tests/files/logs/murmur fail2ban/tests/files/logs/mysqld-auth @@ -357,6 +370,8 @@ files/gentoo-confd files/gentoo-initd files/ipmasq-ZZZzzz_fail2ban.rul files/logwatch/fail2ban +files/logwatch/fail2ban-0.8.log +files/logwatch/fail2ban-0.9.log files/macosx-initd files/monit/fail2ban files/nagios/check_fail2ban @@ -374,8 +389,11 @@ man/fail2ban-regex.1 man/fail2ban-regex.h2m man/fail2ban-server.1 man/fail2ban-server.h2m +man/fail2ban-testcases.1 +man/fail2ban-testcases.h2m man/generate-man man/jail.conf.5 +.pylintrc README.md README.Solaris RELEASE From 482252dbd4d6576a376158c5b49da82e5af8b7c4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:35:03 -0500 Subject: [PATCH 014/147] ENH: prep for 0.9.6 release (as of tomorrow) --- ChangeLog | 12 ++++++------ README.md | 6 +++--- RELEASE | 8 ++++---- fail2ban/version.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index e86fce18..0852a360 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,7 +6,7 @@ Fail2Ban: Changelog =================== -ver. 0.9.6 (2016/XX/XX) - wanna-be-released +ver. 0.9.6 (2016/12/10) - stretch-is-coming ----------- 0.9.x line is no longer heavily developed. If you are interested in @@ -18,7 +18,7 @@ releases. induced a subsequent error: last position of log file will be never retrieved (gh-795) * Fixed a distribution related bug within testReadStockJailConfForceEnabled (e.g. test-cases faults on Fedora, see gh-1353) -* Fixed pythonic filters and test scripts (running via wrong python version, +* Fixed pythonic filters and test scripts (running via wrong python version, uses "fail2ban-python" now); * Fixed test case "testSetupInstallRoot" for not default python version (also using direct call, out of virtualenv); @@ -33,7 +33,7 @@ releases. * `filter.d/dovecot.conf` - Fixed failregex ignores failures through some not relevant info (gh-1623) * `filter.d/ignorecommands/apache-fakegooglebot` - - Fixed error within apache-fakegooglebot, that will be called + - Fixed error within apache-fakegooglebot, that will be called with wrong python version (gh-1506) * `filter.d/assp.conf` - Extended failregex and test cases to handle ASSP V1 and V2 (gh-1494) @@ -57,11 +57,11 @@ releases. (gh-1586, gh-1606 and gh-1607) ### Enhancements -* DateTemplate regexp extended with the word-end boundary, additionally to +* DateTemplate regexp extended with the word-end boundary, additionally to word-start boundary -* Introduces new command "fail2ban-python", as automatically created symlink to +* Introduces new command "fail2ban-python", as automatically created symlink to python executable, where fail2ban currently installed (resp. its modules are located): - - allows to use the same version, fail2ban currently running, e.g. in + - allows to use the same version, fail2ban currently running, e.g. in external scripts just via replace python with fail2ban-python: ```diff -#!/usr/bin/env python diff --git a/README.md b/README.md index c2ef6c70..998e43e2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.5 2016/07/15 + v0.9.6 2016/12/10 ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -39,8 +39,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.5.tar.bz2 - cd fail2ban-0.9.5 + tar xvfj fail2ban-0.9.6.tar.bz2 + cd fail2ban-0.9.6 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index 6ad9b52d..b879fa20 100644 --- a/RELEASE +++ b/RELEASE @@ -53,7 +53,7 @@ Preparation or an alternative for comparison with previous release - git diff 0.9.5 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST + git diff 0.9.6 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST sort MANIFEST | uniq | sponge MANIFEST * Run:: @@ -70,7 +70,7 @@ Preparation * clean up current directory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.5/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.6/ * Only differences should be files that you don't want distributed. @@ -83,7 +83,7 @@ Preparation * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.5.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.6.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -106,7 +106,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.5 + git tag -s 0.9.6 Pre Release =========== diff --git a/fail2ban/version.py b/fail2ban/version.py index 57aa3c28..194918d7 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.5" +version = "0.9.6" From 3605155978efc95a2a44337645994dafc2f2b366 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Dec 2016 09:36:08 -0500 Subject: [PATCH 015/147] updated man pages --- man/fail2ban-client.1 | 6 +++--- man/fail2ban-regex.1 | 4 ++-- man/fail2ban-server.1 | 6 +++--- man/fail2ban-testcases.1 | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index f67e71ea..ff3e5f77 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-CLIENT "1" "July 2016" "fail2ban-client v0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-CLIENT "1" "December 2016" "fail2ban-client v0.9.6" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.9.5 reads log file that contains password failure report +Fail2Ban v0.9.6 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 1db372cc..1ed2c327 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-REGEX "1" "July 2016" "fail2ban-regex 0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-REGEX "1" "December 2016" "fail2ban-regex 0.9.6" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 96f0c3e5..5278302c 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-SERVER "1" "July 2016" "fail2ban-server v0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-SERVER "1" "December 2016" "fail2ban-server v0.9.6" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.9.5 reads log file that contains password failure report +Fail2Ban v0.9.6 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 1c2f1a8e..658555ac 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH FAIL2BAN-TESTCASES "1" "July 2016" "fail2ban-testcases 0.9.5" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. +.TH FAIL2BAN-TESTCASES "1" "December 2016" "fail2ban-testcases 0.9.6" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From 4a1fd888f030a6a329e2a3fdf51130b8ee76d570 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 11 Dec 2016 00:49:09 -0500 Subject: [PATCH 016/147] Carry on development --- ChangeLog | 17 ++++++++++++++++- README.md | 2 +- RELEASE | 2 +- fail2ban/version.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0852a360..b9fadce5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,13 +6,28 @@ Fail2Ban: Changelog =================== -ver. 0.9.6 (2016/12/10) - stretch-is-coming + +ver. 0.9.x (2016/??/??) - wanna-be-released ----------- 0.9.x line is no longer heavily developed. If you are interested in new features (e.g. IPv6 support), please consider 0.10 branch and its releases. +### Fixes + +### New Features +* New Actions: + +* New Filters: + + +### Enhancements + + +ver. 0.9.6 (2016/12/10) - stretch-is-coming +----------- + ### Fixes * Misleading add resp. enable of (already available) jail in database, that induced a subsequent error: last position of log file will be never retrieved (gh-795) diff --git a/README.md b/README.md index 998e43e2..3f0e3253 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.6 2016/12/10 + v0.9.6.dev0 2016/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/RELEASE b/RELEASE index b879fa20..c4f62d7a 100644 --- a/RELEASE +++ b/RELEASE @@ -190,7 +190,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.7 (2016/XX/XXX) - wanna-be-released + ver. 0.9.8 (2016/XX/XXX) - wanna-be-released ----------- ### Fixes diff --git a/fail2ban/version.py b/fail2ban/version.py index 194918d7..c2a8dc59 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.6" +version = "0.9.6.dev0" From 31a1560eaae663cb3b740829e3262198a909d1c4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 11 Dec 2016 15:13:11 -0500 Subject: [PATCH 017/147] minor typos (thanks Vincent Lefevre, Debian #847785) --- config/action.d/firewallcmd-rich-logging.conf | 2 +- config/action.d/firewallcmd-rich-rules.conf | 2 +- config/filter.d/assp.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index 1b88c2d9..c4a8b6f7 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -29,7 +29,7 @@ actioncheck = # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' port port='' protocol='' log prefix='f2b-' level='' limit value='/m' " # service name example: # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' service name='' log prefix='f2b-' level='' limit value='/m' " -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp +# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index 4e39df54..5bf10b03 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -27,7 +27,7 @@ actioncheck = # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' port port='' protocol='' " # service name example: # firewall-cmd --zone= --add-rich-rule="rule family='ipv4' source address='' service name='' " -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges seperated by a comma or space for an example: http, https, 22-60, 18 smtp +# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='' port port='$p' protocol='' "; done diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index 278e25cb..ddf18f33 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -8,7 +8,7 @@ # [Definition] -# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibilty reasons. +# Note: First three failregex matches below are for ASSP V1 with the remaining being designed for V2. Deleting the V1 regex is recommended but I left it in for compatibility reasons. __assp_actions = (?:dropping|refusing) From 55e107310fd0697d583aa8971e2b1db48e49c5ea Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sat, 7 Jan 2017 14:24:54 +0200 Subject: [PATCH 018/147] Added config for AbuseIPDB, ony tested on Ubuntu 16.04 --- THANKS | 2 + config/action.d/abuseipdb.conf | 74 ++++++++++++++++++++++++++++++++++ config/jail.conf | 21 +++++++++- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 config/action.d/abuseipdb.conf diff --git a/THANKS b/THANKS index 8f746f29..7537cb92 100644 --- a/THANKS +++ b/THANKS @@ -16,6 +16,7 @@ Alexander Koeppe (IPv6 support) Alexandre Perrin (kAworu) Amir Caspi Amy +Andrew James Collett (ajcollett) Andrew St. Jean Andrey G. Grozin Andy Fragen @@ -111,6 +112,7 @@ Sean DuBois Sebastian Arcus Serg G. Brester Sergey Safarov +Shaun C. Sireyessire silviogarbes Stefan Tatschner diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf new file mode 100644 index 00000000..7bd3f68a --- /dev/null +++ b/config/action.d/abuseipdb.conf @@ -0,0 +1,74 @@ +# Fail2ban configuration file +# +# Action to report IP address to abuseipdb.com +# You must sign up to obtain an API key from abuseipdb.com. +# +# IMPORTANT: +# +# Reporting an IP of abuse is a serious complaint. Make sure that it is +# serious. Fail2ban developers and network owners recommend you only use this +# action for: +# * The recidive where the IP has been banned multiple times +# * Where maxretry has been set quite high, beyond the normal user typing +# password incorrectly. +# * For filters that have a low likelihood of receiving human errors +# +# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# Added to fail2ban by Andrew James Collett (ajcollett) + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# +# ** IMPORTANT! ** +# +# By default, this posts directly to AbuseIPDB's API, unfortunately +# this results in a lot of backslashes/escapes appearing in the +# reports. This also may include info like your hostname. +# If you have your own web server with PHP available, you can +# use my (Shaun's) helper PHP script by commenting out the first #actionban +# line below, uncommenting the second one, and pointing the URL at +# wherever you install the helper script. For the PHP helper script, see +# +# +# --ciphers ecdhe_ecdsa_aes_256_sha is used to workaround a +# "NSS error -12286" from curl as it attempts to connect using +# SSLv3. See https://www.centos.org/forums/viewtopic.php?t=52732 +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = curl --fail --ciphers ecdhe_ecdsa_aes_256_sha --data 'key=' --data-urlencode 'comment=' --data 'ip=' --data 'category=' "https://www.abuseipdb.com/report/json" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] +# Option: abuseipdb_apikey +# Notes Your API key from abuseipdb.com +# Values: STRING Default: None +abuseipdb_apikey = diff --git a/config/jail.conf b/config/jail.conf index b7c927e2..f5d24621 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API -# +# # See the IMPORTANT note in action.d/blocklist_de.conf for when to # use this action. Create a file jail.d/blocklist_de.local containing # [Init] @@ -206,6 +206,23 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag # Report ban via badips.com (uses action.d/badips.conf for reporting only) # action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +# Actions to report to abuseipdb.com via API. +# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# If you want cleaner reports that ensure no user data see the helper script at the above IP +# NOTE: These reports may include sensitive Info. + +# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf +action_abuseipdb_fraud = abuseipdb[abuseipdb_category="3"] +action_abuseipdb_ddos = abuseipdb[abuseipdb_category="4"] +action_abuseipdb_proxy = abuseipdb[abuseipdb_category="9"] +action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"] +action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"] +action_abuseipdb_blogspam = abuseipdb[abuseipdb_category="12"] +action_abuseipdb_portscan = abuseipdb[abuseipdb_category="14"] +action_abuseipdb_hack = abuseipdb[abuseipdb_category="15"] +action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"] +action_abuseipdb_spoofing = abuseipdb[abuseipdb_category="17"] +action_abuseipdb_sshbrute = abuseipdb[abuseipdb_category="18"] # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -330,7 +347,7 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s -# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` # and define `limit_req` and `limit_req_zone` as described in nginx documentation # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html # or for example see in 'config/filter.d/nginx-limit-req.conf' From 1c41390f7caed9a9e6712c83d80774bf7bf1c14f Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:26:11 +0200 Subject: [PATCH 019/147] Restructured the way the catagories work. Jail.conf is cleaner and abuseipdb.conf is more flexible. --- config/action.d/abuseipdb.conf | 19 +++++++++++++++++++ config/jail.conf | 28 ++++++++++++---------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index 7bd3f68a..3bcfb065 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -16,6 +16,23 @@ # Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) +## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] +# ID Title Description +# 3 Fraud Orders +# 4 DDoS Attack +# 9 Open Proxy +# 10 Web Spam +# 11 Email Spam +# 14 Port Scan +# 18 Brute-Force +# 19 Bad Web Bot +# 20 Exploited Host +# 21 Web App Attack +# 22 SSH Secure Shell (SSH) abuse. Use this category in combination with more specific categories. +# 23 IoT Targeted +# See https://abuseipdb.com/categories for more descriptions + [Definition] # Option: actionstart @@ -71,4 +88,6 @@ actionunban = # Option: abuseipdb_apikey # Notes Your API key from abuseipdb.com # Values: STRING Default: None +# Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. +# You will need to set the catagory in the action call. abuseipdb_apikey = diff --git a/config/jail.conf b/config/jail.conf index f5d24621..7ba343fb 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -206,23 +206,19 @@ action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", ag # Report ban via badips.com (uses action.d/badips.conf for reporting only) # action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] -# Actions to report to abuseipdb.com via API. -# See action.d/abuseipdb.conf and https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban -# If you want cleaner reports that ensure no user data see the helper script at the above IP +# Report ban via abuseipdb.com. +# +# See action.d/abuseipdb.conf and +# https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban +# # NOTE: These reports may include sensitive Info. - -# IMPORTANT: Register for abuseipdb [https://www.abuseipdb.com], get, and set the api key at the bottom of action.d/abuseipdb.conf -action_abuseipdb_fraud = abuseipdb[abuseipdb_category="3"] -action_abuseipdb_ddos = abuseipdb[abuseipdb_category="4"] -action_abuseipdb_proxy = abuseipdb[abuseipdb_category="9"] -action_abuseipdb_forumspam = abuseipdb[abuseipdb_category="10"] -action_abuseipdb_emailspam = abuseipdb[abuseipdb_category="11"] -action_abuseipdb_blogspam = abuseipdb[abuseipdb_category="12"] -action_abuseipdb_portscan = abuseipdb[abuseipdb_category="14"] -action_abuseipdb_hack = abuseipdb[abuseipdb_category="15"] -action_abuseipdb_sqlinject = abuseipdb[abuseipdb_category="16"] -action_abuseipdb_spoofing = abuseipdb[abuseipdb_category="17"] -action_abuseipdb_sshbrute = abuseipdb[abuseipdb_category="18"] +# If you want cleaner reports that ensure no user data see the helper script at the above website. +# +# IMPORTANT: This action relies on a api_key being added to the above action conf, +# and the appropriate catagories set. +# Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] +# See action.d/abuseipdb.conf for catagories +action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local From b35391e768c4a476cc176959bad1571ccfd40708 Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:30:00 +0200 Subject: [PATCH 020/147] Update jail.conf Fixing spacing --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7ba343fb..09675904 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -185,7 +185,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API -# +# # See the IMPORTANT note in action.d/blocklist_de.conf for when to # use this action. Create a file jail.d/blocklist_de.local containing # [Init] @@ -330,7 +330,7 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 - + [openhab-auth] filter = openhab From 10d61e077942c25ac5a026a6c6da2b3d2bb6ee3f Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:39:12 +0200 Subject: [PATCH 021/147] Fixed the spaces again --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 09675904..31f0c2be 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -330,7 +330,7 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 - + [openhab-auth] filter = openhab From 3991f51f30f10a1086f0999089aa25c8fedb653d Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:45:35 +0200 Subject: [PATCH 022/147] Update jail.conf Sigh, added a space back that I somehow missed in Vim, despite it being a rebase... --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 31f0c2be..560136a6 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -343,7 +343,7 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s -# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` # and define `limit_req` and `limit_req_zone` as described in nginx documentation # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html # or for example see in 'config/filter.d/nginx-limit-req.conf' From 18d09b6d8ef6c72b83fd99f81832ffe2e7c2ce8d Mon Sep 17 00:00:00 2001 From: Andrew James Collett Date: Sun, 8 Jan 2017 09:50:58 +0200 Subject: [PATCH 023/147] Updated changelog. --- ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7964a9ba..5f5496c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -83,6 +83,10 @@ TODO: implementing of options resp. other tasks from PR #1346 if configuration is clean (fails by wrong configured jails if option `-t` specified) * New command action parameter `actionrepair` - command executed in order to restore sane environment in error case of `actioncheck`. +* Reporting via abuseipdb.com: + - Bans can now be reported to abuseipdb + - Catagories must be set in the config + - Relevant log lines included in report ### Enhancements * Huge increasing of fail2ban performance and especially test-cases performance (see gh-1109) From fb27d9ce83191ecf2887b84453f7ea0dae9e4c8f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 10:59:53 +0100 Subject: [PATCH 024/147] fail2ban-regex: fix for systemd-journal (see gh-1657) --- fail2ban/client/fail2banregex.py | 31 ++++++++++++++----------------- fail2ban/server/filtersystemd.py | 8 ++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index ff55c417..d111e09c 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -41,12 +41,12 @@ from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError try: # pragma: no cover - from systemd import journal from ..server.filtersystemd import FilterSystemd except ImportError: - journal = None + FilterSystemd = None from ..version import version +from .jailreader import JailReader from .filterreader import FilterReader from ..server.filter import Filter, FileContainer from ..server.failregex import RegexException @@ -80,7 +80,7 @@ def pprint_list(l, header=None): s = '' output( s + "| " + "\n| ".join(l) + '\n`-' ) -def journal_lines_gen(myjournal): # pragma: no cover +def journal_lines_gen(flt, myjournal): # pragma: no cover while True: try: entry = myjournal.get_next() @@ -88,7 +88,7 @@ def journal_lines_gen(myjournal): # pragma: no cover continue if not entry: break - yield FilterSystemd.formatJournalEntry(entry) + yield flt.formatJournalEntry(entry) def get_opt_parser(): # use module docstring for help output @@ -524,25 +524,22 @@ class Fail2banRegex(object): except IOError as e: output( e ) return False - elif cmd_log == "systemd-journal": # pragma: no cover - if not journal: + elif cmd_log.startswith("systemd-journal"): # pragma: no cover + if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + output( "Use systemd journal" ) + output( "Use encoding : %s" % self._encoding ) + backend, beArgs = JailReader.extractOptions(cmd_log) + flt = FilterSystemd(None, **beArgs) + flt.setLogEncoding(self._encoding) + myjournal = flt.getJournalReader() journalmatch = self._journalmatch self.setDatePattern(None) if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) - return False + flt.addJournalMatch(journalmatch) output( "Use journal match : %s" % " ".join(journalmatch) ) - test_lines = journal_lines_gen(myjournal) + test_lines = journal_lines_gen(flt, myjournal) else: output( "Use single line : %s" % shortstr(cmd_log) ) test_lines = [ cmd_log ] diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 0d720a5e..d858142b 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -178,6 +178,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalMatch(self): return self.__matches + ## + # Get journal reader + # + # @return journal reader + + def getJournalReader(self): + return self.__journal + ## # Format journal log entry into syslog style # From 2009f1c4346597dcc4fd27151d220ea4a7806fdd Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 10:59:53 +0100 Subject: [PATCH 025/147] fail2ban-regex: fix for systemd-journal (see gh-1657) --- fail2ban/client/fail2banregex.py | 31 ++++++++++++++----------------- fail2ban/server/filtersystemd.py | 8 ++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 71f50955..13fa35d9 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -43,12 +43,12 @@ from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError try: - from systemd import journal from ..server.filtersystemd import FilterSystemd except ImportError: - journal = None + FilterSystemd = None from ..version import version +from .jailreader import JailReader from .filterreader import FilterReader from ..server.filter import Filter, FileContainer from ..server.failregex import RegexException @@ -82,7 +82,7 @@ def pprint_list(l, header=None): s = '' output( s + "| " + "\n| ".join(l) + '\n`-' ) -def journal_lines_gen(myjournal): +def journal_lines_gen(flt, myjournal): # pragma: no cover while True: try: entry = myjournal.get_next() @@ -90,7 +90,7 @@ def journal_lines_gen(myjournal): continue if not entry: break - yield FilterSystemd.formatJournalEntry(entry) + yield flt.formatJournalEntry(entry) def get_opt_parser(): # use module docstring for help output @@ -513,25 +513,22 @@ class Fail2banRegex(object): except IOError as e: output( e ) return False - elif cmd_log == "systemd-journal": # pragma: no cover - if not journal: + elif cmd_log.startswith("systemd-journal"): # pragma: no cover + if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + output( "Use systemd journal" ) + output( "Use encoding : %s" % self.encoding ) + backend, beArgs = JailReader.extractOptions(cmd_log) + flt = FilterSystemd(None, **beArgs) + flt.setLogEncoding(self.encoding) + myjournal = flt.getJournalReader() journalmatch = self._journalmatch self.setDatePattern(None) if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) - return False + flt.addJournalMatch(journalmatch) output( "Use journal match : %s" % " ".join(journalmatch) ) - test_lines = journal_lines_gen(myjournal) + test_lines = journal_lines_gen(flt, myjournal) else: output( "Use single line : %s" % shortstr(cmd_log) ) test_lines = [ cmd_log ] diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 3023155c..908112a7 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -174,6 +174,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover v = Filter.uni_decode(x, self.getLogEncoding()) return v + ## + # Get journal reader + # + # @return journal reader + + def getJournalReader(self): + return self.__journal + ## # Format journal log entry into syslog style # From f8d35a7c9c8b783b6cc40ef13c0462a5ad8dcaf0 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 11:16:17 +0100 Subject: [PATCH 026/147] changelog entry --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b9fadce5..0fd91b70 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ new features (e.g. IPv6 support), please consider 0.10 branch and its releases. ### Fixes +* Fixed a systemd-journal handling in fail2ban-regex (gh-1657) ### New Features * New Actions: From a9523aefbb9b179e6a18ae163d44dc9f9b4b7695 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 12:51:11 +0100 Subject: [PATCH 027/147] sshd.conf: fixed non-anchored part of regex (misleading match of colon inside IPv6 address instead of `: ` in the reason-part by missing space). --- ChangeLog | 4 ++++ config/filter.d/sshd.conf | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0fd91b70..126bd2e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,10 @@ releases. ### Fixes * Fixed a systemd-journal handling in fail2ban-regex (gh-1657) +* filter.d/sshd.conf + - Fixed non-anchored part of failregex (misleading match of colon inside + IPv6 address instead of `: ` in the reason-part by missing space, gh-1658) + (0.10th resp. IPv6 relevant only, amend for gh-1479) ### New Features * New Actions: diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 35cd8754..d5a66cc8 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -20,7 +20,7 @@ _daemon = sshd failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*$ ^%(__prefix_line)s(?:error: PAM: )?User not known to the underlying authentication module for .* from \s*$ - ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user):|(?:(?:(?! from ).)*)$) + ^%(__prefix_line)sFailed \S+ for (?Pinvalid user )?(?P(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)) from (?: port \d+)?(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM \s*$ ^%(__prefix_line)s[iI](?:llegal|nvalid) user .*? from (?: port \d+)?\s*$ ^%(__prefix_line)sUser .+ from not allowed because not listed in AllowUsers\s*$ From bf872213bdad3228510a977740eaac3aac2d612a Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Jan 2017 13:48:17 +0100 Subject: [PATCH 028/147] amend for 7019640eb3d1f901e52b4c00cb6d029e82fdda17 (fix-gh-1658): sshd test-cases extended with IPv6 to cover this fix --- fail2ban/tests/files/logs/sshd | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 15ec4274..0879a2da 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -9,8 +9,14 @@ May 29 20:56:56 imago sshd[28732]: error: PAM: Authentication failure for test-i #2 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.69" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 50273 ssh2 # failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "194.117.26.70" } Feb 25 14:34:10 belka sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345 +# failJSON: { "time": "2005-02-25T14:34:10", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:10 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 12345 +# failJSON: { "time": "2005-02-25T14:34:11", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +Feb 25 14:34:11 belka sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 #3 # failJSON: { "time": "2005-01-05T01:31:41", "match": true , "host": "1.2.3.4" } @@ -118,6 +124,8 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +# failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } +Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -128,6 +136,8 @@ Nov 11 08:04:52 redbamboo sshd[2737]: Failed password for invalid user test from # failJSON: { "time": "2005-07-05T18:22:44", "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } Jul 05 18:22:44 mercury sshd[4669]: Failed publickey for graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI +# failJSON: { "time": "2005-07-05T18:22:45", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Failed publickey for ..." } +Jul 05 18:22:45 mercury sshd[4670]: Failed publickey for graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI # failJSON: { "match": false } Nov 23 21:50:19 sshd[8148]: Disconnecting: Too many authentication failures for root [preauth] From 6f190b6e611cfee8796f6720e8df17c73e0e63ee Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Wed, 11 Jan 2017 19:04:33 +0100 Subject: [PATCH 029/147] readme.md: added IPv6 launch logo for 0.10th branch Closes gh-1647 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index be0be636..ee654acb 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. + + | Since v0.10 fail2ban supports the matching of the IPv6 addresses. +------|------ This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs are available in fail2ban(1) manpage and on the website http://www.fail2ban.org From 4a65e069e1437d165b3265d6caba623e02327462 Mon Sep 17 00:00:00 2001 From: oliverdorn Date: Thu, 12 Jan 2017 22:07:46 +0100 Subject: [PATCH 030/147] Solution for issue #1665 Solves the issue of authentic GoogleBots being banned by apache-fakegooglebots. --- files/debian-initd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/debian-initd b/files/debian-initd index 0d2bed9a..d6660215 100755 --- a/files/debian-initd +++ b/files/debian-initd @@ -22,7 +22,7 @@ # rename this file: (sudo) mv /etc/init.d/fail2ban.init /etc/init.d/fail2ban # same with the logrotate file: (sudo) mv /etc/logrotate.d/fail2ban.logrotate /etc/logrotate.d/fail2ban # -PATH=/usr/sbin:/usr/bin:/sbin:/bin +PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin DESC="authentication failure monitor" NAME=fail2ban From ee3c787cc68f6d043c8702c8f53a566277e2ee02 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 13 Jan 2017 19:06:17 +0100 Subject: [PATCH 031/147] Recognize restored (from database) tickets after restart (tell action restored state of the ticket); Prevent executing of several actions (e.g. mail, send-mail etc) on restart (bans were already notified). Test cases extended (smtp and by restart in ServerReloadTest). Closes gh-1141 Closes gh-921 --- config/action.d/complain.conf | 3 +- config/action.d/dshield.conf | 10 ++++- config/action.d/helpers-common.conf | 3 ++ config/action.d/mail-buffered.conf | 8 +++- config/action.d/mail-whois-lines.conf | 4 +- config/action.d/mail-whois.conf | 4 +- config/action.d/mail.conf | 7 +++- config/action.d/sendmail-buffered.conf | 4 +- config/action.d/sendmail-geoip-lines.conf | 3 +- .../sendmail-whois-ipjailmatches.conf | 4 +- config/action.d/sendmail-whois-ipmatches.conf | 4 +- config/action.d/sendmail-whois-lines.conf | 3 +- config/action.d/sendmail-whois-matches.conf | 4 +- config/action.d/sendmail-whois.conf | 4 +- config/action.d/sendmail.conf | 4 +- config/action.d/smtp.py | 2 + config/action.d/xarf-login-attack.conf | 7 +++- fail2ban/server/action.py | 4 +- fail2ban/server/actions.py | 4 ++ fail2ban/tests/action_d/test_smtp.py | 15 ++++++- fail2ban/tests/fail2banclienttestcase.py | 42 +++++++++++++++---- 21 files changed, 114 insertions(+), 29 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index e4ceb35f..d0156a44 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -58,7 +58,8 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = oifs=${IFS}; +actionban = %(_bypass_if_restored)s + oifs=${IFS}; IFS=.; SEP_IP=( ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=,; ADDRESSES=$(echo $ADDRESSES) IFS=${oifs} diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index a0041986..35eaa3be 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -26,6 +26,10 @@ # configure how often the buffer is flushed). # +[INCLUDES] + +before = helpers-common.conf + [Definition] # Option: actionstart @@ -64,7 +68,8 @@ actioncheck = # few seconds out, are incorrect. See # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047 # -actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'` +actionban = %(_bypass_if_restored)s + TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'` DATETIME="`perl -e '@t=localtime(