From 68d829c1dde3979073414623c9dd4890dd5ee72c Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Tue, 14 Feb 2017 18:12:06 +0800 Subject: [PATCH 01/43] Add a path configuration for Arch Linux --- MANIFEST | 1 + config/paths-arch.conf | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 config/paths-arch.conf diff --git a/MANIFEST b/MANIFEST index f03f185b..6832f8cf 100644 --- a/MANIFEST +++ b/MANIFEST @@ -148,6 +148,7 @@ config/filter.d/webmin-auth.conf config/filter.d/wuftpd.conf config/filter.d/xinetd-fail.conf config/jail.conf +config/paths-arch.conf config/paths-common.conf config/paths-debian.conf config/paths-fedora.conf diff --git a/config/paths-arch.conf b/config/paths-arch.conf new file mode 100644 index 00000000..270ce65d --- /dev/null +++ b/config/paths-arch.conf @@ -0,0 +1,32 @@ +# Arch + +[INCLUDES] + +before = paths-common.conf + +after = paths-overrides.local + + +[DEFAULT] + +apache_error_log = /var/log/httpd/*error_log + +apache_access_log = /var/log/httpd/*access_log + +exim_main_log = /var/log/exim/main.log + +mysql_log = /var/log/mariadb/mariadb.log + /var/log/mysqld.log + +roundcube_errors_log = /var/log/roundcubemail/errors + +# These services will log to the journal via syslog, so use the journal by +# default. +syslog_backend = systemd +sshd_backend = systemd +dropbear_backend = systemd +proftpd_backend = systemd +pureftpd_backend = systemd +wuftpd_backend = systemd +postfix_backend = systemd +dovecot_backend = systemd From c546f852070377455985169d7efaadd65bb85e34 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 7 May 2017 13:02:32 +0200 Subject: [PATCH 02/43] filter.d/exim.conf: cherry-picked from 0.10, match complex time like `D=2m42s` (closes gh-1766) --- config/filter.d/exim-common.conf | 4 +++- config/filter.d/exim.conf | 6 +++--- fail2ban/tests/files/logs/exim | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index 0e1b74fa..b3b25750 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -9,7 +9,9 @@ after = exim-common.local [Definition] -host_info = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?\[\](?::\d+)? (?:I=\[\S+\](:\d+)? )?(?:U=\S+ )?(?:P=e?smtp )? +host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)? +host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s +host_info = %(host_info_pre)s\[\]%(host_info_suf)s pid = (?: \[\d+\])? # DEV Notes: diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 27d73426..c5352008 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -15,12 +15,12 @@ before = exim-common.conf failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ - ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d+s(?: C=\S*)?\s*$ - ^%(pid)s ([\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ + ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S+s(?: C=\S*)?\s*$ + ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 44d9e34b..144335d7 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -63,6 +63,8 @@ # failJSON: { "time": "2016-04-01T11:08:39", "match": true , "host": "192.0.2.1" } 2016-04-01 11:08:39 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.1]:1418 I=[172.89.0.6]:25 D=34s C=EHLO,AUTH +# failJSON: { "time": "2016-04-01T11:08:40", "match": true , "host": "192.0.2.2" } +2016-04-01 11:08:40 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.2]:1418 I=[172.89.0.6]:25 D=2m42s C=QUIT # failJSON: { "time": "2016-04-01T11:09:21", "match": true , "host": "192.0.2.1" } 2016-04-01 11:09:21 [18648] SMTP protocol error in "AUTH LOGIN" H=host.example.com (SERVER) [192.0.2.1]:4692 I=[172.89.0.6]:25 AUTH command used when not advertised # failJSON: { "time": "2016-03-27T16:48:48", "match": true , "host": "192.0.2.1" } From 3d64d705f30c31bfb3a45468b5ee18e7c3dbc18f Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 7 May 2017 13:17:43 +0200 Subject: [PATCH 03/43] try to fix travis integration of pypy3: setuptools recently dropped support for Python 3.0 - 3.2, but old pypy3 based on Python 3.2.5 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9ef607da..40376075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,9 @@ python: # - 3.2 - 3.3 - 3.4 - - pypy3 + # disabled since setuptools dropped support for Python 3.0 - 3.2 + # - pypy3 + - pypy3.3-5.2-alpha1 before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi From 0600d51511c348a0a3e2fc20fa823fbe235df6c9 Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 7 May 2017 13:53:06 +0200 Subject: [PATCH 04/43] filter.d/exim.conf: added new reason for "rejected RCPT" regex: Unrouteable address --- config/filter.d/exim.conf | 2 +- fail2ban/tests/files/logs/exim | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index c5352008..7848fe00 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -15,7 +15,7 @@ before = exim-common.conf failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user)\s*$ + ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$ ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$ ^%(pid)s SMTP protocol error in "AUTH \S*(?: \S*)?" %(host_info)sAUTH command used when not advertised\s*$ diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 144335d7..326cdf28 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -72,3 +72,5 @@ # failJSON: { "time": "2017-04-23T22:45:59", "match": true , "host": "192.0.2.2", "desc": "optional part (...)" } 2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com) +# failJSON: { "time": "2017-05-01T07:42:42", "match": true , "host": "192.0.2.3", "desc": "rejected RCPT - Unrouteable address" } +2017-05-01 07:42:42 H=some.rev.dns.if.found (the.connector.reports.this.name) [192.0.2.3] F= rejected RCPT : Unrouteable address From bea3a62a37a6bbfdf65057d6769147a2bfe7cd5f Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 7 May 2017 13:56:24 +0200 Subject: [PATCH 05/43] update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 7d5929d7..c60ff660 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,8 @@ releases. - Fixed filenames for apache and nginx log files (gh-1667) * filter.d/exim.conf - optional part `(...)` after host-name before `[IP]` (gh-1751) + - new reason "Unrouteable address" for "rejected RCPT" regex (gh-1762) + - match of complex time like `D=2m42s` in regex "no MAIL in SMTP connection" (gh-1766) * filter.d/sshd.conf - new aggressive rules (gh-864): - Connection reset by peer (multi-line rule during authorization process) From 2a4dafce5b8b583f2a914dd38d6add8bd4addd16 Mon Sep 17 00:00:00 2001 From: Sander Hoentjen Date: Mon, 8 May 2017 14:11:07 +0200 Subject: [PATCH 06/43] don't call setLogTarget from setSyslogSocket when holding the lock If we do, setLogTarget will wait for the lock, but it will never be released --- fail2ban/server/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index facbe393..56367ed7 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -638,9 +638,9 @@ class Server: if self.__syslogSocket == syslogsocket: return True self.__syslogSocket = syslogsocket - # Conditionally reload, logtarget depends on socket path when SYSLOG - return self.__logTarget != "SYSLOG"\ - or self.setLogTarget(self.__logTarget) + # Conditionally reload, logtarget depends on socket path when SYSLOG + return self.__logTarget != "SYSLOG"\ + or self.setLogTarget(self.__logTarget) def getLogTarget(self): with self.__loggingLock: From 663d526d7481228a7da568e2baf8b08e2924a3e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 10 May 2017 21:31:09 -0400 Subject: [PATCH 07/43] Added newly added files to MANIFEST --- MANIFEST | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MANIFEST b/MANIFEST index b12e3163..e91ccff0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -37,6 +37,7 @@ config/action.d/mail-whois-common.conf config/action.d/mail-whois.conf config/action.d/mail-whois-lines.conf config/action.d/mynetwatchman.conf +config/action.d/netscaler.conf config/action.d/nftables-allports.conf config/action.d/nftables-common.conf config/action.d/nftables-multiport.conf @@ -83,6 +84,7 @@ config/filter.d/courier-auth.conf config/filter.d/courier-smtp.conf config/filter.d/cyrus-imap.conf config/filter.d/directadmin.conf +config/filter.d/domino-smtp.conf config/filter.d/dovecot.conf config/filter.d/dropbear.conf config/filter.d/drupal-auth.conf @@ -138,6 +140,7 @@ config/filter.d/sogo-auth.conf config/filter.d/solid-pop3d.conf config/filter.d/squid.conf config/filter.d/squirrelmail.conf +config/filter.d/sshd-aggressive.conf config/filter.d/sshd.conf config/filter.d/sshd-ddos.conf config/filter.d/stunnel.conf @@ -278,6 +281,7 @@ fail2ban/tests/files/logs/courier-auth fail2ban/tests/files/logs/courier-smtp fail2ban/tests/files/logs/cyrus-imap fail2ban/tests/files/logs/directadmin +fail2ban/tests/files/logs/domino-smtp fail2ban/tests/files/logs/dovecot fail2ban/tests/files/logs/dropbear fail2ban/tests/files/logs/drupal-auth @@ -330,6 +334,7 @@ fail2ban/tests/files/logs/solid-pop3d fail2ban/tests/files/logs/squid fail2ban/tests/files/logs/squirrelmail fail2ban/tests/files/logs/sshd +fail2ban/tests/files/logs/sshd-aggressive fail2ban/tests/files/logs/sshd-ddos fail2ban/tests/files/logs/stunnel fail2ban/tests/files/logs/suhosin From 35280044ffdb6a8166fc6ad5497cfae7032c71a6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 10 May 2017 21:38:57 -0400 Subject: [PATCH 08/43] Preparing for 0.9.7 release --- ChangeLog | 4 ++-- README.md | 2 +- fail2ban/version.py | 2 +- man/fail2ban-client.1 | 4 ++-- man/fail2ban-regex.1 | 2 +- man/fail2ban-server.1 | 4 ++-- man/fail2ban-testcases.1 | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index c60ff660..ec855ab0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,7 +7,7 @@ Fail2Ban: Changelog =================== -ver. 0.9.x (2016/??/??) - wanna-be-released +ver. 0.9.7 (2017/05/11) - awaiting-victory ----------- 0.9.x line is no longer heavily developed. If you are interested in @@ -40,7 +40,7 @@ releases. * filter.d/cyrus-imap.conf - accept entries without login-info resp. hostname before IP address (gh-1707) * Filter tests extended with check of all config-regexp, that contains greedy catch-all - before ``, that is hard-anchored at end or precise sub expression after `` + before ``, that is hard-anchored at end or precise sub expression after `` ### New Features * New Actions: diff --git a/README.md b/README.md index 3f0e3253..89844d1d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.6.dev0 2016/??/?? + v0.9.7 2017/05/11 ## Fail2Ban: ban hosts that cause multiple authentication errors diff --git a/fail2ban/version.py b/fail2ban/version.py index c2a8dc59..e8344517 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.6.dev0" +version = "0.9.7" diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index ff3e5f77..34a9d894 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "December 2016" "fail2ban-client v0.9.6" "User Commands" +.TH FAIL2BAN-CLIENT "1" "May 2017" "fail2ban-client v0.9.7" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.9.6 reads log file that contains password failure report +Fail2Ban v0.9.7 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 1ed2c327..d117ee38 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "December 2016" "fail2ban-regex 0.9.6" "User Commands" +.TH FAIL2BAN-REGEX "1" "May 2017" "fail2ban-regex 0.9.7" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 5278302c..b71f6b52 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,12 +1,12 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "December 2016" "fail2ban-server v0.9.6" "User Commands" +.TH FAIL2BAN-SERVER "1" "May 2017" "fail2ban-server v0.9.7" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.9.6 reads log file that contains password failure report +Fail2Ban v0.9.7 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .PP Only use this command for debugging purpose. Start the server with diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 658555ac..3d250f5c 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "December 2016" "fail2ban-testcases 0.9.6" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "May 2017" "fail2ban-testcases 0.9.7" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS From a5cdb9c977fb640df2a12b3110235c2472042456 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 May 2017 09:10:45 +0200 Subject: [PATCH 09/43] exim test cases extended: cover short form of the logging (without session-id, gh-1771) --- fail2ban/tests/files/logs/exim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 326cdf28..8f3ac1bb 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -60,6 +60,8 @@ 2016-03-21 04:07:49 [25874] 1ahr79-0006jK-G9 SMTP connection from (voyeur.webair.com) [174.137.147.204]:44884 I=[172.89.0.6]:25 closed by DROP in ACL # failJSON: { "time": "2016-03-21T04:33:13", "match": true , "host": "206.214.71.53" } 2016-03-21 04:33:13 [26074] 1ahrVl-0006mY-79 SMTP connection from riveruse.com [206.214.71.53]:39865 I=[172.89.0.6]:25 closed by DROP in ACL +# failJSON: { "time": "2016-03-21T04:33:14", "match": true , "host": "192.0.2.33", "desc": "short form without optional session-id" } +2016-03-21 04:33:14 SMTP connection from (some.domain) [192.0.2.33] closed by DROP in ACL # failJSON: { "time": "2016-04-01T11:08:39", "match": true , "host": "192.0.2.1" } 2016-04-01 11:08:39 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.1]:1418 I=[172.89.0.6]:25 D=34s C=EHLO,AUTH From 407b2ea936b836cbe9cdb59fc8e40fcce6d2959e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 11 May 2017 11:17:27 -0400 Subject: [PATCH 10/43] life is going on --- ChangeLog | 15 +++++++++++++-- README.md | 6 +++--- RELEASE | 14 +++++++------- fail2ban/version.py | 2 +- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index ec855ab0..8e7ebdb0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,14 +6,25 @@ Fail2Ban: Changelog =================== - -ver. 0.9.7 (2017/05/11) - awaiting-victory +ver. 0.9.8 (2016/XX/XXX) - wanna-be-released ----------- 0.9.x line is no longer heavily developed. If you are interested in new features (e.g. IPv6 support), please consider 0.10 branch and its releases. + +### Fixes + +### New Features + +### Enhancements + + + +ver. 0.9.7 (2017/05/11) - awaiting-victory +----------- + ### Fixes * Fixed a systemd-journal handling in fail2ban-regex (gh-1657) * filter.d/sshd.conf diff --git a/README.md b/README.md index 89844d1d..1138a3ac 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.7 2017/05/11 + v0.9.7.dev0 2017/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -39,8 +39,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.6.tar.bz2 - cd fail2ban-0.9.6 + tar xvfj fail2ban-0.9.7.tar.bz2 + cd fail2ban-0.9.7 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index c4f62d7a..0a3e0497 100644 --- a/RELEASE +++ b/RELEASE @@ -53,7 +53,7 @@ Preparation or an alternative for comparison with previous release - git diff 0.9.6 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST + git diff 0.9.7 | grep -B2 'index 0000000..' | grep -B1 'new file mode' | sed -n -e '/^diff /s,.* b/,,gp' >> MANIFEST sort MANIFEST | uniq | sponge MANIFEST * Run:: @@ -66,24 +66,24 @@ Preparation * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: - tar -C /tmp -jxf dist/fail2ban-0.9.6.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.9.7.tar.bz2 * clean up current directory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.6/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.7/ * Only differences should be files that you don't want distributed. * Ensure the tests work from the tarball:: - cd /tmp/fail2ban-0.9.6/ && bin/fail2ban-testcases + cd /tmp/fail2ban-0.9.7/ && bin/fail2ban-testcases * Add/finalize the corresponding entry in the ChangeLog * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.6.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.7.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -106,7 +106,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.6 + git tag -s 0.9.8 Pre Release =========== @@ -190,7 +190,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.8 (2016/XX/XXX) - wanna-be-released + ver. 0.9.9 (2016/XX/XXX) - wanna-be-released ----------- ### Fixes diff --git a/fail2ban/version.py b/fail2ban/version.py index e8344517..3f4edc54 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.7" +version = "0.9.7.dev0" From e340d0d2b285dbc7b46c10beed9d1ea1ba04b09b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 May 2017 16:51:08 +0200 Subject: [PATCH 11/43] Fixed detection of directory-based log-rotation of pyinotify backend. If directory moved and the target is not watched path, so the monitoring of it could not be continued. Now fixed with pending files await a monitoring if there (resp. its directories) appears again (respawn). Closes gh-1769 --- fail2ban/server/filter.py | 3 +- fail2ban/server/filterpyinotify.py | 92 +++++++++++++++++++++++++++--- fail2ban/tests/filtertestcase.py | 46 ++++++++++++--- 3 files changed, 124 insertions(+), 17 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 526f54ea..c88a6c6d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -895,7 +895,8 @@ class FileFilter(Filter): # see http://python.org/dev/peps/pep-3151/ except IOError as e: logSys.error("Unable to open %s", filename) - logSys.exception(e) + if e.errno != 2: + logSys.exception(e) return False except OSError as e: # pragma: no cover - requires race condition to tigger this logSys.error("Error opening %s", filename) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 73c82099..faf01560 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -25,13 +25,14 @@ __license__ = "GPL" import logging from distutils.version import LooseVersion +import os from os.path import dirname, sep as pathsep import pyinotify from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime +from .mytime import MyTime, time from .utils import Utils from ..helpers import getLogger @@ -52,6 +53,11 @@ except Exception as e: # Gets the instance of the logger. logSys = getLogger(__name__) +# Override pyinotify default logger/init-handler: +def _pyinotify_logger_init(): + return logSys +pyinotify._logger_init = _pyinotify_logger_init +pyinotify.log = logSys ## # Log reader class. @@ -73,6 +79,9 @@ class FilterPyinotify(FileFilter): # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__watches = dict() + self.__pending = dict() + self.__pendingChkTime = 0 + self.__pendingNextTime = 0 logSys.debug("Created FilterPyinotify") def callback(self, event, origin=''): @@ -84,15 +93,36 @@ class FilterPyinotify(FileFilter): logSys.debug("Ignoring creation of directory %s", path) return # check if that is a file we care about - if not path in self.__watches: + if path not in self.__watches: logSys.debug("Ignoring creation of %s we do not monitor", path) return - else: - # we need to substitute the watcher with a new one, so first - # remove old one - self._delFileWatcher(path) - # place a new one - self._addFileWatcher(path) + self._refreshFileWatcher(path) + elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF): + # fix pyinotify behavior with '-unknown-path' (if target not watched also): + if (event.mask & pyinotify.IN_MOVE_SELF and path not in self.__watches and + path.endswith('-unknown-path') + ): + path = path[:-len('-unknown-path')] + # watch was removed for some reasons (log-rotate?): + if not os.path.isfile(path): + for log in self.getLogs(): + logpath = log.getFileName() + if logpath.startswith(path): + # check exists (rotated): + if event.mask & pyinotify.IN_MOVE_SELF or not os.path.isfile(logpath): + self._addPendingFile(logpath, event) + else: + path = logpath + break + if path not in self.__watches: + logSys.debug("Ignoring event of %s we do not monitor", path) + return + if not os.path.isfile(path): + if self.containsLogPath(path): + self._addPendingFile(path, event) + logSys.debug("Ignoring watching/rotation event (%s) for %s", event.maskname, path) + return + self._refreshFileWatcher(path) # do nothing if idle: if self.idle: return @@ -113,6 +143,44 @@ class FilterPyinotify(FileFilter): self.failManager.cleanup(MyTime.time()) self.__modified = False + def _addPendingFile(self, path, event): + if path not in self.__pending: + self.__pending[path] = self.sleeptime / 10; + logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s", + path, event.maskname, event.pathname) + + def _checkPendingFiles(self): + if self.__pending: + ntm = time.time() + if ntm > self.__pendingNextTime: + found = {} + minTime = 60 + for path, retardTM in self.__pending.iteritems(): + if ntm - self.__pendingChkTime > retardTM: + if not os.path.isfile(path): # not found - prolong for next time + if retardTM < 60: retardTM *= 2 + if minTime > retardTM: minTime = retardTM + self.__pending[path] = retardTM + continue + found[path] = 1 + self._refreshFileWatcher(path) + for path in found: + try: + del self.__pending[path] + except KeyError: pass + self.__pendingChkTime = time.time() + self.__pendingNextTime = self.__pendingChkTime + minTime + # process now because we'he missed it in monitoring: + for path in found: + self._process_file(path) + + def _refreshFileWatcher(self, oldPath, newPath=None): + # we need to substitute the watcher with a new one, so first + # remove old one + self._delFileWatcher(oldPath) + # place a new one + self._addFileWatcher(newPath or oldPath) + def _addFileWatcher(self, path): wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY) self.__watches.update(wd) @@ -139,7 +207,9 @@ class FilterPyinotify(FileFilter): if not (path_dir in self.__watches): # we need to watch also the directory for IN_CREATE self.__watches.update( - self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO)) + self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | + pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF | + pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR)) logSys.debug("Added monitor for the parent directory %s", path_dir) self._addFileWatcher(path) @@ -177,6 +247,9 @@ class FilterPyinotify(FileFilter): # slow check events while idle: def __check_events(self, *args, **kwargs): + # check pending files (logrotate ready): + self._checkPendingFiles() + if self.idle: if Utils.wait_for(lambda: not self.active or not self.idle, self.sleeptime * 10, self.sleeptime @@ -209,6 +282,7 @@ class FilterPyinotify(FileFilter): super(FilterPyinotify, self).stop() # Stop the notifier thread self.__notifier.stop() + self.__notifier.stop = lambda *args: 0; # prevent dual stop ## # Wait for exit with cleanup. diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ce665e72..a9b31fd4 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -43,7 +43,7 @@ from ..server.failmanager import FailManagerEmpty from ..server.ipdns import DNSUtils, IPAddr from ..server.mytime import MyTime from ..server.utils import Utils, uni_decode -from .utils import setUpMyTime, tearDownMyTime, mtimesleep, LogCaptureTestCase +from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase from .dummyjail import DummyJail TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -942,17 +942,21 @@ def get_monitor_failures_testcase(Filter_): skip=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def test_move_file(self): - # if we move file into a new location while it has been open already - self.file.close() - self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=14, mode='w') + def _wait4failures(self, count=2): # Poll might need more time self.assertTrue(self.isEmpty(_maxWaitTime(5)), "Queue must be empty but it is not: %s." % (', '.join([str(x) for x in self.jail.queue]))) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) - Utils.wait_for(lambda: self.filter.failManager.getFailTotal() == 2, _maxWaitTime(10)) + Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) + self.assertEqual(self.filter.failManager.getFailTotal(), count) + + def test_move_file(self): + # if we move file into a new location while it has been open already + self.file.close() + self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, + n=14, mode='w') + self._wait4failures() self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... @@ -967,6 +971,34 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) + @with_tmpdir + def test_move_dir(self, tmp): + self.file.close() + self.filter.delLogPath(self.name) + # if we rename parent dir into a new location (simulate directory-base log rotation) + tmpsub1 = os.path.join(tmp, "1") + tmpsub2 = os.path.join(tmp, "2") + os.mkdir(tmpsub1) + self.name = os.path.join(tmpsub1, os.path.basename(self.name)) + os.close(os.open(self.name, os.O_CREAT|os.O_APPEND)); # create empty file + self.filter.addLogPath(self.name, autoSeek=False) + + self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, + skip=12, n=1, mode='w') + self.file.close() + self._wait4failures(1) + + # rotate whole directory: rename directory 1 as 2: + os.rename(tmpsub1, tmpsub2) + os.mkdir(tmpsub1) + self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, + skip=12, n=1, mode='w') + self.file.close() + self._wait4failures(2) + # stop before tmpdir deleted (just prevents many monitor events) + self.filter.stop() + + def _test_move_into_file(self, interim_kill=False): # if we move a new file into the location of an old (monitored) file _copy_lines_between_files(GetFailures.FILENAME_01, self.name, From 7b614a7a15b001e0e0e34765143dd43ea1095139 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 May 2017 22:01:37 +0200 Subject: [PATCH 12/43] differentiate between watched directories and files (refreshing monitoring of files/dirs expected different flags for watcher) --- fail2ban/server/filterpyinotify.py | 168 ++++++++++++++++++----------- 1 file changed, 104 insertions(+), 64 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index faf01560..6394ecef 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -78,7 +78,8 @@ class FilterPyinotify(FileFilter): self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() - self.__watches = dict() + self.__watchFiles = dict() + self.__watchDirs = dict() self.__pending = dict() self.__pendingChkTime = 0 self.__pendingNextTime = 0 @@ -87,45 +88,56 @@ class FilterPyinotify(FileFilter): def callback(self, event, origin=''): logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event) path = event.pathname + # check watching of this path: + isWF = isWD = False + if path in self.__watchDirs: + isWD = True + elif path in self.__watchFiles: + isWF = True + # fix pyinotify behavior with '-unknown-path' (if target not watched also): + if (event.mask & pyinotify.IN_MOVE_SELF and + path.endswith('-unknown-path') and not isWF and not isWD + ): + path = path[:-len('-unknown-path')] + isWD = path in self.__watchDirs + assumeNoDir = False if event.mask & ( pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO ): + # refresh watched dir (may be expected): + if isWD: + self._refreshWatcher(path, isDir=True) + return # skip directories altogether if event.mask & pyinotify.IN_ISDIR: logSys.debug("Ignoring creation of directory %s", path) return # check if that is a file we care about - if path not in self.__watches: + if not isWF: logSys.debug("Ignoring creation of %s we do not monitor", path) return - self._refreshFileWatcher(path) + self._refreshWatcher(path) elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF): - # fix pyinotify behavior with '-unknown-path' (if target not watched also): - if (event.mask & pyinotify.IN_MOVE_SELF and path not in self.__watches and - path.endswith('-unknown-path') - ): - path = path[:-len('-unknown-path')] # watch was removed for some reasons (log-rotate?): - if not os.path.isfile(path): - for log in self.getLogs(): - logpath = log.getFileName() - if logpath.startswith(path): - # check exists (rotated): - if event.mask & pyinotify.IN_MOVE_SELF or not os.path.isfile(logpath): - self._addPendingFile(logpath, event) - else: - path = logpath - break - if path not in self.__watches: - logSys.debug("Ignoring event of %s we do not monitor", path) - return - if not os.path.isfile(path): - if self.containsLogPath(path): - self._addPendingFile(path, event) - logSys.debug("Ignoring watching/rotation event (%s) for %s", event.maskname, path) - return - self._refreshFileWatcher(path) + assumeNoDir = event.mask & (pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF) + if isWD and (assumeNoDir or not os.path.isdir(path)): + self._addPending(path, event, isDir=True) + elif not isWF: + for logpath in self.__watchDirs: + if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)): + self._addPending(logpath, event, isDir=True) + # pending file: + for logpath in self.__watchFiles: + if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isfile(logpath)): + self._addPending(logpath, event) + if isWF and not os.path.isfile(path): + self._addPending(path, event) + return # do nothing if idle: if self.idle: return + # be sure we process a file: + if not isWF: + logSys.debug("Ignoring event (%s) of %s we do not monitor", event.maskname, path) + return self._process_file(path) def _process_file(self, path): @@ -143,27 +155,36 @@ class FilterPyinotify(FileFilter): self.failManager.cleanup(MyTime.time()) self.__modified = False - def _addPendingFile(self, path, event): + def _addPending(self, path, event, isDir=False): if path not in self.__pending: - self.__pending[path] = self.sleeptime / 10; + self.__pending[path] = [self.sleeptime / 10, isDir]; + self.__pendingNextTime = 0 logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s", path, event.maskname, event.pathname) - def _checkPendingFiles(self): + def _delPending(self, path): + try: + del self.__pending[path] + except KeyError: pass + + def _checkPending(self): if self.__pending: ntm = time.time() if ntm > self.__pendingNextTime: found = {} minTime = 60 - for path, retardTM in self.__pending.iteritems(): + for path, (retardTM, isDir) in self.__pending.iteritems(): if ntm - self.__pendingChkTime > retardTM: - if not os.path.isfile(path): # not found - prolong for next time + chkpath = os.path.isdir if isDir else os.path.isfile + if not chkpath(path): # not found - prolong for next time if retardTM < 60: retardTM *= 2 if minTime > retardTM: minTime = retardTM - self.__pending[path] = retardTM + self.__pending[path][0] = retardTM continue - found[path] = 1 - self._refreshFileWatcher(path) + logSys.log(logging.MSG, "Log presence detected for %s %s", + "directory" if isDir else "file", path) + found[path] = isDir + self._refreshWatcher(path, isDir=isDir) for path in found: try: del self.__pending[path] @@ -171,24 +192,32 @@ class FilterPyinotify(FileFilter): self.__pendingChkTime = time.time() self.__pendingNextTime = self.__pendingChkTime + minTime # process now because we'he missed it in monitoring: - for path in found: - self._process_file(path) + for path, isDir in found.iteritems(): + if not isDir: + self._process_file(path) - def _refreshFileWatcher(self, oldPath, newPath=None): + def _refreshWatcher(self, oldPath, newPath=None, isDir=False): + if not newPath: newPath = oldPath # we need to substitute the watcher with a new one, so first - # remove old one - self._delFileWatcher(oldPath) - # place a new one - self._addFileWatcher(newPath or oldPath) + # remove old one and then place a new one + if not isDir: + self._delFileWatcher(oldPath) + self._addFileWatcher(newPath) + else: + self._delDirWatcher(oldPath) + self._addDirWatcher(newPath) def _addFileWatcher(self, path): + # we need to watch also the directory for IN_CREATE + self._addDirWatcher(dirname(path)) + # add file watcher: wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY) - self.__watches.update(wd) + self.__watchFiles.update(wd) logSys.debug("Added file watcher for %s", path) def _delFileWatcher(self, path): try: - wdInt = self.__watches.pop(path) + wdInt = self.__watchFiles.pop(path) wd = self.__monitor.rm_watch(wdInt) if wd[wdInt]: logSys.debug("Removed file watcher for %s", path) @@ -197,21 +226,30 @@ class FilterPyinotify(FileFilter): pass return False + def _addDirWatcher(self, path_dir): + # Add watch for the directory: + if path_dir not in self.__watchDirs: + self.__watchDirs.update( + self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | + pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF | + pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR)) + logSys.debug("Added monitor for the parent directory %s", path_dir) + + def _delDirWatcher(self, path_dir): + # Remove watches for the directory: + try: + wdInt = self.__watchDirs.pop(path_dir) + self.__monitor.rm_watch(wdInt) + except KeyError: # pragma: no cover + pass + logSys.debug("Removed monitor for the parent directory %s", path_dir) + ## # Add a log file path # # @param path log file path def _addLogPath(self, path): - path_dir = dirname(path) - if not (path_dir in self.__watches): - # we need to watch also the directory for IN_CREATE - self.__watches.update( - self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE | - pyinotify.IN_MOVED_TO | pyinotify.IN_MOVE_SELF | - pyinotify.IN_DELETE_SELF | pyinotify.IN_ISDIR)) - logSys.debug("Added monitor for the parent directory %s", path_dir) - self._addFileWatcher(path) self._process_file(path) @@ -223,18 +261,18 @@ class FilterPyinotify(FileFilter): def _delLogPath(self, path): if not self._delFileWatcher(path): logSys.error("Failed to remove watch on path: %s", path) + self._delPending(path) path_dir = dirname(path) - if not len([k for k in self.__watches - if k.startswith(path_dir + pathsep)]): + for k in self.__watchFiles: + if k.startswith(path_dir + pathsep): + path_dir = None + break + if path_dir: # Remove watches for the directory # since there is no other monitored file under this directory - try: - wdInt = self.__watches.pop(path_dir) - self.__monitor.rm_watch(wdInt) - except KeyError: # pragma: no cover - pass - logSys.debug("Removed monitor for the parent directory %s", path_dir) + self._delDirWatcher(path_dir) + self._delPending(path_dir) # pyinotify.ProcessEvent default handler: def __process_default(self, event): @@ -247,14 +285,16 @@ class FilterPyinotify(FileFilter): # slow check events while idle: def __check_events(self, *args, **kwargs): - # check pending files (logrotate ready): - self._checkPendingFiles() - if self.idle: if Utils.wait_for(lambda: not self.active or not self.idle, self.sleeptime * 10, self.sleeptime ): pass + + # check pending files/dirs (logrotate ready): + if not self.idle: + self._checkPending() + self.ticks += 1 return pyinotify.ThreadedNotifier.check_events(self.__notifier, *args, **kwargs) From 9841fe52c34c966d143e17e447268fce4e896214 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 12:52:16 +0200 Subject: [PATCH 13/43] fixed cleanup for Gamin backend (by interim stop in the test-cases) --- fail2ban/server/filtergamin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 106e4c0f..3baf8c54 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -143,6 +143,8 @@ class FilterGamin(FileFilter): # Desallocates the resources used by Gamin. def __cleanup(self): + if not self.monitor: + return for filename in self.getLogPaths(): self.monitor.stop_watch(filename) self.monitor = None From 5c1d01bf58635f10dc2a5c6c273c2a8b983b0ee8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 14:31:18 +0200 Subject: [PATCH 14/43] code review, try to make recognition of pending files fewer sporadic (error prone) --- fail2ban/server/filterpyinotify.py | 50 +++++++++++++++++------------- fail2ban/tests/filtertestcase.py | 1 - 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 6394ecef..4926f9b7 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -38,7 +38,7 @@ from ..helpers import getLogger if not hasattr(pyinotify, '__version__') \ - or LooseVersion(pyinotify.__version__) < '0.8.3': + or LooseVersion(pyinotify.__version__) < '0.8.3': # pragma: no cover raise ImportError("Fail2Ban requires pyinotify >= 0.8.3") # Verify that pyinotify is functional on this system @@ -46,7 +46,7 @@ if not hasattr(pyinotify, '__version__') \ try: manager = pyinotify.WatchManager() del manager -except Exception as e: +except Exception as e: # pragma: no cover raise ImportError("Pyinotify is probably not functional on this system: %s" % str(e)) @@ -54,7 +54,7 @@ except Exception as e: logSys = getLogger(__name__) # Override pyinotify default logger/init-handler: -def _pyinotify_logger_init(): +def _pyinotify_logger_init(): # pragma: no cover return logSys pyinotify._logger_init = _pyinotify_logger_init pyinotify.log = logSys @@ -124,10 +124,6 @@ class FilterPyinotify(FileFilter): for logpath in self.__watchDirs: if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)): self._addPending(logpath, event, isDir=True) - # pending file: - for logpath in self.__watchFiles: - if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isfile(logpath)): - self._addPending(logpath, event) if isWF and not os.path.isfile(path): self._addPending(path, event) return @@ -155,12 +151,14 @@ class FilterPyinotify(FileFilter): self.failManager.cleanup(MyTime.time()) self.__modified = False - def _addPending(self, path, event, isDir=False): + def _addPending(self, path, reason, isDir=False): if path not in self.__pending: self.__pending[path] = [self.sleeptime / 10, isDir]; self.__pendingNextTime = 0 + if isinstance(reason, pyinotify.Event): + reason = [reason.maskname, reason.pathname] logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s", - path, event.maskname, event.pathname) + path, *reason) def _delPending(self, path): try: @@ -174,17 +172,18 @@ class FilterPyinotify(FileFilter): found = {} minTime = 60 for path, (retardTM, isDir) in self.__pending.iteritems(): - if ntm - self.__pendingChkTime > retardTM: - chkpath = os.path.isdir if isDir else os.path.isfile - if not chkpath(path): # not found - prolong for next time - if retardTM < 60: retardTM *= 2 - if minTime > retardTM: minTime = retardTM - self.__pending[path][0] = retardTM - continue - logSys.log(logging.MSG, "Log presence detected for %s %s", - "directory" if isDir else "file", path) - found[path] = isDir - self._refreshWatcher(path, isDir=isDir) + if ntm - self.__pendingChkTime < retardTM: + if minTime > retardTM: minTime = retardTM + continue + chkpath = os.path.isdir if isDir else os.path.isfile + if not chkpath(path): # not found - prolong for next time + if retardTM < 60: retardTM *= 2 + if minTime > retardTM: minTime = retardTM + self.__pending[path][0] = retardTM + continue + logSys.log(logging.MSG, "Log presence detected for %s %s", + "directory" if isDir else "file", path) + found[path] = isDir for path in found: try: del self.__pending[path] @@ -193,7 +192,16 @@ class FilterPyinotify(FileFilter): self.__pendingNextTime = self.__pendingChkTime + minTime # process now because we'he missed it in monitoring: for path, isDir in found.iteritems(): - if not isDir: + self._refreshWatcher(path, isDir=isDir) + if isDir: + for logpath in self.__watchFiles: + if logpath.startswith(path + pathsep): + if not os.path.isfile(logpath): + self._addPending(logpath, ['FROM_PARDIR', path]) + else: + self._refreshWatcher(logpath) + self._process_file(logpath) + else: self._process_file(path) def _refreshWatcher(self, oldPath, newPath=None, isDir=False): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index a9b31fd4..ec2dea89 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -957,7 +957,6 @@ def get_monitor_failures_testcase(Filter_): self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') self._wait4failures() - self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') From ff1c6718da50f3d7eaf447bd399211f1fde7708b Mon Sep 17 00:00:00 2001 From: Filippo Tessarotto Date: Tue, 7 Feb 2017 12:58:04 +0100 Subject: [PATCH 15/43] Postfix RBL: 554 & SMTP Cherry-pick of 607568f5da9e6b962c43d59280d72bd678c08afa (see gh-1686) --- config/filter.d/postfix-rbl.conf | 2 +- fail2ban/tests/files/logs/postfix-rbl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf index c3f8c332..0a9078f0 100644 --- a/config/filter.d/postfix-rbl.conf +++ b/config/filter.d/postfix-rbl.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = postfix(-\w+)?/smtpd -failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ +failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl index eff01bf9..6aeac03b 100644 --- a/fail2ban/tests/files/logs/postfix-rbl +++ b/fail2ban/tests/files/logs/postfix-rbl @@ -3,3 +3,6 @@ Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.examp # failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" } Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from= to= proto=ESMTP helo= + +# failJSON: { "time": "2005-02-07T12:25:45", "match": true , "host": "87.236.233.182" } +Feb 7 12:25:45 xxx12345 postfix/smtpd[13275]: NOQUEUE: reject: RCPT from unknown[87.236.233.182]: 554 5.7.1 Service unavailable; Client host [87.236.233.182] blocked using rbl.example.com; https://www.example.com/query/ip/87.236.233.182; from= to= proto=SMTP helo= From 16a84ca0b5e5e5624e8d5871c356eb6d9c4c4b68 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 15:20:43 +0200 Subject: [PATCH 16/43] code review --- fail2ban/server/filter.py | 2 +- fail2ban/server/filterpyinotify.py | 111 +++++++++++++++-------------- 2 files changed, 57 insertions(+), 56 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index c88a6c6d..fce02a7a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -895,7 +895,7 @@ class FileFilter(Filter): # see http://python.org/dev/peps/pep-3151/ except IOError as e: logSys.error("Unable to open %s", filename) - if e.errno != 2: + if e.errno != 2: # errno.ENOENT logSys.exception(e) return False except OSError as e: # pragma: no cover - requires race condition to tigger this diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 4926f9b7..7785b84f 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -89,23 +89,12 @@ class FilterPyinotify(FileFilter): logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event) path = event.pathname # check watching of this path: - isWF = isWD = False - if path in self.__watchDirs: - isWD = True - elif path in self.__watchFiles: + isWF = False + isWD = path in self.__watchDirs + if not isWD and path in self.__watchFiles: isWF = True - # fix pyinotify behavior with '-unknown-path' (if target not watched also): - if (event.mask & pyinotify.IN_MOVE_SELF and - path.endswith('-unknown-path') and not isWF and not isWD - ): - path = path[:-len('-unknown-path')] - isWD = path in self.__watchDirs assumeNoDir = False if event.mask & ( pyinotify.IN_CREATE | pyinotify.IN_MOVED_TO ): - # refresh watched dir (may be expected): - if isWD: - self._refreshWatcher(path, isDir=True) - return # skip directories altogether if event.mask & pyinotify.IN_ISDIR: logSys.debug("Ignoring creation of directory %s", path) @@ -116,8 +105,14 @@ class FilterPyinotify(FileFilter): return self._refreshWatcher(path) elif event.mask & (pyinotify.IN_IGNORED | pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF): - # watch was removed for some reasons (log-rotate?): assumeNoDir = event.mask & (pyinotify.IN_MOVE_SELF | pyinotify.IN_DELETE_SELF) + # fix pyinotify behavior with '-unknown-path' (if target not watched also): + if (assumeNoDir and + path.endswith('-unknown-path') and not isWF and not isWD + ): + path = path[:-len('-unknown-path')] + isWD = path in self.__watchDirs + # watch was removed for some reasons (log-rotate?): if isWD and (assumeNoDir or not os.path.isdir(path)): self._addPending(path, event, isDir=True) elif not isWF: @@ -153,7 +148,7 @@ class FilterPyinotify(FileFilter): def _addPending(self, path, reason, isDir=False): if path not in self.__pending: - self.__pending[path] = [self.sleeptime / 10, isDir]; + self.__pending[path] = [Utils.DEFAULT_SLEEP_INTERVAL, isDir]; self.__pendingNextTime = 0 if isinstance(reason, pyinotify.Event): reason = [reason.maskname, reason.pathname] @@ -166,43 +161,49 @@ class FilterPyinotify(FileFilter): except KeyError: pass def _checkPending(self): - if self.__pending: - ntm = time.time() - if ntm > self.__pendingNextTime: - found = {} - minTime = 60 - for path, (retardTM, isDir) in self.__pending.iteritems(): - if ntm - self.__pendingChkTime < retardTM: - if minTime > retardTM: minTime = retardTM - continue - chkpath = os.path.isdir if isDir else os.path.isfile - if not chkpath(path): # not found - prolong for next time - if retardTM < 60: retardTM *= 2 - if minTime > retardTM: minTime = retardTM - self.__pending[path][0] = retardTM - continue - logSys.log(logging.MSG, "Log presence detected for %s %s", - "directory" if isDir else "file", path) - found[path] = isDir - for path in found: - try: - del self.__pending[path] - except KeyError: pass - self.__pendingChkTime = time.time() - self.__pendingNextTime = self.__pendingChkTime + minTime - # process now because we'he missed it in monitoring: - for path, isDir in found.iteritems(): - self._refreshWatcher(path, isDir=isDir) - if isDir: - for logpath in self.__watchFiles: - if logpath.startswith(path + pathsep): - if not os.path.isfile(logpath): - self._addPending(logpath, ['FROM_PARDIR', path]) - else: - self._refreshWatcher(logpath) - self._process_file(logpath) - else: - self._process_file(path) + if not self.__pending: + return + ntm = time.time() + if ntm < self.__pendingNextTime: + return + found = {} + minTime = 60 + for path, (retardTM, isDir) in self.__pending.iteritems(): + if ntm - self.__pendingChkTime < retardTM: + if minTime > retardTM: minTime = retardTM + continue + chkpath = os.path.isdir if isDir else os.path.isfile + if not chkpath(path): # not found - prolong for next time + if retardTM < 60: retardTM *= 2 + if minTime > retardTM: minTime = retardTM + self.__pending[path][0] = retardTM + continue + logSys.log(logging.MSG, "Log presence detected for %s %s", + "directory" if isDir else "file", path) + found[path] = isDir + for path in found: + try: + del self.__pending[path] + except KeyError: pass + self.__pendingChkTime = time.time() + self.__pendingNextTime = self.__pendingChkTime + minTime + # process now because we've missed it in monitoring: + for path, isDir in found.iteritems(): + # refresh monitoring of this: + self._refreshWatcher(path, isDir=isDir) + if isDir: + # check all files belong to this dir: + for logpath in self.__watchFiles: + if logpath.startswith(path + pathsep): + # if still no file - add to pending, otherwise refresh and process: + if not os.path.isfile(logpath): + self._addPending(logpath, ('FROM_PARDIR', path)) + else: + self._refreshWatcher(logpath) + self._process_file(logpath) + else: + # process (possibly no old events for it from watcher): + self._process_file(path) def _refreshWatcher(self, oldPath, newPath=None, isDir=False): if not newPath: newPath = oldPath @@ -286,7 +287,7 @@ class FilterPyinotify(FileFilter): def __process_default(self, event): try: self.callback(event, origin='Default ') - except Exception as e: + except Exception as e: # pragma: no cover logSys.error("Error in FilterPyinotify callback: %s", e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) self.ticks += 1 @@ -300,7 +301,7 @@ class FilterPyinotify(FileFilter): pass # check pending files/dirs (logrotate ready): - if not self.idle: + if not self.idle and self.active: self._checkPending() self.ticks += 1 From f099558bcff64a65f8dc3e185838e1ce3c391576 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 15:43:16 +0200 Subject: [PATCH 17/43] try to fix travis build for pypy3 (after switch to 'pypy3.3-5.2-alpha1' the test cases seems to never run anymore). --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40376075..dc5b1e8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,9 @@ python: # - pypy3 - pypy3.3-5.2-alpha1 before_install: - - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi + - echo "running under $TRAVIS_PYTHON_VERSION" + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi - travis_retry sudo apt-get update -qq # Set this so sudo executes the correct python binary # Anything not using sudo will already have the correct environment From 62e580b7cf647a2f66c19a833e5aa703ad1c19f8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 18:47:00 +0200 Subject: [PATCH 18/43] pyinotify: switch from ThreadedNotifier to Notifier: - Filter instance is already a thread; - avoid stop pyinotify processing if an interim error occurs (and breaks main-loop, e. g. during multi-threaded processing by add/remove log-files) --- fail2ban/server/filterpyinotify.py | 73 ++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 7785b84f..29e29eca 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -290,23 +290,10 @@ class FilterPyinotify(FileFilter): except Exception as e: # pragma: no cover logSys.error("Error in FilterPyinotify callback: %s", e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) + # incr common error counter: + self.commonError() self.ticks += 1 - # slow check events while idle: - def __check_events(self, *args, **kwargs): - if self.idle: - if Utils.wait_for(lambda: not self.active or not self.idle, - self.sleeptime * 10, self.sleeptime - ): - pass - - # check pending files/dirs (logrotate ready): - if not self.idle and self.active: - self._checkPending() - - self.ticks += 1 - return pyinotify.ThreadedNotifier.check_events(self.__notifier, *args, **kwargs) - ## # Main loop. # @@ -317,26 +304,64 @@ class FilterPyinotify(FileFilter): prcevent = pyinotify.ProcessEvent() prcevent.process_default = self.__process_default ## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds) - self.__notifier = pyinotify.ThreadedNotifier(self.__monitor, + self.__notifier = pyinotify.Notifier(self.__monitor, prcevent, timeout=self.sleeptime * 1000) - self.__notifier.check_events = self.__check_events - self.__notifier.start() logSys.debug("[%s] filter started (pyinotifier)", self.jailName) + while self.active: + try: + + # slow check events while idle: + if self.idle: + if Utils.wait_for(lambda: not self.active or not self.idle, + self.sleeptime * 10, self.sleeptime + ): + if not self.active: + break + + # default pyinotify handling using Notifier: + self.__notifier.process_events() + if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime): + if not self.active: + break + self.__notifier.read_events() + + # check pending files/dirs (logrotate ready): + if not self.idle: + self._checkPending() + + except Exception as e: # pragma: no cover + if not self.active: # if not active - error by stop... + break + logSys.error("Caught unhandled exception in main cycle: %r", e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + # incr common error counter: + self.commonError() + + self.ticks += 1 + + logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) + self.__notifier = None + return True ## # Call super.stop() and then stop the 'Notifier' def stop(self): + if self.__notifier: # stop the notifier + self.__notifier.stop() + # stop filter thread: super(FilterPyinotify, self).stop() - # Stop the notifier thread - self.__notifier.stop() - self.__notifier.stop = lambda *args: 0; # prevent dual stop + self.join() + if self.__notifier: # stop the notifier + self.__notifier.stop() + self.__notifier.stop = lambda *args: 0; # prevent dual stop ## # Wait for exit with cleanup. def join(self): + self.join = lambda *args: 0 self.__cleanup() super(FilterPyinotify, self).join() logSys.debug("[%s] filter terminated (pyinotifier)", self.jailName) @@ -346,6 +371,6 @@ class FilterPyinotify(FileFilter): def __cleanup(self): if self.__notifier: - self.__notifier.join() # to not exit before notifier does - self.__notifier = None - self.__monitor = None + if Utils.wait_for(lambda: not self.__notifier, self.sleeptime * 10): + self.__notifier = None + self.__monitor = None From 050076ae42f13c74ae8ba418f5b1f40be96515cd Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 May 2017 19:05:40 +0200 Subject: [PATCH 19/43] code review + coverage fixes --- fail2ban/server/filterpyinotify.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 29e29eca..71540f3c 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -137,14 +137,15 @@ class FilterPyinotify(FileFilter): TODO -- RF: this is a common logic and must be shared/provided by FileFilter """ - self.getFailures(path) - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False + if not self.idle: + self.getFailures(path) + try: + while True: + ticket = self.failManager.toBan() + self.jail.putFailTicket(ticket) + except FailManagerEmpty: + self.failManager.cleanup(MyTime.time()) + self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -268,7 +269,7 @@ class FilterPyinotify(FileFilter): # @param path the log file to delete def _delLogPath(self, path): - if not self._delFileWatcher(path): + if not self._delFileWatcher(path): # pragma: no cover logSys.error("Failed to remove watch on path: %s", path) self._delPending(path) @@ -315,14 +316,12 @@ class FilterPyinotify(FileFilter): if Utils.wait_for(lambda: not self.active or not self.idle, self.sleeptime * 10, self.sleeptime ): - if not self.active: - break + if not self.active: break # default pyinotify handling using Notifier: self.__notifier.process_events() if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime): - if not self.active: - break + if not self.active: break self.__notifier.read_events() # check pending files/dirs (logrotate ready): @@ -341,7 +340,7 @@ class FilterPyinotify(FileFilter): logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None - + return True ## @@ -353,9 +352,6 @@ class FilterPyinotify(FileFilter): # stop filter thread: super(FilterPyinotify, self).stop() self.join() - if self.__notifier: # stop the notifier - self.__notifier.stop() - self.__notifier.stop = lambda *args: 0; # prevent dual stop ## # Wait for exit with cleanup. From 17b0945a70e21395ebe2139ec18458155fe06433 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 16 May 2017 09:43:52 +0200 Subject: [PATCH 20/43] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 78de41da..73df1383 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,7 @@ TODO: implementing of options resp. other tasks from PR #1346 * fail2ban-regex: fixed matched output by multi-line (buffered) parsing * fail2ban-regex: support for multi-line debuggex URL implemented (gh-422) * fixed ipv6-action errors on systems not supporting ipv6 and vice versa (gh-1741) +* fixed directory-based log-rotate for pyinotify-backend (gh-1778) ### New Features * New Actions: From 9b83a3128d16fb545798283850a2760036915234 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 17 May 2017 12:23:28 +0200 Subject: [PATCH 21/43] code review, try to increase coverage --- fail2ban/server/filterpyinotify.py | 7 ++----- fail2ban/tests/filtertestcase.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 71540f3c..d945fc2d 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -115,7 +115,7 @@ class FilterPyinotify(FileFilter): # watch was removed for some reasons (log-rotate?): if isWD and (assumeNoDir or not os.path.isdir(path)): self._addPending(path, event, isDir=True) - elif not isWF: + elif not isWF: # pragma: no cover (assume too sporadic) for logpath in self.__watchDirs: if logpath.startswith(path + pathsep) and (assumeNoDir or not os.path.isdir(logpath)): self._addPending(logpath, event, isDir=True) @@ -182,14 +182,11 @@ class FilterPyinotify(FileFilter): logSys.log(logging.MSG, "Log presence detected for %s %s", "directory" if isDir else "file", path) found[path] = isDir - for path in found: - try: - del self.__pending[path] - except KeyError: pass self.__pendingChkTime = time.time() self.__pendingNextTime = self.__pendingChkTime + minTime # process now because we've missed it in monitoring: for path, isDir in found.iteritems(): + self._delPending(path) # refresh monitoring of this: self._refreshWatcher(path, isDir=isDir) if isDir: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ec2dea89..2bb64132 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -973,6 +973,7 @@ def get_monitor_failures_testcase(Filter_): @with_tmpdir def test_move_dir(self, tmp): self.file.close() + self.filter.setMaxRetry(10) self.filter.delLogPath(self.name) # if we rename parent dir into a new location (simulate directory-base log rotation) tmpsub1 = os.path.join(tmp, "1") @@ -987,13 +988,25 @@ def get_monitor_failures_testcase(Filter_): self.file.close() self._wait4failures(1) - # rotate whole directory: rename directory 1 as 2: - os.rename(tmpsub1, tmpsub2) + # rotate whole directory: rename directory 1 as 2a: + os.rename(tmpsub1, tmpsub2 + 'a') os.mkdir(tmpsub1) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=1, mode='w') self.file.close() self._wait4failures(2) + + # rotate whole directory: rename directory 1 as 2b: + os.rename(tmpsub1, tmpsub2 + 'b') + # wait a bit in-between (try to increase coverage, should find pending file for pending dir): + self.waitForTicks(2) + os.mkdir(tmpsub1) + self.waitForTicks(2) + self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, + skip=12, n=1, mode='w') + self.file.close() + self._wait4failures(3) + # stop before tmpdir deleted (just prevents many monitor events) self.filter.stop() From 96c3b06abbb11f7eb8735acd7741d62854e425ba Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 19 May 2017 13:25:49 +0200 Subject: [PATCH 22/43] amend to #1778: repair notifier wait-cycle (too long timeout in polling, too late check for pending files, too long stop) --- fail2ban/server/filterpyinotify.py | 32 ++++++++++++++++++++---------- fail2ban/tests/filtertestcase.py | 1 + 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 71540f3c..53f2cbda 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -82,7 +82,7 @@ class FilterPyinotify(FileFilter): self.__watchDirs = dict() self.__pending = dict() self.__pendingChkTime = 0 - self.__pendingNextTime = 0 + self.__pendingMinTime = 60 logSys.debug("Created FilterPyinotify") def callback(self, event, origin=''): @@ -150,7 +150,7 @@ class FilterPyinotify(FileFilter): def _addPending(self, path, reason, isDir=False): if path not in self.__pending: self.__pending[path] = [Utils.DEFAULT_SLEEP_INTERVAL, isDir]; - self.__pendingNextTime = 0 + self.__pendingMinTime = 0 if isinstance(reason, pyinotify.Event): reason = [reason.maskname, reason.pathname] logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s", @@ -165,7 +165,7 @@ class FilterPyinotify(FileFilter): if not self.__pending: return ntm = time.time() - if ntm < self.__pendingNextTime: + if ntm < self.__pendingChkTime + self.__pendingMinTime: return found = {} minTime = 60 @@ -187,7 +187,7 @@ class FilterPyinotify(FileFilter): del self.__pending[path] except KeyError: pass self.__pendingChkTime = time.time() - self.__pendingNextTime = self.__pendingChkTime + minTime + self.__pendingMinTime = minTime # process now because we've missed it in monitoring: for path, isDir in found.iteritems(): # refresh monitoring of this: @@ -295,6 +295,12 @@ class FilterPyinotify(FileFilter): self.commonError() self.ticks += 1 + @property + def __notify_maxtout(self): + # timeout for pyinotify must be set in milliseconds (fail2ban time values are + # floats contain seconds), max 0.5 sec (additionally regards pending check time) + return min(self.sleeptime, 0.5, self.__pendingMinTime) * 1000 + ## # Main loop. # @@ -304,9 +310,8 @@ class FilterPyinotify(FileFilter): def run(self): prcevent = pyinotify.ProcessEvent() prcevent.process_default = self.__process_default - ## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds) self.__notifier = pyinotify.Notifier(self.__monitor, - prcevent, timeout=self.sleeptime * 1000) + prcevent, timeout=self.__notify_maxtout) logSys.debug("[%s] filter started (pyinotifier)", self.jailName) while self.active: try: @@ -314,13 +319,19 @@ class FilterPyinotify(FileFilter): # slow check events while idle: if self.idle: if Utils.wait_for(lambda: not self.active or not self.idle, - self.sleeptime * 10, self.sleeptime + min(self.sleeptime * 10, self.__pendingMinTime), + min(self.sleeptime, self.__pendingMinTime) ): if not self.active: break # default pyinotify handling using Notifier: self.__notifier.process_events() - if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime): + + # wait for events / timeout: + notify_maxtout = self.__notify_maxtout + def __check_events(): + return not self.active or self.__notifier.check_events(timeout=notify_maxtout) + if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)): if not self.active: break self.__notifier.read_events() @@ -347,11 +358,10 @@ class FilterPyinotify(FileFilter): # Call super.stop() and then stop the 'Notifier' def stop(self): - if self.__notifier: # stop the notifier - self.__notifier.stop() # stop filter thread: super(FilterPyinotify, self).stop() - self.join() + if self.__notifier: # stop the notifier + self.__notifier.stop() ## # Wait for exit with cleanup. diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ec2dea89..20a7ad50 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -996,6 +996,7 @@ def get_monitor_failures_testcase(Filter_): self._wait4failures(2) # stop before tmpdir deleted (just prevents many monitor events) self.filter.stop() + self.filter.join() def _test_move_into_file(self, interim_kill=False): From c7ddf1f940aa30f7d31dc1bf6e4d895713a6e38b Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 19 May 2017 15:36:06 +0200 Subject: [PATCH 23/43] [systemd-backend] implicit closing journal descriptor by stop filter. Partially cherry-picked from 0.10 (d153555a07ec81dabe4678ea06c8e644afc8ec40) --- ChangeLog | 2 ++ fail2ban/server/filtersystemd.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8e7ebdb0..c798768d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,8 @@ releases. ### Fixes +* Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991. + Partially back-ported from v.0.10. ### New Features diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 908112a7..73db618e 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -291,6 +291,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover except FailManagerEmpty: self.failManager.cleanup(MyTime.time()) + # close journal: + try: + if self.__journal: + self.__journal.close() + except Exception as e: # pragma: no cover + logSys.error("Close journal failed: %r", e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) logSys.debug((self.jail is not None and self.jail.name or "jailless") +" filter terminated") return True From e8f217390449120a3c3c7fe4ca502b37eb8d2c56 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Tue, 23 May 2017 17:27:12 +0200 Subject: [PATCH 24/43] New logtimezone jail option This new option allows to force the time zone on log lines that don't bear a time zone indication (GitHub issue #1773), so it behaves actually with respect to log line contents as a default time zone. For the time being, only fixed offset timezones (UTC or UTC[+-]hhmm) are supported, but the implementation is designed to later on treat the case of logical timezones with DST, e.g., Europe/Paris etc. In particular, the timezone name gets passed all the way to the strptime module, and the resulting offset is computed for the given log line, even though for now, it doesn't actually depend on it. Also, the DateTemplate subclass gets to choose whether to use it or not. For instance, it doesn't make sense to apply a time zone offset to Unix timestamps. The drawback is to introduce an API change for DateTemplate. I hope it's internal enough for that not being a problem. --- fail2ban/client/jailreader.py | 1 + fail2ban/server/datedetector.py | 4 +-- fail2ban/server/datetemplate.py | 16 +++++++---- fail2ban/server/filter.py | 27 ++++++++++++++++-- fail2ban/server/server.py | 6 ++++ fail2ban/server/strptime.py | 39 ++++++++++++++++++++++++-- fail2ban/server/transmitter.py | 6 ++++ fail2ban/tests/clientreadertestcase.py | 12 ++++++++ fail2ban/tests/config/jail.conf | 4 +++ fail2ban/tests/datedetectortestcase.py | 15 ++++++++++ fail2ban/tests/filtertestcase.py | 10 +++++++ fail2ban/tests/servertestcase.py | 4 +++ 12 files changed, 133 insertions(+), 11 deletions(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index ca092990..ce0ed3b6 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -101,6 +101,7 @@ class JailReader(ConfigReader): ["string", "filter", ""]] opts = [["bool", "enabled", False], ["string", "logpath", None], + ["string", "logtimezone", None], ["string", "logencoding", None], ["string", "backend", "auto"], ["int", "maxretry", None], diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index dd5d198f..39a15828 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -423,7 +423,7 @@ class DateDetector(object): logSys.log(logLevel, " no template.") return (None, None) - def getTime(self, line, timeMatch=None): + def getTime(self, line, timeMatch=None, default_tz=None): """Attempts to return the date on a log line using templates. This uses the templates' `getDate` method in an attempt to find @@ -449,7 +449,7 @@ class DateDetector(object): template = timeMatch[1] if template is not None: try: - date = template.getDate(line, timeMatch[0]) + date = template.getDate(line, timeMatch[0], default_tz=default_tz) if date is not None: if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug logSys.log(logLevel, " got time %f for %r using template %s", diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 1d0b014b..44ea54a8 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -158,7 +158,7 @@ class DateTemplate(object): return dateMatch @abstractmethod - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Abstract method, which should return the date for a log line This should return the date for a log line, typically taking the @@ -169,6 +169,8 @@ class DateTemplate(object): ---------- line : str Log line, of which the date should be extracted from. + default_tz: if no explicit time zone is present in the line + passing this will interpret it as in that time zone. Raises ------ @@ -200,13 +202,14 @@ class DateEpoch(DateTemplate): regex = r"((?P(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))" self.setRegex(regex, wordBegin='start', wordEnd=True) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. Parameters ---------- line : str Log line, of which the date should be extracted from. + default_tz: ignored, Unix timestamps are time zone independent Returns ------- @@ -277,7 +280,7 @@ class DatePatternRegex(DateTemplate): regex = r'(?iu)' + regex super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. This uses a custom version of strptime, using the named groups @@ -287,6 +290,7 @@ class DatePatternRegex(DateTemplate): ---------- line : str Log line, of which the date should be extracted from. + default_tz: optionally used to correct timezone Returns ------- @@ -297,7 +301,8 @@ class DatePatternRegex(DateTemplate): if not dateMatch: dateMatch = self.matchDate(line) if dateMatch: - return reGroupDictStrptime(dateMatch.groupdict()), dateMatch + return (reGroupDictStrptime(dateMatch.groupdict(), default_tz=default_tz), + dateMatch) class DateTai64n(DateTemplate): @@ -315,13 +320,14 @@ class DateTai64n(DateTemplate): # We already know the format for TAI64N self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin) - def getDate(self, line, dateMatch=None): + def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. Parameters ---------- line : str Log line, of which the date should be extracted from. + default_tz: ignored, since TAI is time zone independent Returns ------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index fce02a7a..8d1eb856 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -40,6 +40,7 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from .utils import Utils from ..helpers import getLogger, PREFER_ENC +from .strptime import validateTimeZone # Gets the instance of the logger. logSys = getLogger(__name__) @@ -102,6 +103,8 @@ class Filter(JailThread): self.checkAllRegex = False ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## if set, treat log lines without explicit time zone to be in this time zone + self.logtimezone = None ## Ticks counter self.ticks = 0 @@ -307,6 +310,22 @@ class Filter(JailThread): return pattern, templates[0].name return None + ## + # Set the log default time zone + # + # @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM) + + def setLogTimeZone(self, tz): + self.logtimezone = validateTimeZone(tz) + + ## + # Get the log default timezone + # + # @return symbolic timezone (a string) + + def getLogTimeZone(self): + return self.logtimezone + ## # Set the maximum retry value. # @@ -621,7 +640,8 @@ class Filter(JailThread): self.__lastDate = date elif timeText: - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) + dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3], + default_tz=self.logtimezone) if dateTimeMatch is None: logSys.error("findFailure failed to parse timeText: %s", timeText) @@ -972,7 +992,10 @@ class FileFilter(Filter): break (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: - dateTimeMatch = self.dateDetector.getTime(line[timeMatch.start():timeMatch.end()], (timeMatch, template)) + dateTimeMatch = self.dateDetector.getTime( + line[timeMatch.start():timeMatch.end()], + (timeMatch, template), + default_tz=self.logtimezone) else: nextp = container.tell() if nextp > maxp: diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 56367ed7..e3b22c44 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -379,6 +379,12 @@ class Server: def getDatePattern(self, name): return self.__jails[name].filter.getDatePattern() + def setLogTimeZone(self, name, tz): + self.__jails[name].filter.setLogTimeZone(tz) + + def getLogTimeZone(self, name): + return self.__jails[name].filter.getLogTimeZone() + def setIgnoreCommand(self, name, value): self.__jails[name].filter.setIgnoreCommand(value) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 55bdcc8c..aff9db92 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -17,6 +17,7 @@ # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +import re import time import calendar import datetime @@ -26,6 +27,7 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() +FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$') def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): """ Build century regex for last year and the next years (distance). @@ -78,7 +80,36 @@ def getTimePatternRE(): names[key] = "%%%s" % key return (patt, names) -def reGroupDictStrptime(found_dict, msec=False): + +def validateTimeZone(tz): + """Validate a timezone. + + For now this accepts only the UTC[+-]hhmm format. + In the future, it may be extended for named time zones (such as Europe/Paris) + present on the system, if a suitable tz library is present. + """ + m = FIXED_OFFSET_TZ_RE.match(tz) + if m is None: + raise ValueError("Unknown or unsupported time zone: %r" % tz) + return tz + +def zone2offset(tz, dt): + """Return the proper offset, in minutes according to given timezone at a given time. + + Parameters + ---------- + tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have + been validated already) + dt: datetime instance for offset computation + """ + if tz == 'UTC': + return 0 + unsigned = int(tz[4:6])*60 + int(tz[6:]) + if tz[3] == '-': + return -unsigned + return unsigned + +def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """Return time from dictionary of strptime fields This is tweaked from python built-in _strptime. @@ -88,7 +119,8 @@ def reGroupDictStrptime(found_dict, msec=False): found_dict : dict Dictionary where keys represent the strptime fields, and values the respective value. - + default_tz : default timezone to apply if nothing relevant is in found_dict + (may be a non-fixed one in the future) Returns ------- float @@ -209,6 +241,9 @@ def reGroupDictStrptime(found_dict, msec=False): # Actully create date date_result = datetime.datetime( year, month, day, hour, minute, second, fraction) + # Correct timezone if not supplied in the log linge + if tzoffset is None and default_tz is not None: + tzoffset = zone2offset(default_tz, date_result) # Add timezone info if tzoffset is not None: date_result -= datetime.timedelta(seconds=tzoffset * 60) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index bc9edd43..ecc2a138 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -261,6 +261,10 @@ class Transmitter: value = command[2] self.__server.setDatePattern(name, value) return self.__server.getDatePattern(name) + elif command[1] == "logtimezone": + value = command[2] + self.__server.setLogTimeZone(name, value) + return self.__server.getLogTimeZone(name) elif command[1] == "maxretry": value = command[2] self.__server.setMaxRetry(name, int(value)) @@ -363,6 +367,8 @@ class Transmitter: return self.__server.getFindTime(name) elif command[1] == "datepattern": return self.__server.getDatePattern(name) + elif command[1] == "logtimezone": + return self.__server.getLogTimeZone(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) elif command[1] == "maxlines": diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index bfa68e03..6cc2f659 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -196,6 +196,14 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.isEnabled()) self.assertLogged("Invalid action definition 'joho[foo'") + def testJailLogTimeZone(self): + jail = JailReader('tz_correct', basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + self.assertEqual(jail.options['logtimezone'], 'UTC+0200') + def testJailFilterBrokenDef(self): jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) @@ -533,10 +541,14 @@ class JailsReaderTest(LogCaptureTestCase): ]], ['add', 'parse_to_end_of_jail.conf', 'auto'], ['set', 'parse_to_end_of_jail.conf', 'addfailregex', ''], + ['set', 'tz_correct', 'addfailregex', ''], + ['set', 'tz_correct', 'logtimezone', 'UTC+0200'], ['start', 'emptyaction'], ['start', 'missinglogfiles'], ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'], + ['add', 'tz_correct', 'auto'], + ['start', 'tz_correct'], ['config-error', "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], ['config-error', diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 64c1b830..3dcbf634 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -47,3 +47,7 @@ action = thefunkychickendance [parse_to_end_of_jail.conf] enabled = true action = + +[tz_correct] +enabled = true +logtimezone = UTC+0200 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 39ab7173..16515f1b 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -89,6 +89,21 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(datelog, dateUnix) self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') + def testDefaultTimeZone(self): + log = "2017-01-23 15:00:00" + datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300') + # so in UTC, it was noon + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 12, 0, 0)) + + datelog, _ = self.datedetector.getTime(log, default_tz='UTC') + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 15, 0, 0)) + + datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430') + self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), + datetime.datetime(2017, 1, 23, 19, 30, 0)) + def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand """ diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 745dcca7..cb0edb06 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -289,6 +289,16 @@ class BasicFilter(unittest.TestCase): ("^%Y-%m-%d-%H%M%S.%f %z **", "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **")) + def testGetSetLogTimeZone(self): + self.assertEqual(self.filter.getLogTimeZone(), None) + self.filter.setLogTimeZone('UTC') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC') + self.filter.setLogTimeZone('UTC-0400') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC-0400') + self.filter.setLogTimeZone('UTC+0200') + self.assertEqual(self.filter.getLogTimeZone(), 'UTC+0200') + self.assertRaises(ValueError, self.filter.setLogTimeZone, 'not-a-time-zone') + def testAssertWrongTime(self): self.assertRaises(AssertionError, lambda: _assert_equal_entries(self, diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 32f794a6..68b9951c 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -311,6 +311,10 @@ class Transmitter(TransmitterBase): "datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName) self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName) + def testLogTimeZone(self): + self.setGetTest("logtimezone", "UTC+0400", "UTC+0400", jail=self.jailName) + self.setGetTestNOK("logtimezone", "not-a-time-zone", jail=self.jailName) + def testJailUseDNS(self): self.setGetTest("usedns", "yes", jail=self.jailName) self.setGetTest("usedns", "warn", jail=self.jailName) From 5bb6be01636ca515ded2b906bb9c7f72ea85b96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Tue, 30 May 2017 02:05:38 +0200 Subject: [PATCH 25/43] IPv6 address may overlap --- config/action.d/helpers-common.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf index b04c7f7a..ccbcb172 100644 --- a/config/action.d/helpers-common.conf +++ b/config/action.d/helpers-common.conf @@ -5,7 +5,7 @@ # (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... # _grep_logs = logpath=""; grep -E %(_grep_logs_args)s $logpath | -_grep_logs_args = '(^|[^0-9])([^0-9]|$)' +_grep_logs_args = '(^|[^0-9:])([^0-9:]|$)' # Used for actions, that should not by executed if ticket was restored: _bypass_if_restored = if [ '' = '1' ]; then exit 0; fi; @@ -13,4 +13,4 @@ _bypass_if_restored = if [ '' = '1' ]; then exit 0; fi; [Init] greplimit = tail -n grepmax = 1000 -grepopts = -m \ No newline at end of file +grepopts = -m From 80cc47b75ff14858d9a7bab8410ce982745508da Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 30 May 2017 09:14:43 +0200 Subject: [PATCH 26/43] Update helpers-common.conf fixed grep pattern: escape dot-char in search-IP and more restrictive boundaries (IPv6-capable) --- config/action.d/helpers-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf index ccbcb172..5799d9d3 100644 --- a/config/action.d/helpers-common.conf +++ b/config/action.d/helpers-common.conf @@ -5,7 +5,7 @@ # (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... # _grep_logs = logpath=""; grep -E %(_grep_logs_args)s $logpath | -_grep_logs_args = '(^|[^0-9:])([^0-9:]|$)' +_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)" # Used for actions, that should not by executed if ticket was restored: _bypass_if_restored = if [ '' = '1' ]; then exit 0; fi; From 228d25c548a3f62d8606054709fcb29ff3367775 Mon Sep 17 00:00:00 2001 From: Marcel Bischoff Date: Tue, 30 May 2017 20:27:44 +0200 Subject: [PATCH 27/43] Update Kerio Connect filter (#1455) * Update Kerio Connect filter Fixed regex for some log entries that did not get recognized and some additional error formats are added. * Add missing colon, GitHub address * Add filter tests * Add missing test --- config/filter.d/kerio.conf | 8 +++++++- fail2ban/tests/files/logs/kerio | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/config/filter.d/kerio.conf b/config/filter.d/kerio.conf index 313c9b36..ec1c09df 100644 --- a/config/filter.d/kerio.conf +++ b/config/filter.d/kerio.conf @@ -3,9 +3,14 @@ [Definition] failregex = ^ SMTP Spam attack detected from , - ^ IP address found in DNS blacklist \S+, mail from \S+ to \S+$ + ^ IP address found in DNS blacklist ^ Relay attempt from IP address ^ Attempt to deliver to unknown recipient \S+, from \S+, IP address $ + ^ Failed SMTP login from + ^ SMTP: User \S+ doesn't exist. Attempt from IP address + ^ Client with IP address has no reverse DNS entry, connection rejected before SMTP greeting$ + ^ Administration login into Web Administration from failed: IP address not allowed$ + ^ Message from IP address , sender \S+ rejected: sender domain does not exist$ ignoreregex = @@ -16,5 +21,6 @@ datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\] # DEV NOTES: # # Author: A.P. Lawrence +# Updated by: M. Bischoff # # Based off: http://aplawrence.com/Kerio/fail2ban.html diff --git a/fail2ban/tests/files/logs/kerio b/fail2ban/tests/files/logs/kerio index c9368c22..f7aa0eb7 100644 --- a/fail2ban/tests/files/logs/kerio +++ b/fail2ban/tests/files/logs/kerio @@ -25,5 +25,20 @@ # failJSON: { "time": "2013-12-13T01:11:04", "match": true, "host": "218.85.253.185" } [13/Dec/2013 01:11:04] Attempt to deliver to unknown recipient , from , IP address 218.85.253.185 +# failJSON: { "time": "2017-05-29T17:29:29", "match": true, "host": "185.140.108.56" } +[29/May/2017 17:29:29] IP address 185.140.108.56 found in DNS blacklist SpamCop, mail from to rejected +# failJSON: { "time": "2017-05-17T19:43:42", "match": true, "host": "185.140.108.26" } +[17/May/2017 19:43:42] SMTP: User printer@verinion.com doesn't exist. Attempt from IP address 185.140.108.26. +# failJSON: { "time": "2017-05-17T19:44:25", "match": true, "host": "184.171.168.211" } +[17/May/2017 19:44:25] Client with IP address 184.171.168.211 has no reverse DNS entry, connection rejected before SMTP greeting + +# failJSON: { "time": "2017-05-17T19:45:27", "match": true, "host": "170.178.167.136" } +[17/May/2017 19:45:27] Administration login into Web Administration from 170.178.167.136 failed: IP address not allowed + +# failJSON: { "time": "2017-05-17T22:14:57", "match": true, "host": "67.211.219.82" } +[17/May/2017 22:14:57] Message from IP address 67.211.219.82, sender rejected: sender domain does not exist + +# failJSON: { "time": "2017-05-18T07:25:15", "match": true, "host": "212.92.127.112" } +[18/May/2017 07:25:15] Failed SMTP login from 212.92.127.112 with SASL method CRAM-MD5. From 5214c1c5d1611e0c4a8f3c960a4b833ee767f7bc Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Tue, 30 May 2017 20:31:48 +0200 Subject: [PATCH 28/43] Update changelog (gh-1455) --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index c798768d..5f472f81 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,7 +21,7 @@ releases. ### New Features ### Enhancements - +* filter.d/kerio.conf - filter extended with new rules (see gh-1455) ver. 0.9.7 (2017/05/11) - awaiting-victory From b93e47b12f69419692fc8275358b498f01b30cbe Mon Sep 17 00:00:00 2001 From: Peter Nowee Date: Wed, 31 May 2017 15:24:21 +0200 Subject: [PATCH 29/43] dovecot: Match also when user field is empty Commit 5678d08 of 2016-11-26 changed: ( user=<\S*>,)? to: ( user=<[^>]+>,)? The change from `*` (zero or more times) to `+` (one or more times) may not have been intended. It will miss lines containing, for example: Aborted login (tried to use disallowed plaintext auth): user=<> This commit reverts the `+` back to `*`. --- config/filter.d/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index a3dc60bb..68c78c7a 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -10,7 +10,7 @@ before = common.conf _daemon = (auth|dovecot(-auth)?|auth-worker) failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ - ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ + ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$ ^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,\): unknown user\s*$ ^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,,\S*\): invalid credentials\s*$ From 8cb4ae0242986a80e1d3eedd217fafbfeee082b7 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 9 Jun 2017 13:55:30 +0200 Subject: [PATCH 30/43] Code review and small optimizations, prepared to provide offset-based time zones for date-detectors (parsing of input-string) --- fail2ban/server/datedetector.py | 15 ++++++++++++-- fail2ban/server/filter.py | 20 +++++++++---------- fail2ban/server/strptime.py | 27 ++++++++++++++------------ fail2ban/tests/datedetectortestcase.py | 11 ++++++++--- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 39a15828..90cfe1fd 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -27,6 +27,7 @@ import time from threading import Lock from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch +from .strptime import validateTimeZone from .utils import Utils from ..helpers import getLogger @@ -222,6 +223,8 @@ class DateDetector(object): self.__firstUnused = 0 # pre-match pattern: self.__preMatch = None + # default TZ (if set, treat log lines without explicit time zone to be in this time zone): + self.__default_tz = None def _appendTemplate(self, template, ignoreDup=False): name = template.name @@ -423,7 +426,15 @@ class DateDetector(object): logSys.log(logLevel, " no template.") return (None, None) - def getTime(self, line, timeMatch=None, default_tz=None): + @property + def default_tz(self): + return self.__default_tz + + @default_tz.setter + def default_tz(self, value): + self.__default_tz = validateTimeZone(value) + + def getTime(self, line, timeMatch=None): """Attempts to return the date on a log line using templates. This uses the templates' `getDate` method in an attempt to find @@ -449,7 +460,7 @@ class DateDetector(object): template = timeMatch[1] if template is not None: try: - date = template.getDate(line, timeMatch[0], default_tz=default_tz) + date = template.getDate(line, timeMatch[0], default_tz=self.__default_tz) if date is not None: if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug logSys.log(logLevel, " got time %f for %r using template %s", diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8d1eb856..a8f99998 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -34,13 +34,12 @@ from .failmanager import FailManagerEmpty, FailManager from .ipdns import DNSUtils, IPAddr from .ticket import FailTicket from .jailthread import JailThread -from .datedetector import DateDetector +from .datedetector import DateDetector, validateTimeZone from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from .utils import Utils from ..helpers import getLogger, PREFER_ENC -from .strptime import validateTimeZone # Gets the instance of the logger. logSys = getLogger(__name__) @@ -88,6 +87,8 @@ class Filter(JailThread): ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## if set, treat log lines without explicit time zone to be in this time zone + self.__logtimezone = None ## External command self.__ignoreCommand = False ## Default or preferred encoding (to decode bytes from file or journal): @@ -103,8 +104,6 @@ class Filter(JailThread): self.checkAllRegex = False ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True - ## if set, treat log lines without explicit time zone to be in this time zone - self.logtimezone = None ## Ticks counter self.ticks = 0 @@ -285,6 +284,7 @@ class Filter(JailThread): return else: dd = DateDetector() + dd.default_tz = self.__logtimezone if not isinstance(pattern, (list, tuple)): pattern = filter(bool, map(str.strip, re.split('\n+', pattern))) for pattern in pattern: @@ -316,7 +316,9 @@ class Filter(JailThread): # @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM) def setLogTimeZone(self, tz): - self.logtimezone = validateTimeZone(tz) + validateTimeZone(tz); # avoid setting of wrong value, but hold original + self.__logtimezone = tz + if self.dateDetector: self.dateDetector.default_tz = self.__logtimezone ## # Get the log default timezone @@ -324,7 +326,7 @@ class Filter(JailThread): # @return symbolic timezone (a string) def getLogTimeZone(self): - return self.logtimezone + return self.__logtimezone ## # Set the maximum retry value. @@ -640,8 +642,7 @@ class Filter(JailThread): self.__lastDate = date elif timeText: - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3], - default_tz=self.logtimezone) + dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) if dateTimeMatch is None: logSys.error("findFailure failed to parse timeText: %s", timeText) @@ -994,8 +995,7 @@ class FileFilter(Filter): if timeMatch: dateTimeMatch = self.dateDetector.getTime( line[timeMatch.start():timeMatch.end()], - (timeMatch, template), - default_tz=self.logtimezone) + (timeMatch, template)) else: nextp = container.tell() if nextp > maxp: diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index aff9db92..68ff2f41 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -27,7 +27,7 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() -FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$') +FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$') def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): """ Build century regex for last year and the next years (distance). @@ -84,30 +84,33 @@ def getTimePatternRE(): def validateTimeZone(tz): """Validate a timezone. - For now this accepts only the UTC[+-]hhmm format. + For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional). In the future, it may be extended for named time zones (such as Europe/Paris) present on the system, if a suitable tz library is present. """ + if tz is None: + return None m = FIXED_OFFSET_TZ_RE.match(tz) if m is None: raise ValueError("Unknown or unsupported time zone: %r" % tz) - return tz + tz = m.group(1) + if tz is None or tz == '': # UTC/GMT + return 0; # fixed zero offzet + return zone2offset(tz, 0) def zone2offset(tz, dt): """Return the proper offset, in minutes according to given timezone at a given time. Parameters ---------- - tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have - been validated already) - dt: datetime instance for offset computation + tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have + been validated already) + dt: datetime instance for offset computation """ - if tz == 'UTC': - return 0 - unsigned = int(tz[4:6])*60 + int(tz[6:]) - if tz[3] == '-': - return -unsigned - return unsigned + if isinstance(tz, basestring): + # [+-]1 * (hh*60 + mm) + return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5])) + return tz def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """Return time from dictionary of strptime fields diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 16515f1b..6d728b5e 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -91,19 +91,24 @@ class DateDetectorTest(LogCaptureTestCase): def testDefaultTimeZone(self): log = "2017-01-23 15:00:00" - datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300') + dd = self.datedetector + dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log) # so in UTC, it was noon self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), datetime.datetime(2017, 1, 23, 12, 0, 0)) - datelog, _ = self.datedetector.getTime(log, default_tz='UTC') + dd.default_tz='UTC'; datelog, _ = dd.getTime(log) self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), datetime.datetime(2017, 1, 23, 15, 0, 0)) + self.assertEqual(dd.default_tz, 0); # utc == 0 - datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430') + dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log) self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), datetime.datetime(2017, 1, 23, 19, 30, 0)) + self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ') + dd.default_tz = None + def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand """ From 9f41d1e3819de407a5e992bd4699566d60c1e941 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 9 Jun 2017 14:55:44 +0200 Subject: [PATCH 31/43] Normalize zone2offset (usable within reGroupDictStrptime), tests simplified and extended with more cases (covers precedence of input-zone over default, etc.) --- fail2ban/server/strptime.py | 19 +++++++------ fail2ban/tests/datedetectortestcase.py | 38 ++++++++++++++++---------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 68ff2f41..56f1c753 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -27,7 +27,7 @@ from .mytime import MyTime locale_time = LocaleTime() timeRE = TimeRE() -FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$') +FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$') def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): """ Build century regex for last year and the next years (distance). @@ -108,8 +108,15 @@ def zone2offset(tz, dt): dt: datetime instance for offset computation """ if isinstance(tz, basestring): - # [+-]1 * (hh*60 + mm) - return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5])) + if len(tz) <= 3: # short tz (hh only) + # [+-]hh --> [+-]hh*60 + return int(tz)*60 + if tz[3] != ':': + # [+-]hhmm --> [+-]1 * (hh*60 + mm) + return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5])) + else: + # [+-]hh:mm --> [+-]1 * (hh*60 + mm) + return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6])) return tz def reGroupDictStrptime(found_dict, msec=False, default_tz=None): @@ -202,11 +209,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): if z in ("Z", "UTC", "GMT"): tzoffset = 0 else: - tzoffset = int(z[1:3]) * 60 # Hours... - if len(z)>3: - tzoffset += int(z[-2:]) # ...and minutes - if z.startswith("-"): - tzoffset = -tzoffset + tzoffset = zone2offset(z, 0); # currently offset-based only elif key == 'Z': z = val if z in ("UTC", "GMT"): diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 6d728b5e..cbf1ac21 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -90,21 +90,31 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') def testDefaultTimeZone(self): - log = "2017-01-23 15:00:00" dd = self.datedetector - dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log) - # so in UTC, it was noon - self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), - datetime.datetime(2017, 1, 23, 12, 0, 0)) - - dd.default_tz='UTC'; datelog, _ = dd.getTime(log) - self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), - datetime.datetime(2017, 1, 23, 15, 0, 0)) - self.assertEqual(dd.default_tz, 0); # utc == 0 - - dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log) - self.assertEqual(datetime.datetime.utcfromtimestamp(datelog), - datetime.datetime(2017, 1, 23, 19, 30, 0)) + dt = datetime.datetime + logdt = "2017-01-23 15:00:00" + dtUTC = dt(2017, 1, 23, 15, 0) + for tz, log, desired in ( + ('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon + ('UTC', logdt, dtUTC), # UTC + ('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)), + ('GMT+12', logdt, dt(2017, 1, 23, 3, 0)), + (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework + ('UTC+0300', logdt+' GMT', dtUTC), # GMT wins + ('UTC', logdt+' GMT', dtUTC), # GMT wins + ('UTC-0430', logdt+' GMT', dtUTC), # GMT wins + (None, logdt+' GMT', dtUTC), # GMT wins + ('UTC', logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins + (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins + ('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins + (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins + (None, logdt+' Z', dtUTC), # Z wins (UTC) + ): + logSys.debug('== test %r with TZ %r', log, tz) + dd.default_tz=tz; datelog, _ = dd.getTime(log) + val = dt.utcfromtimestamp(datelog) + self.assertEqual(val, desired, + "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz)) self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ') dd.default_tz = None From 39c4acf6bd0726131bbc3848d514ec66341ce70f Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 9 Jun 2017 15:52:14 +0200 Subject: [PATCH 32/43] small amend white-spaces (no functional changes) + a bit optimized `zone2offset` --- fail2ban/server/strptime.py | 22 +++++++++++----------- fail2ban/tests/datedetectortestcase.py | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 56f1c753..2da27c58 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -107,17 +107,17 @@ def zone2offset(tz, dt): been validated already) dt: datetime instance for offset computation """ - if isinstance(tz, basestring): - if len(tz) <= 3: # short tz (hh only) - # [+-]hh --> [+-]hh*60 - return int(tz)*60 - if tz[3] != ':': - # [+-]hhmm --> [+-]1 * (hh*60 + mm) - return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5])) - else: - # [+-]hh:mm --> [+-]1 * (hh*60 + mm) - return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6])) - return tz + if isinstance(tz, int): + return tz + if len(tz) <= 3: # short tz (hh only) + # [+-]hh --> [+-]hh*60 + return int(tz)*60 + if tz[3] != ':': + # [+-]hhmm --> [+-]1 * (hh*60 + mm) + return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5])) + else: + # [+-]hh:mm --> [+-]1 * (hh*60 + mm) + return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6])) def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """Return time from dictionary of strptime fields diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index cbf1ac21..56970ac5 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -99,16 +99,16 @@ class DateDetectorTest(LogCaptureTestCase): ('UTC', logdt, dtUTC), # UTC ('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)), ('GMT+12', logdt, dt(2017, 1, 23, 3, 0)), - (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework + (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework ('UTC+0300', logdt+' GMT', dtUTC), # GMT wins ('UTC', logdt+' GMT', dtUTC), # GMT wins ('UTC-0430', logdt+' GMT', dtUTC), # GMT wins - (None, logdt+' GMT', dtUTC), # GMT wins + (None, logdt+' GMT', dtUTC), # GMT wins ('UTC', logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins - (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins + (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins ('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins - (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins - (None, logdt+' Z', dtUTC), # Z wins (UTC) + (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins + (None, logdt+' Z', dtUTC), # Z wins (UTC) ): logSys.debug('== test %r with TZ %r', log, tz) dd.default_tz=tz; datelog, _ = dd.getTime(log) From 030f89bf7a7877224e7095fdf34f898354b25f63 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 9 Jun 2017 20:29:34 +0200 Subject: [PATCH 33/43] Implemented zone abbreviations (DST, etc.) and abbr+-offset functionality (accept zones like 'CET+0100'), for the list of abbreviations see strptime.TZ_STR; Tokens `%z` and `%Z` are more precise now; Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations and/or offset-based zones; # TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching, # check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), # to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' # with wrong TZ "error", which is at least not backwards compatible. # Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations. --- fail2ban/server/strptime.py | 114 ++++++++++++++++++++----- fail2ban/tests/datedetectortestcase.py | 25 +++++- 2 files changed, 116 insertions(+), 23 deletions(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 2da27c58..dbf75d21 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -26,8 +26,9 @@ from _strptime import LocaleTime, TimeRE, _calc_julian_from_U_or_W from .mytime import MyTime locale_time = LocaleTime() -timeRE = TimeRE() -FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$') + +TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?" +FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,)) def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): """ Build century regex for last year and the next years (distance). @@ -40,10 +41,20 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo exprset |= set( cent(now[1].year + i) for i in (-1, distance) ) return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset) -#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz): -#timeRE['z'] = r"%s?(?PZ|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z'] -timeRE['Z'] = r"(?P[A-Z]{3,5})" -timeRE['z'] = r"(?PZ|UTC|GMT|[+-]\d{2}(?::?[0-5]\d)?)" +timeRE = TimeRE() + +# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching, +# check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive), +# to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' +# with wrong TZ "error", which is at least not backwards compatible. +# Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations. +timeRE['Z'] = r"(?PZ|[A-Z]{3,5})" +timeRE['z'] = r"(?PZ|UTC|GMT|[+-][01]\d(?::?\d{2})?)" + +# Note: this extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase, +# see todo above. Don't use them in default date-patterns (if not anchored, few precise resp. optional). +timeRE['ExZ'] = r"(?P%s)" % (TZ_ABBR_RE,) +timeRE['Exz'] = r"(?P(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE) # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: @@ -82,20 +93,22 @@ def getTimePatternRE(): def validateTimeZone(tz): - """Validate a timezone. + """Validate a timezone and convert it to offset if it can (offset-based TZ). - For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional). + For now this accepts the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional). + Additionally it accepts all zone abbreviations mentioned below in TZ_STR. + Note that currently this zone abbreviations are offset-based and used fixed + offset without automatically DST-switch (if CET used then no automatically CEST-switch). + In the future, it may be extended for named time zones (such as Europe/Paris) - present on the system, if a suitable tz library is present. + present on the system, if a suitable tz library is present (pytz). """ if tz is None: return None m = FIXED_OFFSET_TZ_RE.match(tz) if m is None: raise ValueError("Unknown or unsupported time zone: %r" % tz) - tz = m.group(1) - if tz is None or tz == '': # UTC/GMT - return 0; # fixed zero offzet + tz = m.groups() return zone2offset(tz, 0) def zone2offset(tz, dt): @@ -103,21 +116,29 @@ def zone2offset(tz, dt): Parameters ---------- - tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have - been validated already) - dt: datetime instance for offset computation + tz: symbolic timezone or offset (for now only TZA?([+-]hh:?mm?)? is supported, + as value are accepted: + int offset; + string in form like 'CET+0100' or 'UTC' or '-0400'; + tuple (or list) in form (zone name, zone offset); + dt: datetime instance for offset computation (currently unused) """ if isinstance(tz, int): return tz - if len(tz) <= 3: # short tz (hh only) + if isinstance(tz, basestring): + return validateTimeZone(tz) + tz, tzo = tz + if tzo is None or tzo == '': # without offset + return TZ_ABBR_OFFS[tz] + if len(tzo) <= 3: # short tzo (hh only) # [+-]hh --> [+-]hh*60 - return int(tz)*60 - if tz[3] != ':': + return TZ_ABBR_OFFS[tz] + int(tzo)*60 + if tzo[3] != ':': # [+-]hhmm --> [+-]1 * (hh*60 + mm) - return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5])) + return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[3:5])) else: # [+-]hh:mm --> [+-]1 * (hh*60 + mm) - return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6])) + return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[4:6])) def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """Return time from dictionary of strptime fields @@ -275,3 +296,56 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): if msec: # pragma: no cover - currently unused tm += fraction/1000000.0 return tm + + +TZ_ABBR_OFFS = {'':0, None:0} +TZ_STR = ''' + -12 Y + -11 X NUT SST + -10 W CKT HAST HST TAHT TKT + -9 V AKST GAMT GIT HADT HNY + -8 U AKDT CIST HAY HNP PST PT + -7 T HAP HNR MST PDT + -6 S CST EAST GALT HAR HNC MDT + -5 R CDT COT EASST ECT EST ET HAC HNE PET + -4 Q AST BOT CLT COST EDT FKT GYT HAE HNA PYT + -3 P ADT ART BRT CLST FKST GFT HAA PMST PYST SRT UYT WGT + -2 O BRST FNT PMDT UYST WGST + -1 N AZOT CVT EGT + 0 Z EGST GMT UTC WET WT + 1 A CET DFT WAT WEDT WEST + 2 B CAT CEDT CEST EET SAST WAST + 3 C EAT EEDT EEST IDT MSK + 4 D AMT AZT GET GST KUYT MSD MUT RET SAMT SCT + 5 E AMST AQTT AZST HMT MAWT MVT PKT TFT TJT TMT UZT YEKT + 6 F ALMT BIOT BTT IOT KGT NOVT OMST YEKST + 7 G CXT DAVT HOVT ICT KRAT NOVST OMSST THA WIB + 8 H ACT AWST BDT BNT CAST HKT IRKT KRAST MYT PHT SGT ULAT WITA WST + 9 I AWDT IRKST JST KST PWT TLT WDT WIT YAKT + 10 K AEST ChST PGT VLAT YAKST YAPT + 11 L AEDT LHDT MAGT NCT PONT SBT VLAST VUT + 12 M ANAST ANAT FJT GILT MAGST MHT NZST PETST PETT TVT WFT + 13 FJST NZDT + 11.5 NFT + 10.5 ACDT LHST + 9.5 ACST + 6.5 CCT MMT + 5.75 NPT + 5.5 SLT + 4.5 AFT IRDT + 3.5 IRST + -2.5 HAT NDT + -3.5 HNT NST NT + -4.5 HLV VET + -9.5 MART MIT +''' + +def _init_TZ_ABBR(): + """Initialized TZ_ABBR_OFFS dictionary (TZ -> offset in minutes)""" + for tzline in map(str.split, TZ_STR.split('\n')): + if not len(tzline): continue + tzoffset = int(float(tzline[0]) * 60) + for tz in tzline[1:]: + TZ_ABBR_OFFS[tz] = tzoffset + +_init_TZ_ABBR() diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 56970ac5..02facf30 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -90,16 +90,32 @@ class DateDetectorTest(LogCaptureTestCase): self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') def testDefaultTimeZone(self): - dd = self.datedetector + # use special date-pattern (with %Exz), because %z currently does not supported + # zone abbreviations except Z|UTC|GMT. + dd = DateDetector() + dd.appendTemplate('^%ExY-%Exm-%Exd %H:%M:%S(?: ?%Exz)?') dt = datetime.datetime logdt = "2017-01-23 15:00:00" dtUTC = dt(2017, 1, 23, 15, 0) for tz, log, desired in ( + # no TZ in input-string: ('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon ('UTC', logdt, dtUTC), # UTC ('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)), ('GMT+12', logdt, dt(2017, 1, 23, 3, 0)), (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework + # CET: + ('CET', logdt, dt(2017, 1, 23, 14, 0)), + ('+0100', logdt, dt(2017, 1, 23, 14, 0)), + ('CEST-01', logdt, dt(2017, 1, 23, 14, 0)), + # CEST: + ('CEST', logdt, dt(2017, 1, 23, 13, 0)), + ('+0200', logdt, dt(2017, 1, 23, 13, 0)), + ('CET+01', logdt, dt(2017, 1, 23, 13, 0)), + ('CET+0100', logdt, dt(2017, 1, 23, 13, 0)), + # check offset in minutes: + ('CET+0130', logdt, dt(2017, 1, 23, 12, 30)), + # TZ in input-string have precedence: ('UTC+0300', logdt+' GMT', dtUTC), # GMT wins ('UTC', logdt+' GMT', dtUTC), # GMT wins ('UTC-0430', logdt+' GMT', dtUTC), # GMT wins @@ -108,13 +124,16 @@ class DateDetectorTest(LogCaptureTestCase): (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins ('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins - (None, logdt+' Z', dtUTC), # Z wins (UTC) + ('UTC+0300', logdt+' Z', dtUTC), # Z wins (UTC) + ('GMT+12', logdt+' CET', dt(2017, 1, 23, 14, 0)), # CET wins + ('GMT+12', logdt+' CEST', dt(2017, 1, 23, 13, 0)), # CEST wins + ('GMT+12', logdt+' CET+0130', dt(2017, 1, 23, 12, 30)), # CET+0130 wins ): logSys.debug('== test %r with TZ %r', log, tz) dd.default_tz=tz; datelog, _ = dd.getTime(log) val = dt.utcfromtimestamp(datelog) self.assertEqual(val, desired, - "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz)) + "wrong offset %r != %r by %r with default TZ %r (%r)" % (val, desired, log, tz, dd.default_tz)) self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ') dd.default_tz = None From 12259bb3c7aac2e33b9cc8f601c0e556b5401bd5 Mon Sep 17 00:00:00 2001 From: Georges Racinet Date: Fri, 9 Jun 2017 20:39:03 +0200 Subject: [PATCH 34/43] man and ChangeLog for logtimezone --- ChangeLog | 2 +- man/jail.conf.5 | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index cd4776ee..7c99da91 100644 --- a/ChangeLog +++ b/ChangeLog @@ -98,7 +98,7 @@ TODO: implementing of options resp. other tasks from PR #1346 e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743), the actions having `actionflush` do not execute `actionunban` for each single ticket * add new command `actionflush` default for several iptables/iptables-ipset actions (and common include); - +* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773) ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc ----------- diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5a75369c..a43346e1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -177,6 +177,25 @@ Ensure syslog or the program that generates the log file isn't configured to com .TP .B logencoding encoding of log files used for decoding. Default value of "auto" uses current system locale. +.TP +.B logtimezone +Force the time zone for log lines that don't have one. + +If this option is not specified, log lines from which no explicit time zone has been found are interpreted by fail2ban in its own system time zone, and that may turn to be inappropriate. While the best practice is to configure the monitored applications to include explicit offsets, this option is meant to handle cases where that is not possible. + +The supported time zones in this option are those with fixed offset: Z, UTC[+-]hhmm (you can also use GMT as an alias to UTC). + +This option has no effect on log lines on which an explicit time zone has been found. +Examples: + +.RS +.nf + logtimezone = UTC + logtimezone = UTC+0200 + logtimezone = GMT-0100 +.fi +.RE + .TP .B banaction banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. From 23c2d05250db5da1cf111da96115fce0e41c9ea3 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Fri, 9 Jun 2017 20:51:28 +0200 Subject: [PATCH 35/43] Update changelog (new enhancements from gh-1792) --- ChangeLog | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7c99da91..a2eac62b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -88,17 +88,32 @@ TODO: implementing of options resp. other tasks from PR #1346 the parsing of log-entries contain new-line chars (as single entry); - if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)` could be used in regex to enable it; -* implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742): +* Implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742): - new action parameter `actionstart_on_demand` (bool) can be set to prevent/allow starting action on demand (default retrieved automatically, if some conditional parameter `param?family=...` presents in action properties), see `action.d/pf.conf` for example; - additionally `actionstop` will be executed only for families previously executing `actionstart` (starting on demand only) -* introduced new command `actionflush`: executed in order to flush all bans at once +* Introduced new command `actionflush`: executed in order to flush all bans at once e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743), the actions having `actionflush` do not execute `actionunban` for each single ticket -* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include); -* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773) +* Add new command `actionflush` default for several iptables/iptables-ipset actions (and common include); +* Add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773) +* Implemented zone abbreviations (like CET, CEST, etc.) and abbr+-offset functionality (accept zones + like 'CET+0100'), for the list of abbreviations see strptime.TZ_STR; +* Tokens `%z` and `%Z` are changed (more precise now); +* Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations and/or offset-based + zones (implemented as enhancement using custom `datepattern`, because may be too dangerous for default + patterns and tokens like `%z`); + Note: the extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase. + Don't use them in default date-patterns (if not anchored, few precise resp. optional). + Because python currently does not support mixing of case-sensitive with case-insensitive matching, + the TZ (in uppercase) cannot be combined with `%a`/`%b` etc (that are currently case-insensitive), + to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' with + wrong TZ "error". + Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone + abbreviations. + ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc ----------- From 6f3d425c4d4d8d86faa67a3e058e1426c74a7680 Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 12 Jun 2017 10:53:28 -0700 Subject: [PATCH 36/43] Update postfix filters and tests --- ChangeLog | 6 ++++-- THANKS | 1 + config/filter.d/postfix.conf | 6 +++--- fail2ban/tests/files/logs/postfix | 9 +++++++++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 33a0c926..54d48140 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,10 +22,12 @@ TODO: implementing of options resp. other tasks from PR #1346 (by using tag `` instead of buffering with `maxlines`); - optional parameter `mode` rewritten: normal (default), ddos, extra or aggressive (combines all), see sshd for regex details) -* filter.d/sendmail-reject.conf: +* `filter.d/sendmail-reject.conf`: - rewritten using `prefregex` and used MLFID-related multi-line parsing; - optional parameter `mode` introduced: normal (default), extra or aggressive -* filter.d/haproxy-http-auth: do not mistake client port for part of an IPv6 address (gh-1745) +* `filter.d/haproxy-http-auth`: do not mistake client port for part of an IPv6 address (gh-1745) +* `filter.d/postfix-sasl.conf` + - updated to latest postfix formats * `action.d/complain.conf` - fixed using new tag `` (sh/dash compliant now) * `action.d/sendmail-geoip-lines.conf` diff --git a/THANKS b/THANKS index 8cc8f7bd..7861ceb5 100644 --- a/THANKS +++ b/THANKS @@ -61,6 +61,7 @@ John Thoe Jacques Lav!gnotte Johannes Weberhofer Jason H Martin +Jeaye Wilkerson Jisoo Park Joel M Snyder Jonathan Kamens diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 005ed585..1dbc23c0 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,12 +15,12 @@ _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) .+$ failregex = ^RCPT from \S+\[\]: 554 5\.7\.1 - ^RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ - ^RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*> + ^RCPT from \S+\[\]: 450 4\.7\.1 (<\S+>)?: Helo command rejected: Host not found; from=<\S*> to=<\S*> proto=\S+ helo=.*$ ^EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; ^VRFY from \S+\[\]: 550 5\.1\.1 ^RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ - ^after \S+ from [^[]*\[\]:?$ + ^after \S+ from [^[]*\[\]:? ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 78f72287..c38ba311 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -35,3 +35,12 @@ Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s27 # failJSON: { "time": "2005-04-12T02:24:11", "match": true , "host": "62.138.2.143" } Apr 12 02:24:11 xxx postfix/smtps/smtpd[42]: NOQUEUE: reject: EHLO from astra4139.startdedicated.de[62.138.2.143]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= + +# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" } +Jun 12 08:58:35 xxx postfix/smtpd[27296]: NOQUEUE: reject: RCPT from unknown[1.2.3.4]: 450 4.7.1 Client host rejected: cannot find your reverse hostname, [2.3.4.5]; from= to= proto=ESMTP helo= + +# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" } +Jun 12 08:58:35 xxx postfix/smtpd[2931]: NOQUEUE: reject: RCPT from unknown[1.2.3.4]: 450 4.7.1 : Helo command rejected: Host not found; from= to= proto=SMTP helo= + +# failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" } +Jun 12 08:58:35 xxx postfix/smtpd[13533]: improper command pipelining after AUTH from unknown[1.2.3.4]: QUIT From 2b358bc1a4c985c07120d8b4bc7e025339926bdc Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 15 Jun 2017 11:16:19 +0200 Subject: [PATCH 37/43] filter.d/apache-overflows.conf: rewritten without end-anchor ($), because apache-log could contain very long URLs (and/or referrer), the parsing of it anchored way may be very vulnerable (at least as regards the system resources, see gh-1790). --- config/filter.d/apache-overflows.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 74e44b8e..02a2ef20 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,11 +8,15 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)(, referer: \S+)?$ +failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) ignoreregex = # DEV Notes: +# +# [sebres] Because this apache-log could contain very long URLs (and/or referrer), +# the parsing of it anchored way may be very vulnerable (at least as regards +# the system resources, see gh-1790). Thus rewritten without end-anchor ($). # # fgrep -r 'URI too long' httpd-2.* # httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line); From e1234a5249d7d9ca604970d80c54353a86b60aba Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 15 Jun 2017 11:40:07 +0200 Subject: [PATCH 38/43] ChangeLog update --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5f472f81..e99bab04 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ releases. ### Fixes * Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991. Partially back-ported from v.0.10. +* filter.d/apache-overflows.conf: + - Fixes resources greedy expression (see gh-1790); + - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs. ### New Features From fd32e908e31add6f5eed677ce2c2f3e77e50ee49 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 15 Jun 2017 18:28:37 +0200 Subject: [PATCH 39/43] fixes restoring of tickets from database for jails with persistent ban (if `bantime = -1`) --- fail2ban/server/database.py | 2 +- fail2ban/tests/databasetestcase.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 77786f57..f4f9b6c2 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -593,7 +593,7 @@ class Fail2BanDb(object): if ip is not None: query += " AND ip=?" queryArgs.append(ip) - if forbantime is not None: + if forbantime not in (None, -1): # not specified or persistent (all) query += " AND timeofban > ?" queryArgs.append(fromtime - forbantime) if ip is None: diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index b1c68a8f..cbfc1517 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -358,6 +358,19 @@ class DatabaseTest(LogCaptureTestCase): self.assertEqual(len(tickets), 2) ticket = self.db.getCurrentBans(jail=None, ip="127.0.0.1"); self.assertEqual(ticket.getIP(), "127.0.0.1") + + # positive case (1 ticket not yet expired): + tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15, + fromtime=MyTime.time()) + self.assertEqual(len(tickets), 1) + # negative case (all are expired in 1year): + tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15, + fromtime=MyTime.time() + MyTime.str2seconds("1year")) + self.assertEqual(len(tickets), 0) + # persistent bantime (-1), so never expired: + tickets = self.db.getCurrentBans(jail=self.jail, forbantime=-1, + fromtime=MyTime.time() + MyTime.str2seconds("1year")) + self.assertEqual(len(tickets), 2) def testActionWithDB(self): # test action together with database functionality From efeca8fdeb7470ba5605762a8b9a97fa56943204 Mon Sep 17 00:00:00 2001 From: "Serg G. Brester" Date: Mon, 19 Jun 2017 16:25:46 +0200 Subject: [PATCH 40/43] postfix.conf: removes unneeded end-anchoring like `.*$`, etc. also removes several dynamic content at end, which are of no avail there. Additionally normalizes optional part (mail-ID) after reason number. --- config/filter.d/postfix.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 1dbc23c0..1cf54f14 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,11 +15,11 @@ _daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] prefregex = ^%(__prefix_line)s(?:NOQUEUE: reject:|improper command pipelining) .+$ failregex = ^RCPT from \S+\[\]: 554 5\.7\.1 - ^RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*> - ^RCPT from \S+\[\]: 450 4\.7\.1 (<\S+>)?: Helo command rejected: Host not found; from=<\S*> to=<\S*> proto=\S+ helo=.*$ - ^EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; + ^RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your (reverse )?hostname\b + ^RCPT from \S+\[\]: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b + ^EHLO from \S+\[\]: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b ^VRFY from \S+\[\]: 550 5\.1\.1 - ^RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ + ^RCPT from \S+\[\]: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b ^after \S+ from [^[]*\[\]:? ignoreregex = From 691c080dc747ba159bb3e68b1a7725a4ab2f23cc Mon Sep 17 00:00:00 2001 From: Johannes Weberhofer Date: Fri, 19 May 2017 15:23:58 +0200 Subject: [PATCH 41/43] Added roundcube authentication filter, new jail and log-examples --- config/filter.d/roundcube.conf | 21 +++++++++++++++++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/roundcube | 4 ++++ 3 files changed, 29 insertions(+) create mode 100644 config/filter.d/roundcube.conf create mode 100644 fail2ban/tests/files/logs/roundcube diff --git a/config/filter.d/roundcube.conf b/config/filter.d/roundcube.conf new file mode 100644 index 00000000..9665883b --- /dev/null +++ b/config/filter.d/roundcube.conf @@ -0,0 +1,21 @@ +# Fail2Ban configuration file for roundcube web server authentication failures +# +# This filter needs "$config['log_driver']" set to "syslog" in the roundcube configuration +# + +[INCLUDES] + +before = common.conf + +[Definition] + +failregex = ^%(__prefix_line)sroundcube\[(\d*)\]: <\S* IMAP Error: Login failed for (\S*) from \..*$ + +ignoreregex = + +[Init] + +backend = systemd + +journalmatch = SYSLOG_IDENTIFIER=roundcube + diff --git a/config/jail.conf b/config/jail.conf index 135c9a2b..bd9ba876 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -379,6 +379,10 @@ logpath = %(lighttpd_error_log)s # Webmail and groupware servers # +[roundcube] +port = http,https +backend = %(syslog_backend)s + [roundcube-auth] port = http,https diff --git a/fail2ban/tests/files/logs/roundcube b/fail2ban/tests/files/logs/roundcube new file mode 100644 index 00000000..b491fa2b --- /dev/null +++ b/fail2ban/tests/files/logs/roundcube @@ -0,0 +1,4 @@ +May 19 06:07:48 server roundcube[21296]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) +May 19 06:11:37 server roundcube[22926]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) +May 19 06:13:18 server roundcube[21528]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) +May 19 06:36:53 server roundcube[27572]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) From d3ae70beb6f73f289843157307f079678c156457 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Jun 2017 18:05:29 +0200 Subject: [PATCH 42/43] filter.d/roundcube-auth.conf: Use the same filter-file and jail also when logging errors to journal instead to a local file. Additionally fixes more complex injections on username. --- config/filter.d/roundcube-auth.conf | 6 ++++-- config/filter.d/roundcube.conf | 21 --------------------- config/jail.conf | 6 ++---- fail2ban/tests/files/logs/roundcube | 4 ---- fail2ban/tests/files/logs/roundcube-auth | 19 +++++++++++++++---- 5 files changed, 21 insertions(+), 35 deletions(-) delete mode 100644 config/filter.d/roundcube.conf delete mode 100644 fail2ban/tests/files/logs/roundcube diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index 886cf2d6..bab62651 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -13,8 +13,10 @@ before = common.conf [Definition] -failregex = ^\s*(\[\])?(%(__hostname)s\s*(roundcube:)?\s*(<[\w]+>)? IMAP Error)?: (FAILED login|Login failed) for .*? from (\. .* in .*?/rcube_imap\.php on line \d+ \(\S+ \S+\))?$ - ^\[\]:\s*(<[\w]+>)? Failed login for [\w\-\.\+]+(@[\w\-\.\+]+\.[a-zA-Z]{2,6})? from in session \w+( \(error: \d\))?$ +prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+>)? IMAP Error)?: .+$ + +failregex = ^(?:FAILED login|Login failed) for .* from (\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$ + ^(?:<[\w]+> )?Failed login for .* from in session \w+( \(error: \d\))?$ ignoreregex = # DEV Notes: diff --git a/config/filter.d/roundcube.conf b/config/filter.d/roundcube.conf deleted file mode 100644 index 9665883b..00000000 --- a/config/filter.d/roundcube.conf +++ /dev/null @@ -1,21 +0,0 @@ -# Fail2Ban configuration file for roundcube web server authentication failures -# -# This filter needs "$config['log_driver']" set to "syslog" in the roundcube configuration -# - -[INCLUDES] - -before = common.conf - -[Definition] - -failregex = ^%(__prefix_line)sroundcube\[(\d*)\]: <\S* IMAP Error: Login failed for (\S*) from \..*$ - -ignoreregex = - -[Init] - -backend = systemd - -journalmatch = SYSLOG_IDENTIFIER=roundcube - diff --git a/config/jail.conf b/config/jail.conf index bd9ba876..21bc898e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -379,14 +379,12 @@ logpath = %(lighttpd_error_log)s # Webmail and groupware servers # -[roundcube] -port = http,https -backend = %(syslog_backend)s - [roundcube-auth] port = http,https logpath = %(roundcube_errors_log)s +# Use following line in your jail.local if roundcube logs to journal. +#backend = %(syslog_backend)s [openwebmail] diff --git a/fail2ban/tests/files/logs/roundcube b/fail2ban/tests/files/logs/roundcube deleted file mode 100644 index b491fa2b..00000000 --- a/fail2ban/tests/files/logs/roundcube +++ /dev/null @@ -1,4 +0,0 @@ -May 19 06:07:48 server roundcube[21296]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) -May 19 06:11:37 server roundcube[22926]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) -May 19 06:13:18 server roundcube[21528]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) -May 19 06:36:53 server roundcube[27572]: IMAP Error: Login failed for test from 178.191.91.44. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) diff --git a/fail2ban/tests/files/logs/roundcube-auth b/fail2ban/tests/files/logs/roundcube-auth index 26868c3e..8c491b38 100644 --- a/fail2ban/tests/files/logs/roundcube-auth +++ b/fail2ban/tests/files/logs/roundcube-auth @@ -8,19 +8,27 @@ Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 1. # Made up to attempts to inject a DoS on the server. Assume the user can manipulate the IMAP error response # # user = admin from 127.0.0.1 -# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 1" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# user = admin from 127.0.0.1. +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 1 (with dot)" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1. from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# # # IMAP server logs user=${username} -# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 2" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) # +# IMAP server logs user=${username} +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 2 (with dot)" } +Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1. from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1. in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) +# # Old roundcube version - no IMAP response -# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 3" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 from 1.2.3.4 # # user = admin from 127.0.0.1 in -# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-07-11T03:06:37", "match": true , "host": "1.2.3.4", "desc": "Injecting on username 4" } Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 127.0.0.1 in from 1.2.3.4. AUTHENTICATE PLAIN: A0002 NO Login failed. user=admin from 127.0.0.1 in in /usr/share/roundcube/program/include/rcube_imap.php on line 205 (POST /wmail/?_task=login&_action=login) # Roundcube 1.0.5 CentOS 6 (/var/log/roundcubemail/errors) @@ -40,3 +48,6 @@ Jul 11 03:06:37 myhostname roundcube: IMAP Error: Login failed for admin from 12 # Roundcube 1.1.1 (/var/log/roundcubemail/userlogins) # failJSON: { "time": "2015-05-10T19:02:52", "match": true , "host": "1.2.3.4" } [10-May-2015 13:02:52 -0400]: <4z506z6r> Failed login for admin@example.com from 1.2.3.4 in session 4z506z6rvddstv6k7jz08hxo27 (error: 0) + +# failJSON: { "time": "2005-05-19T06:07:48", "match": true , "host": "192.0.2.1", "desc": "Roundcube logged to journald instead to a local file."} +May 19 06:07:48 server roundcube[21296]: IMAP Error: Login failed for test from 192.0.2.1. AUTHENTICATE PLAIN: Authentication failed. in /usr/share/php5/Roundcube/rcube_imap.php on line 193 (POST /mail/?_task=login&_action=login) From 9b0f39a17d32b2f78738c8500123cf01b56b550b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 19 Jun 2017 18:07:48 +0200 Subject: [PATCH 43/43] ChangeLog updated --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index c53989f6..cf4411cf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,9 @@ TODO: implementing of options resp. other tasks from PR #1346 * `filter.d/haproxy-http-auth`: do not mistake client port for part of an IPv6 address (gh-1745) * `filter.d/postfix-sasl.conf` - updated to latest postfix formats +* `filter.d/roundcube-auth.conf`: + - fixed regex when logging authentication errors to journal instead to a local file (gh-1159); + - additionally fixed more complex injections on username (e. g. using dot after fake host). * `action.d/complain.conf` - fixed using new tag `` (sh/dash compliant now) * `action.d/sendmail-geoip-lines.conf`