Merge branch '0.11' into wc/debian

ChangeLog

@@ -6,7 +6,7 @@
 Fail2Ban: Changelog
 ===================
 
-ver. 0.11.2-dev (20??/??/??) - development edition
+ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools
 -----------
 
 ### Compatibility:
@@ -61,6 +61,9 @@ ver. 0.11.2-dev (20??/??/??) - development edition
 * `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836)
 * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`
   should be interpolated in definition section (inside the filter-config, gh-2650)
+* `filter.d/dovecot.conf`: 
+  - add managesieve and submission support (gh-2795);
+  - accept messages with more verbose logging (gh-2573);
 * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697)
 * `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle 
   the match of username differently (gh-2693):

MANIFEST

@@ -100,6 +100,8 @@ config/filter.d/exim.conf
 config/filter.d/exim-spam.conf
 config/filter.d/freeswitch.conf
 config/filter.d/froxlor-auth.conf
+config/filter.d/gitlab.conf
+config/filter.d/grafana.conf
 config/filter.d/groupoffice.conf
 config/filter.d/gssftpd.conf
 config/filter.d/guacamole.conf
@@ -139,6 +141,7 @@ config/filter.d/sendmail-auth.conf
 config/filter.d/sendmail-reject.conf
 config/filter.d/sieve.conf
 config/filter.d/slapd.conf
+config/filter.d/softethervpn.conf
 config/filter.d/sogo-auth.conf
 config/filter.d/solid-pop3d.conf
 config/filter.d/squid.conf
@@ -267,6 +270,8 @@ fail2ban/tests/files/database_v1.db
 fail2ban/tests/files/database_v2.db
 fail2ban/tests/files/filter.d/substition.conf
 fail2ban/tests/files/filter.d/testcase01.conf
+fail2ban/tests/files/filter.d/testcase02.conf
+fail2ban/tests/files/filter.d/testcase02.local
 fail2ban/tests/files/filter.d/testcase-common.conf
 fail2ban/tests/files/ignorecommand.py
 fail2ban/tests/files/logs/3proxy
@@ -301,6 +306,8 @@ fail2ban/tests/files/logs/exim
 fail2ban/tests/files/logs/exim-spam
 fail2ban/tests/files/logs/freeswitch
 fail2ban/tests/files/logs/froxlor-auth
+fail2ban/tests/files/logs/gitlab
+fail2ban/tests/files/logs/grafana
 fail2ban/tests/files/logs/groupoffice
 fail2ban/tests/files/logs/gssftpd
 fail2ban/tests/files/logs/guacamole
@@ -338,6 +345,7 @@ fail2ban/tests/files/logs/sendmail-auth
 fail2ban/tests/files/logs/sendmail-reject
 fail2ban/tests/files/logs/sieve
 fail2ban/tests/files/logs/slapd
+fail2ban/tests/files/logs/softethervpn
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/squid

config/filter.d/apache-noscript.conf

@@ -17,9 +17,9 @@ before = apache-common.conf
 
 [Definition]
 
-script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl)
+script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/)
 
-prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^(?:does not exist|not found or unable to stat): <script>\b
             ^'<script>\S*' not found or unable to stat

config/filter.d/dovecot.conf

@@ -10,15 +10,15 @@ before = common.conf
 _auth_worker = (?:dovecot: )?auth(?:-worker)?
 _daemon = (?:dovecot(?:-auth)?|auth)
 
-prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
             ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
-            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$
+            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)
             <mdre-<mode>>
 
-mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
 
 mdre-normal = 
 

config/filter.d/phpmyadmin-syslog.conf

@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the phpMyAdmin-syslog
+# Fail2Ban filter for the phpMyAdmin-syslog
 #
 
 [INCLUDES]

config/filter.d/proftpd.conf

@@ -1,4 +1,4 @@
-# Fail2Ban fitler for the Proftpd FTP daemon
+# Fail2Ban filter for the Proftpd FTP daemon
 #
 # Set "UseReverseDNS off" in proftpd.conf to avoid the need for DNS.
 # See: http://www.proftpd.org/docs/howto/DNS.html

fail2ban/tests/files/logs/apache-noscript

@@ -20,3 +20,6 @@
 [Sun Mar 11 08:56:20.913548 2018] [proxy_fcgi:error] [pid 742:tid 140142593419008] [client 192.0.2.106:50900] AH01071: Got error 'Primary script unknown\n'
 # failJSON: { "time": "2019-07-09T14:27:42", "match": true , "host": "127.0.0.1", "desc": "script unknown, without \n (gh-2466)" }
 [Tue Jul 09 14:27:42.650548 2019] [proxy_fcgi:error] [pid 22075:tid 140322524440320] [client 127.0.0.1] AH01071: Got error 'Primary script unknown'
+
+# failJSON: { "time": "2020-08-11T08:56:17", "match": true , "host": "192.0.2.1", "desc": "script not found with AH02811 and cgi-bin path segment in script (gh-2805)" }
+[Tue Aug 11 08:56:17.580412 2020] [cgi:error] [pid 27550:tid 140110750279424] [client 192.0.2.1:18071] AH02811: script not found or unable to stat: /usr/lib/cgi-bin/kerbynet

fail2ban/tests/files/logs/dovecot

@@ -43,9 +43,15 @@ Jan 29 05:13:50 mail dovecot: auth: passwd-file(username,1.2.3.4): unknown user
 # failJSON: { "time": "2005-01-29T13:54:06", "match": true , "host": "192.0.2.5" }
 Jan 29 13:54:06 auth-worker(22401): Info: sql(admin@example.de,192.0.2.5,<n4JLdHNVngZGpV2j>): unknown user
 
+#failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573" }
+Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,<LZmGp6mZaMrAqLIZ>): unknown user (SHA1 of given password: f638ff)
+
 #failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.144.226" }
 Jun 11 13:57:17 main dovecot: auth: sql(admin@example.ru,192.168.144.226,<6rXunFtu493AqJDi>): Password mismatch
 
+#failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573"  }
+Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,<LZmGp6mZaMrAqLIZ>): Password mismatch (for LDAP bind) (SHA1 of given password: f638ff)
+
 # failJSON: { "time": "2005-01-29T14:38:51", "match": true , "host": "192.0.2.6", "desc": "PAM Permission denied (gh-1897)" }
 Jan 29 14:38:51 example.com dovecot[24941]: auth-worker(30165): pam(user@example.com,192.0.2.6,<PNHQq8pZhqIKAQGd>): pam_authenticate() failed: Permission denied
 
@@ -94,6 +100,13 @@ Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid com
 # failJSON: { "time": "2004-08-28T06:38:51", "match": true , "host": "192.0.2.3" }
 Aug 28 06:38:51 s166-62-100-187 dovecot: imap-login: Disconnected (auth failed, 1 attempts in 9 secs): user=<administrator@example.com>, method=PLAIN, rip=192.0.2.3, lip=192.168.1.2, TLS: Disconnected, TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)
 
+# failJSON: { "time": "2004-08-29T03:17:18", "match": true , "host": "192.0.2.133" }
+Aug 29 03:17:18 server dovecot: submission-login: Client has quit the connection (auth failed, 1 attempts in 2 secs): user=<user1>, method=LOGIN, rip=192.0.2.133, lip=0.0.0.0
+# failJSON: { "time": "2004-08-29T03:53:52", "match": true , "host": "192.0.2.169" }
+Aug 29 03:53:52 server dovecot: submission-login: Remote closed connection (auth failed, 1 attempts in 2 secs): user=<user4>, method=PLAIN, rip=192.0.2.169, lip=0.0.0.0
+# failJSON: { "time": "2004-08-29T15:33:53", "match": true , "host": "192.0.2.100" }
+Aug 29 15:33:53 server dovecot: managesieve-login: Disconnected: Too many invalid commands. (auth failed, 1 attempts in 2 secs): user=<myself>, method=PLAIN, rip=192.0.2.100, lip=0.0.0.0, TLS, TLSv1.3 with cipher TLS_CHACHA20_POLY1305_SHA256 (256/256 bits)
+
 # ---------------------------------------
 # Test-cases of aggressive mode:
 # ---------------------------------------

fail2ban/tests/samplestestcase.py

@@ -296,7 +296,7 @@ def testSampleRegexsFactory(name, basedir):
 							regexsUsedRe.add(regexList[failregex])
 					except AssertionError as e: # pragma: no cover
 						import pprint
-						raise AssertionError("%s: %s on: %s:%i, line:\n %sregex (%s):\n %s\n"
+						raise AssertionError("%s: %s on: %s:%i, line:\n  %s\nregex (%s):\n  %s\n"
 							"faildata: %s\nfail: %s" % (
 								fltName, e, logFile.filename(), logFile.filelineno(), 
 								line, failregex, regexList[failregex] if failregex != -1 else None,

fail2ban/version.py

@@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black"
 __license__ = "GPL-v2+"
 
-version = "0.11.2-dev"
+version = "0.11.2"
 
 def normVersion():
   """ Returns fail2ban version in normalized machine-readable format"""

man/fail2ban-client.1

@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-CLIENT "1" "February 2020" "fail2ban-client v0.11.2-dev" "User Commands"
+.TH FAIL2BAN-CLIENT "1" "November 2020" "fail2ban-client v0.11.2" "User Commands"
 .SH NAME
 fail2ban-client \- configure and control the server
 .SH SYNOPSIS
 .B fail2ban-client
 [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR
 .SH DESCRIPTION
-Fail2Ban v0.11.2\-dev reads log file that contains password failure report
+Fail2Ban v0.11.2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP

man/fail2ban-python.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-PYTHON "1" "February 2020" "fail2ban-python 0.11.2-dev" "User Commands"
+.TH FAIL2BAN-PYTHON "1" "November 2020" "fail2ban-python 0.11.2" "User Commands"
 .SH NAME
 fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used
 .SH DESCRIPTION

man/fail2ban-regex.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-REGEX "1" "February 2020" "fail2ban-regex 0.11.2-dev" "User Commands"
+.TH FAIL2BAN-REGEX "1" "November 2020" "fail2ban-regex 0.11.2" "User Commands"
 .SH NAME
 fail2ban-regex \- test Fail2ban "failregex" option
 .SH SYNOPSIS

man/fail2ban-server.1

@@ -1,12 +1,12 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-SERVER "1" "February 2020" "fail2ban-server v0.11.2-dev" "User Commands"
+.TH FAIL2BAN-SERVER "1" "November 2020" "fail2ban-server v0.11.2" "User Commands"
 .SH NAME
 fail2ban-server \- start the server
 .SH SYNOPSIS
 .B fail2ban-server
 [\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-Fail2Ban v0.11.2\-dev reads log file that contains password failure report
+Fail2Ban v0.11.2 reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 .SH OPTIONS
 .TP

man/fail2ban-testcases.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.47.4.
-.TH FAIL2BAN-TESTCASES "1" "February 2020" "fail2ban-testcases 0.11.2-dev" "User Commands"
+.TH FAIL2BAN-TESTCASES "1" "November 2020" "fail2ban-testcases 0.11.2" "User Commands"
 .SH NAME
 fail2ban-testcases \- run Fail2Ban unit-tests
 .SH SYNOPSIS

man/jail.conf.5

@@ -276,6 +276,9 @@ It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "pol
 .B usedns
 use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged)..
 .TP
+.B prefregex
+regex (Python \fBreg\fRular \fBex\fRpression) to parse a common part containing in every message (see \fBprefregex\fR in section FILTER FILES for details).
+.TP
 .B failregex
 regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes (see \fBfailregex\fR in section FILTER FILES for details). If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below).
 .TP
@@ -432,7 +435,36 @@ These are used to identify failed authentication attempts in log files and to ex
 
 Like action files, filter files are ini files. The main section is the [Definition] section.
 
-There are two filter definitions used in the [Definition] section:
+There are several standard filter definitions used in the [Definition] section:
+.TP
+.B prefregex
+is the regex (\fBreg\fRular \fBex\fRpression) to parse a common part containing in every message, which is applied after \fBdatepattern\fR found a match, before the search for any \fBfailregex\fR or \fBignoreregex\fR would start.
+.br
+If this regex doesn't match the process is starting immediately with next message and search for any \fBfailregex\fR does not occur.
+.br
+If \fBprefregex\fR contains \fI<F-CONTENT>...</F-CONTENT>\fR, the part of message enclosed between this tags will be extracted and herafter used as whole message for search with \fBfailregex\fR or \fBignoreregex\fR.
+.IP
+For example:
+.nf
+        prefregex = ^%(__prefix_line)s (?:ERROR|FAILURE) <F-CONTENT>.+</F-CONTENT>$
+        failregex = ^user not found
+                    ^authentication failed
+                    ^unknown authentication method
+.fi
+.IP
+You can use \fBprefregex\fR in order to:
+.RS
+.IP
+- specify 1 common regex to match some common part present in every messages (do avoid unneeded match in every \fBfailregex\fR if you have more as one);
+.IP
+- to cut some interesting part of message only (to simplify \fBfailregex\fR) enclosed between tags \fI<F-CONTENT>\fI and \fI</F-CONTENT>\fR;
+.IP
+- to gather some failure identifier (e. g. some prefix matched by \fI<F-MLFID>...<F-MLFID/>\fR tag) to identify several messages belonging to same session, where a connect message containing IP followed by failure message(s) that are not contain IP;
+this provides a new multi-line parsing method as replacement for old (slow an ugly) multi-line parsing using buffering window (\fImaxlines\fR > 1 and \fI<SKIPLINES>\fR);
+.IP
+- to ignore some wrong, too long or even unneeded messages (a.k.a. parasite log traffic) which can be also present in journal, before \fBfailregex\fR search would take place.
+.RE
+
 .TP
 .B failregex
 is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The standard replacement tags can be used as part of the regex:
@@ -451,17 +483,18 @@ is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. T
 \fI<CIDR>\fR - helper regex to match CIDR (simple integer form of net-mask).
 .IP
 \fI<SUBNET>\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional).
+.PP
+\fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message).
+.PP
+For multiline regexs (parsing with \fImaxlines\fR greater that 1) the tag \fI<SKIPLINES>\fR can be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times.
+.br
+This is an obsolete handling and if the lines contain some common identifier, better would be to use new handling (with tags \fI<F-MLFID>...<F-MLFID/>\fR).
 .RE
-.TP
-For multiline regexs the tag \fI<SKIPLINES>\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times.
 
 .TP
 .B ignoreregex
 is the regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex.
 
-
-.PP
-Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options:
 .TP
 \fBmaxlines\fR
 specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to.
@@ -492,7 +525,9 @@ There are several prefixes and words with special meaning that could be specifie
 \fBjournalmatch\fR
 specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend.
 .PP
-Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR.
+Similar to actions, filters may have an [Init] section also (optional since v.0.10). All parameters of both sections [Definition] and [Init] can be overridden (redefined or extended) in \fIjail.conf\fR or \fIjail.local\fR (or in related \fIfilter.d/filter-name.local\fR).
+Every option supplied in the jail to the filter overwrites the value specified in [Init] section, which in turm would overwrite the value in [Definition] section.
+Besides the standard settings of filter both sections can be used to initialize filter-specific options.
 
 Filters can also have a section called [INCLUDES]. This is used to read other configuration files.