From 1cefb7fdc6f160c300bfb7fe329b4402703cb2a7 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 14:40:48 +0100 Subject: [PATCH 01/14] setup.py: small amend to d2d3762ba9fa82b7983bae74cd567702e5c1b96c: allow build without tests also (both build and install accepting `--without-tests` option now) --- setup.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index ce1eedf6..2e2a77fa 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ source_dir = os.path.realpath(os.path.dirname( sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ )) +with_tests = True + # Wrapper to install python binding (to current python version): class install_scripts_f2b(install_scripts): @@ -123,7 +125,7 @@ class install_command_f2b(install): ] def initialize_options(self): self.disable_2to3 = None - self.without_tests = None + self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): global _2to3 @@ -168,6 +170,12 @@ elif "test" in sys.argv: print("python distribute required to execute fail2ban tests") print("") +# if build without tests: +if "build" in sys.argv: + if "--without-tests" in sys.argv: + with_tests = False + sys.argv.remove("--without-tests") + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -224,16 +232,18 @@ setup( 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper - ], + ] + [ + 'bin/fail2ban-testcases', + ] if with_tests else [], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + ] + [ 'fail2ban.tests', 'fail2ban.tests.action_d', - ], + ] if with_tests else [], package_data = { 'fail2ban.tests': [ join(w[0], f).replace("fail2ban/tests/", "", 1) @@ -245,7 +255,7 @@ setup( [ join(w[0], f).replace("fail2ban/tests/", "", 1) for w in os.walk('fail2ban/tests/action_d') for f in w[2]] - }, + } if with_tests else {}, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From a107a8e7d2f1e55e317425cc12acbd813d9e6dee Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 14:59:45 +0100 Subject: [PATCH 02/14] setup.py: cherry-pick from 0.11 (option --without-tests) --- setup.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 8da29268..2e2a77fa 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ source_dir = os.path.realpath(os.path.dirname( sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ )) +with_tests = True + # Wrapper to install python binding (to current python version): class install_scripts_f2b(install_scripts): @@ -119,9 +121,11 @@ class install_scripts_f2b(install_scripts): class install_command_f2b(install): user_options = install.user_options + [ ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), + ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): self.disable_2to3 = None + self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): global _2to3 @@ -132,6 +136,13 @@ class install_command_f2b(install): cmdclass = self.distribution.cmdclass cmdclass['build_py'] = build_py_2to3 cmdclass['build_scripts'] = build_scripts_2to3 + if self.without_tests: + self.distribution.scripts.remove('bin/fail2ban-testcases') + + self.distribution.packages.remove('fail2ban.tests') + self.distribution.packages.remove('fail2ban.tests.action_d') + + del self.distribution.package_data['fail2ban.tests'] install.finalize_options(self) def run(self): install.run(self) @@ -159,6 +170,12 @@ elif "test" in sys.argv: print("python distribute required to execute fail2ban tests") print("") +# if build without tests: +if "build" in sys.argv: + if "--without-tests" in sys.argv: + with_tests = False + sys.argv.remove("--without-tests") + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -208,23 +225,25 @@ setup( license = "GPL", platforms = "Posix", cmdclass = { - 'build_py': build_py, 'build_scripts': build_scripts, + 'build_py': build_py, 'build_scripts': build_scripts, 'install_scripts': install_scripts_f2b, 'install': install_command_f2b }, scripts = [ 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper - ], + ] + [ + 'bin/fail2ban-testcases', + ] if with_tests else [], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + ] + [ 'fail2ban.tests', 'fail2ban.tests.action_d', - ], + ] if with_tests else [], package_data = { 'fail2ban.tests': [ join(w[0], f).replace("fail2ban/tests/", "", 1) @@ -236,7 +255,7 @@ setup( [ join(w[0], f).replace("fail2ban/tests/", "", 1) for w in os.walk('fail2ban/tests/action_d') for f in w[2]] - }, + } if with_tests else {}, data_files = [ ('/etc/fail2ban', glob("config/*.conf") From 5d0e74d2ab8980356b6a6cb2760b43945e4e34b3 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 16:55:55 +0100 Subject: [PATCH 03/14] man/jail.conf.5: documentation extended (prefregex, etc) closes gh-2820 --- man/jail.conf.5 | 49 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index d7722124..dc226ac2 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -276,6 +276,9 @@ It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "pol .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. .TP +.B prefregex +regex (Python \fBreg\fRular \fBex\fRpression) to parse a common part containing in every message (see \fBprefregex\fR in section FILTER FILES for details). +.TP .B failregex regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes (see \fBfailregex\fR in section FILTER FILES for details). If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below). .TP @@ -432,7 +435,36 @@ These are used to identify failed authentication attempts in log files and to ex Like action files, filter files are ini files. The main section is the [Definition] section. -There are two filter definitions used in the [Definition] section: +There are several standard filter definitions used in the [Definition] section: +.TP +.B prefregex +is the regex (\fBreg\fRular \fBex\fRpression) to parse a common part containing in every message, which is applied after \fBdatepattern\fR found a match, before the search for any \fBfailregex\fR or \fBignoreregex\fR would start. +.br +If this regex doesn't match the process is starting immediately with next message and search for any \fBfailregex\fR does not occur. +.br +If \fBprefregex\fR contains \fI...\fR, the part of message enclosed between this tags will be extracted and herafter used as whole message for search with \fBfailregex\fR or \fBignoreregex\fR. +.IP +For example: +.nf + prefregex = ^%(__prefix_line)s (?:ERROR|FAILURE) .+$ + failregex = ^user not found + ^authentication failed + ^unknown authentication method +.fi +.IP +You can use \fBprefregex\fR in order to: +.RS +.IP +- specify 1 common regex to match some common part present in every messages (do avoid unneeded match in every \fBfailregex\fR if you have more as one); +.IP +- to cut some interesting part of message only (to simplify \fBfailregex\fR) enclosed between tags \fI\fI and \fI\fR; +.IP +- to gather some failure identifier (e. g. some prefix matched by \fI...\fR tag) to identify several messages belonging to same session, where a connect message containing IP followed by failure message(s) that are not contain IP; +this provides a new multi-line parsing method as replacement for old (slow an ugly) multi-line parsing using buffering window (\fImaxlines\fR > 1 and \fI\fR); +.IP +- to ignore some wrong, too long or even unneeded messages (a.k.a. parasite log traffic) which can be also present in journal, before \fBfailregex\fR search would take place. +.RE + .TP .B failregex is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The standard replacement tags can be used as part of the regex: @@ -451,17 +483,18 @@ is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. T \fI\fR - helper regex to match CIDR (simple integer form of net-mask). .IP \fI\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +.PP +\fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message). +.PP +For multiline regexs (parsing with \fImaxlines\fR greater that 1) the tag \fI\fR can be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. +.br +This is an obsolete handling and if the lines contain some common identifier, better would be to use new handling (with tags \fI...\fR). .RE -.TP -For multiline regexs the tag \fI\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. .TP .B ignoreregex is the regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex. - -.PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. @@ -492,7 +525,9 @@ There are several prefixes and words with special meaning that could be specifie \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP -Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. +Similar to actions, filters may have an [Init] section also (optional since v.0.10). All parameters of both sections [Definition] and [Init] can be overridden (redefined or extended) in \fIjail.conf\fR or \fIjail.local\fR (or in related \fIfilter.d/filter-name.local\fR). +Every option supplied in the jail to the filter overwrites the value specified in [Init] section, which in turm would overwrite the value in [Definition] section. +Besides the standard settings of filter both sections can be used to initialize filter-specific options. Filters can also have a section called [INCLUDES]. This is used to read other configuration files. From cc64ef25f68240d30a6653a2fba3cb26d574cd65 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 23 Nov 2020 17:25:41 +0100 Subject: [PATCH 04/14] filter.d/apache-noscript.conf: extended to match "script not found" with error AH02811 (and cgi-bin path segment in script) closes gh-2805 --- config/filter.d/apache-noscript.conf | 4 ++-- fail2ban/tests/files/logs/apache-noscript | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 32991cba..dd9452a9 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -17,9 +17,9 @@ before = apache-common.conf [Definition] -script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl) +script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/) -prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) ).+$ +prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) ).+$ failregex = ^(?:does not exist|not found or unable to stat):