Merge branch 'fix-gh-3646': nginx error-log filters extended with support of journal format; closes gh-3646

config/filter.d/nginx-error-common.conf

@@ -0,0 +1,32 @@
+# Generic nginx error_log configuration items (to be used as interpolations) in other
+# filters monitoring nginx error-logs
+#
+
+[DEFAULT]
+
+# Type of log-file resp. log-format (file, short, journal):
+logtype = file
+
+# Daemon definition is to be specialized (if needed) in .conf file
+_daemon = nginx
+
+# Common line prefixes (beginnings) which could be used in filters
+#
+#      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
+#
+# This can be optional (for instance if we match named native log files)
+__prefix = <lt_<logtype>/__prefix>
+
+__err_type = error
+
+__prefix_line = %(__prefix)s\[%(__err_type)s\] \d+#\d+: \*\d+\s+
+
+
+[lt_file]
+__prefix = \s*
+
+[lt_short]
+__prefix = \s*(?:(?!\[)\S+ %(_daemon)s\[\d+\]: [^\[]*)?
+
+[lt_journal]
+__prefix = %(lt_short/__prefix)s

config/filter.d/nginx-forbidden.conf

@@ -16,8 +16,12 @@
 #
 # Author: Michele Bologna https://www.michelebologna.net/
 
+[INCLUDES]
+
+before = nginx-error-common.conf
+
 [Definition]
-failregex = \[error\] \d+#\d+: \*\d+ access forbidden by rule, client: <HOST>
+failregex = ^%(__prefix_line)saccess forbidden by rule, client: <HOST>
 ignoreregex =
 
 datepattern = {^LN-BEG}

config/filter.d/nginx-http-auth.conf

@@ -1,14 +1,23 @@
 # fail2ban filter configuration for nginx
 
+[INCLUDES]
+
+before = nginx-error-common.conf
 
 [Definition]
 
 mode = normal
 
-mdre-auth = ^\s*\[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$
-mdre-fallback = ^\s*\[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: <HOST>
+__err_type = <_ertp-<mode>>
 
+_ertp-auth = error
+mdre-auth = ^%(__prefix_line)suser "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$
+_ertp-fallback = crit
+mdre-fallback = ^%(__prefix_line)sSSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: <HOST>
+
+_ertp-normal = %(_ertp-auth)s
 mdre-normal = %(mdre-auth)s
+_ertp-aggressive = (?:%(_ertp-auth)s|%(_ertp-fallback)s)
 mdre-aggressive = %(mdre-auth)s
                   %(mdre-fallback)s
 

config/filter.d/nginx-limit-req.conf

@@ -23,6 +23,10 @@
 #   ...
 #
 
+[INCLUDES]
+
+before = nginx-error-common.conf
+
 [Definition]
 
 # Specify following expression to define exact zones, if you want to ban IPs limited 
@@ -33,13 +37,16 @@
 #
 ngx_limit_req_zones = [^"]+
 
+# Depending on limit_req_log_level directive (may be: info | notice | warn | error):
+__err_type = [a-z]+
+
 # Use following full expression if you should range limit request to specified 
 # servers, requests, referrers etc. only :
 #
-# failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
+# failregex = ^%(__prefix_line)slimiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
 
 # Shortly, much faster and stable version of regexp:
-failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,
+failregex = ^%(__prefix_line)slimiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,
 
 ignoreregex = 
 

fail2ban/tests/files/logs/nginx-http-auth

@@ -13,6 +13,11 @@
 # failJSON: { "time": "2014-04-03T22:20:40", "match": true, "host": "192.0.2.2", "desc": "trying injection on user name"}
 2014/04/03 22:20:40 [error] 30708#0: *3 user "test": password mismatch, client: 127.0.0.1, server: test, request: "GET / HTTP/1.1", host: "localhost:8443"": was not found in "/etc/nginx/.htpasswd", client: 192.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443"
 
+# filterOptions: [{"logtype": "journal"}]
+
+# failJSON: { "match": true, "host": "192.0.2.3", "desc": "systemd journal message, with optional extra timestamp, gh-3646"}
+host nginx[983478]: 2023/12/09 21:35:20 [error] 983478#983478: *3 user "fakeusername" was not found in "/var/lib/nginx/htpasswd-for-host.example.com", client: 192.0.2.3, server: host.example.com, request: "GET / HTTP/2.0", host: "host.example.com"
+
 # filterOptions: [{"mode": "fallback"}]
 
 # failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" }

fail2ban/tests/files/logs/nginx-limit-req

@@ -10,3 +10,8 @@
 
 # failJSON: { "time": "2016-09-30T08:36:06", "match": true, "host": "2606:2800:220:1:248:1893:25c8:1946" }
 2016/09/30 08:36:06 [error] 22923#0: *4758725916 limiting requests, excess: 15.243 by zone "one", client: 2606:2800:220:1:248:1893:25c8:1946, server: example.com, request: "GET / HTTP/1.1", host: "example.com"
+
+# filterOptions: [{"logtype": "journal"}]
+
+# failJSON: { "match": true , "host": "192.0.2.2" }
+host nginx[983479]: 2023/12/09 21:35:20 [notice] 983479#983479: *22174 limiting requests, excess: 1.495 by zone "one", client: 192.0.2.2, server: example.com, request: "GET /index.htm HTTP/1.1", host: "example.com", referrer: "https://example.com"