Merge branch 'gh-3060': adjusted `filter.d/exim.conf` and `filter.d/exim-spam.conf`:

- messages are prefiltered by `prefregex` now
- filter can bypass additional timestamp or pid that may be logged via systemd-journal or syslog-ng (gh-3060)
closes #3060

ChangeLog

@@ -28,7 +28,9 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition
   (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces
   if available for platform and uses DNS to find local IPv6 as a fallback only
 * improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132)
-* `filter.d/exim.conf`:
+* `filter.d/exim.conf`, `filter.d/exim-spam.conf`:
+  - messages are prefiltered by `prefregex` now
+  - filter can bypass additional timestamp or pid that may be logged via systemd-journal or syslog-ng (gh-3060)
   - rewrite host line regex for all varied exim's log_selector states (gh-3263)
   - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502)
 * `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860)

config/filter.d/exim-common.conf

@@ -11,8 +11,18 @@ after = exim-common.local
 
 _fields_grp = (?: (?!H=)[A-Za-z]{1,4}(?:=\S+)?)*
 host_info = %(_fields_grp)s (?:H=)?(?:[\w.-]+)? ?(?:\(\S+\))? ?\[<ADDR>\](?::\d+)?%(_fields_grp)s
-pid = (?: \[\d+\]| [\w\.-]+ exim\[\d+\]:)?
+pid = (?:\s?\[\d+\]|\s?[\w\.-]+ exim\[\d+\]:){0,2}
 
+logtype = file
+_add_pref = <lt_<logtype>/_add_pref>
+
+__prefix_line = %(pid)s%(_add_pref)s
+
+[lt_journal]
+_add_pref = (?: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})?
+
+[lt_file]
+_add_pref =
 
 # DEV Notes
 # ------------

config/filter.d/exim-spam.conf

@@ -26,11 +26,13 @@ before = exim-common.conf
 
 [Definition]
 
-failregex =  ^%(pid)s \S+%(host_info)s rejected by local_scan\(\): .{0,256}$
+prefregex = ^%(__prefix_line)s<F-CONTENT>.+</F-CONTENT>$
-             ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$
+
-             ^%(pid)s \S+%(host_info)s rejected after DATA: This message contains a virus \(\S+\)\.\s*$
+failregex =  ^\s?\S+%(host_info)s rejected by local_scan\(\): .{0,256}$
-             ^%(pid)s \S+ SA: Action: flagged as Spam but accepted: score=\d+\.\d+ required=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=\S+ \[<HOST>\]\) for <honeypot>$
+             ^%(host_info)s rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$
-             ^%(pid)s \S+ SA: Action: silently tossed message: score=\d+\.\d+ required=\d+\.\d+ trigger=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=(\S+ )?\[<HOST>\]\) for \S+$
+             ^\s?\S+%(host_info)s rejected after DATA: This message contains a virus \(\S+\)\.\s*$
+             ^\s?\S+ SA: Action: flagged as Spam but accepted: score=\d+\.\d+ required=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=\S+ \[<HOST>\]\) for <honeypot>$
+             ^\s?\S+ SA: Action: silently tossed message: score=\d+\.\d+ required=\d+\.\d+ trigger=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=(\S+ )?\[<HOST>\]\) for \S+$
 
 ignoreregex = 
 

config/filter.d/exim.conf

@@ -13,21 +13,20 @@ before = exim-common.conf
 
 [Definition]
 
-# Fre-filter via "prefregex" is currently inactive because of too different failure syntax in exim-log (testing needed):
+prefregex = ^%(__prefix_line)s<F-CONTENT>.+</F-CONTENT>$
-#prefregex = ^%(pid)s <F-CONTENT>\b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+</F-CONTENT>$
+
-
+failregex = ^%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
-failregex = ^%(pid)s%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
+            ^\s?\w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(pid)s \w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+            ^%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
-            ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
+            ^\s?SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$
-            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$
+            ^\s?SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors)
-            ^%(pid)s SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors)
+            ^\s?SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s [A-Z]+ (?:command used when not advertised|authentication mechanism not supported)\s*$
-            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s AUTH command used when not advertised\s*$
+            ^\s?no MAIL in SMTP connection from%(host_info)s
-            ^%(pid)s no MAIL in SMTP connection from%(host_info)s
+            ^\s?(?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$
-            ^%(pid)s (?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$
             <mdre-<mode>>
 
-mdre-aggressive = ^%(pid)s no host name found for IP address <ADDR>$
+mdre-aggressive = ^\s?no host name found for IP address <ADDR>$
-                  ^%(pid)s no IP address found for host \S+ \(during SMTP connection from%(host_info)s\)$
+                  ^\s?no IP address found for host \S+ \(during SMTP connection from%(host_info)s\)$
 
 mdre-normal = 
 

fail2ban/tests/files/logs/exim

@@ -72,6 +72,11 @@
 # failJSON: { "time": "2016-03-21T04:33:14", "match": true , "host": "192.0.2.33", "desc": "short form without optional session-id" }
 2016-03-21 04:33:14 SMTP connection from (some.domain) [192.0.2.33] closed by DROP in ACL
 
+# failJSON: { "time": "2016-04-01T11:08:00", "match": true , "host": "192.0.2.29", "desc": "authentication mechanism not supported, gh-3060" }
+2016-04-01 11:08:00 info exim[8003]: [8003] SMTP protocol error in "AUTH LOGIN" H=(User) [192.0.2.29]:4816 I=[192.0.2.1]:25 Ci=8003 LOGIN authentication mechanism not supported
+# failJSON: { "time": "2016-04-01T11:08:00", "match": true , "host": "192.0.2.29", "desc": "additional pid logged with syslog-ng, gh-3060" }
+2016-04-01 11:08:00 info exim[8001]: [8001] no MAIL in SMTP connection from (User) [192.0.2.29]:20042 I=[192.0.2.1]:25 Ci=8001 D=0.349s C=EHLO,AUTH,QUIT
+
 # failJSON: { "time": "2016-04-01T11:08:39", "match": true , "host": "192.0.2.1" }
 2016-04-01 11:08:39 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.1]:1418 I=[172.89.0.6]:25 D=34s C=EHLO,AUTH
 # failJSON: { "time": "2016-04-01T11:08:40", "match": true , "host": "192.0.2.2" }
@@ -115,3 +120,8 @@
 2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
 # failJSON: { "time": "2022-04-03T21:53:53", "match": true , "host": "63.85.123.6", "desc": "no IP found for host long" }
 2022-04-03 21:53:53 no IP address found for host hos-t.example.tld (during SMTP connection from [63.85.123.6]:49390 I=[31.130.202.17]:25)
+
+# filterOptions: {"logtype": "journal"}
+
+# failJSON: { "match": true , "host": "192.0.2.27", "desc": "systemd-journal entry with additional timestamp, gh-3060" }
+mail.example.com exim[3751842]: 2021-07-17 23:20:49 plain_server authenticator failed for ([192.0.2.17]) [192.0.2.27]: 535 Incorrect authentication data