From c80908837fe2fae5821a53bef82988586dbef2dd Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 Mar 2024 15:31:23 +0100 Subject: [PATCH 1/2] `filter.d/exim.conf`: - messages are prefiltered by `prefregex` now - filter can bypass additional timestamp that may be logged via systemd-journal (gh-3060) --- ChangeLog | 2 ++ config/filter.d/exim-common.conf | 12 +++++++++++- config/filter.d/exim-spam.conf | 12 +++++++----- config/filter.d/exim.conf | 25 ++++++++++++------------- fail2ban/tests/files/logs/exim | 5 +++++ 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 19a444607..9f90ab6d8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,8 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition if available for platform and uses DNS to find local IPv6 as a fallback only * improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132) * `filter.d/exim.conf`: + - messages are prefiltered by `prefregex` now + - filter can bypass additional timestamp that may be logged via systemd-journal (gh-3060) - rewrite host line regex for all varied exim's log_selector states (gh-3263) - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502) * `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860) diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index 0f8c91ff6..8ec6cf4c7 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -11,8 +11,18 @@ after = exim-common.local _fields_grp = (?: (?!H=)[A-Za-z]{1,4}(?:=\S+)?)* host_info = %(_fields_grp)s (?:H=)?(?:[\w.-]+)? ?(?:\(\S+\))? ?\[\](?::\d+)?%(_fields_grp)s -pid = (?: \[\d+\]| [\w\.-]+ exim\[\d+\]:)? +pid = (?:\s?\[\d+\]|\s?[\w\.-]+ exim\[\d+\]:)? +logtype = file +_add_pref = /_add_pref> + +__prefix_line = %(pid)s%(_add_pref)s + +[lt_journal] +_add_pref = (?: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})? + +[lt_file] +_add_pref = # DEV Notes # ------------ diff --git a/config/filter.d/exim-spam.conf b/config/filter.d/exim-spam.conf index 7ce04acad..a1833157b 100644 --- a/config/filter.d/exim-spam.conf +++ b/config/filter.d/exim-spam.conf @@ -26,11 +26,13 @@ before = exim-common.conf [Definition] -failregex = ^%(pid)s \S+%(host_info)s rejected by local_scan\(\): .{0,256}$ - ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$ - ^%(pid)s \S+%(host_info)s rejected after DATA: This message contains a virus \(\S+\)\.\s*$ - ^%(pid)s \S+ SA: Action: flagged as Spam but accepted: score=\d+\.\d+ required=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=\S+ \[\]\) for $ - ^%(pid)s \S+ SA: Action: silently tossed message: score=\d+\.\d+ required=\d+\.\d+ trigger=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=(\S+ )?\[\]\) for \S+$ +prefregex = ^%(__prefix_line)s.+$ + +failregex = ^\s?\S+%(host_info)s rejected by local_scan\(\): .{0,256}$ + ^%(host_info)s rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$ + ^\s?\S+%(host_info)s rejected after DATA: This message contains a virus \(\S+\)\.\s*$ + ^\s?\S+ SA: Action: flagged as Spam but accepted: score=\d+\.\d+ required=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=\S+ \[\]\) for $ + ^\s?\S+ SA: Action: silently tossed message: score=\d+\.\d+ required=\d+\.\d+ trigger=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=(\S+ )?\[\]\) for \S+$ ignoreregex = diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index c9de8c927..79a64158c 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -13,21 +13,20 @@ before = exim-common.conf [Definition] -# Fre-filter via "prefregex" is currently inactive because of too different failure syntax in exim-log (testing needed): -#prefregex = ^%(pid)s \b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+$ - -failregex = ^%(pid)s%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ - ^%(pid)s \w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$ - ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ - ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$ - ^%(pid)s SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors) - ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s AUTH command used when not advertised\s*$ - ^%(pid)s no MAIL in SMTP connection from%(host_info)s - ^%(pid)s (?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$ +prefregex = ^%(__prefix_line)s.+$ + +failregex = ^%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$ + ^\s?\w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$ + ^%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ + ^\s?SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$ + ^\s?SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors) + ^\s?SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s AUTH command used when not advertised\s*$ + ^\s?no MAIL in SMTP connection from%(host_info)s + ^\s?(?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$ > -mdre-aggressive = ^%(pid)s no host name found for IP address $ - ^%(pid)s no IP address found for host \S+ \(during SMTP connection from%(host_info)s\)$ +mdre-aggressive = ^\s?no host name found for IP address $ + ^\s?no IP address found for host \S+ \(during SMTP connection from%(host_info)s\)$ mdre-normal = diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index c66a0f986..f5c600b1b 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -115,3 +115,8 @@ 2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9]) # failJSON: { "time": "2022-04-03T21:53:53", "match": true , "host": "63.85.123.6", "desc": "no IP found for host long" } 2022-04-03 21:53:53 no IP address found for host hos-t.example.tld (during SMTP connection from [63.85.123.6]:49390 I=[31.130.202.17]:25) + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.27", "desc": "systemd-journal entry with additional timestamp, gh-3060" } +mail.example.com exim[3751842]: 2021-07-17 23:20:49 plain_server authenticator failed for ([192.0.2.17]) [192.0.2.27]: 535 Incorrect authentication data From 1ec9237e53a71fb99db605b475a5bf71160ca9b0 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 Mar 2024 15:47:43 +0100 Subject: [PATCH 2/2] bypass additional pid in prefix (may be logged by syslog-ng, gh-3060); matches protocol error with authentication mechanism not supported --- ChangeLog | 2 +- config/filter.d/exim-common.conf | 2 +- config/filter.d/exim.conf | 2 +- fail2ban/tests/files/logs/exim | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9f90ab6d8..78314cbd6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,7 +30,7 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition * improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132) * `filter.d/exim.conf`: - messages are prefiltered by `prefregex` now - - filter can bypass additional timestamp that may be logged via systemd-journal (gh-3060) + - filter can bypass additional timestamp or pid that may be logged via systemd-journal or syslog-ng (gh-3060) - rewrite host line regex for all varied exim's log_selector states (gh-3263) - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502) * `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860) diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index 8ec6cf4c7..78c093bb4 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -11,7 +11,7 @@ after = exim-common.local _fields_grp = (?: (?!H=)[A-Za-z]{1,4}(?:=\S+)?)* host_info = %(_fields_grp)s (?:H=)?(?:[\w.-]+)? ?(?:\(\S+\))? ?\[\](?::\d+)?%(_fields_grp)s -pid = (?:\s?\[\d+\]|\s?[\w\.-]+ exim\[\d+\]:)? +pid = (?:\s?\[\d+\]|\s?[\w\.-]+ exim\[\d+\]:){0,2} logtype = file _add_pref = /_add_pref> diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index 79a64158c..cca32de3c 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -20,7 +20,7 @@ failregex = ^%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unroute ^%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$ ^\s?SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$ ^\s?SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors) - ^\s?SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s AUTH command used when not advertised\s*$ + ^\s?SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s [A-Z]+ (?:command used when not advertised|authentication mechanism not supported)\s*$ ^\s?no MAIL in SMTP connection from%(host_info)s ^\s?(?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$ > diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index f5c600b1b..71fc1e26e 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -72,6 +72,11 @@ # failJSON: { "time": "2016-03-21T04:33:14", "match": true , "host": "192.0.2.33", "desc": "short form without optional session-id" } 2016-03-21 04:33:14 SMTP connection from (some.domain) [192.0.2.33] closed by DROP in ACL +# failJSON: { "time": "2016-04-01T11:08:00", "match": true , "host": "192.0.2.29", "desc": "authentication mechanism not supported, gh-3060" } +2016-04-01 11:08:00 info exim[8003]: [8003] SMTP protocol error in "AUTH LOGIN" H=(User) [192.0.2.29]:4816 I=[192.0.2.1]:25 Ci=8003 LOGIN authentication mechanism not supported +# failJSON: { "time": "2016-04-01T11:08:00", "match": true , "host": "192.0.2.29", "desc": "additional pid logged with syslog-ng, gh-3060" } +2016-04-01 11:08:00 info exim[8001]: [8001] no MAIL in SMTP connection from (User) [192.0.2.29]:20042 I=[192.0.2.1]:25 Ci=8001 D=0.349s C=EHLO,AUTH,QUIT + # failJSON: { "time": "2016-04-01T11:08:39", "match": true , "host": "192.0.2.1" } 2016-04-01 11:08:39 [18643] no MAIL in SMTP connection from host.example.com (SERVER) [192.0.2.1]:1418 I=[172.89.0.6]:25 D=34s C=EHLO,AUTH # failJSON: { "time": "2016-04-01T11:08:40", "match": true , "host": "192.0.2.2" }