From c9b24839e4338059224dcc743eb7f458f850f18e Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 14:27:41 +0100 Subject: [PATCH 1/8] Character detection heuristics for whois output via optional setting in mail-whois*.conf (Closes #1003) when set by user, - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command - converts whois data to UTF-8 character set with iconv - sends the whois output in UTF-8 character set to mail program - avoids that heirloom mailx creates binary attachment for input with unknown character set --- ChangeLog | 5 +++++ config/action.d/mail-whois-common.conf | 24 ++++++++++++++++++++++++ config/action.d/mail-whois-lines.conf | 6 +++++- config/action.d/mail-whois.conf | 6 +++++- 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 config/action.d/mail-whois-common.conf diff --git a/ChangeLog b/ChangeLog index 19eacf83f..11365ee64 100644 --- a/ChangeLog +++ b/ChangeLog @@ -78,6 +78,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * Absorbed DNSUtils.cidr into addr2bin in filter.py, added unittests * Added syslogsocket configuration to fail2ban.conf * Note in the jail.conf for the recidive jail to increase dbpurgeage (gh-964) + * Character detection heuristics for whois output via optional setting in mail-whois*.conf. When set by user, + - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command + - converts whois data to UTF-8 character set with iconv + - sends the whois output in UTF-8 character set to mail program + - avoids that heirloom mailx creates binary attachment for input with unknown character set ver. 0.9.1 (2014/10/29) - better, faster, stronger diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf new file mode 100644 index 000000000..77f490990 --- /dev/null +++ b/config/action.d/mail-whois-common.conf @@ -0,0 +1,24 @@ +# Fail2Ban configuration file +# +# Common settings for mail actions +# +# Users can override the defaults in mail-whois-common.local + +[INCLUDES] + +# Load customizations if any available +after = mail-whois-common.local + +[DEFAULT] +#original character set of whois output will be sent to mail program +_whois = whois || echo "missing whois program" + +# use heuristics to convert charset of whois output to a target +# character set before sending it to a mail program +_whois_target_charset = UTF-8 +_whois_convert_charset = whois | + { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + +# choose between _whois and _whois_convert_charset in mail-whois-common.local +_whois_command = %(_whois)s +#_whois_command = %(_whois_convert_charset)s diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 5f760ac83..35abeea17 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -4,6 +4,10 @@ # Modified-By: Yaroslav Halchenko to include grepping on IP over log files # +[INCLUDES] + +before = mail-whois-common.conf + [Definition] # Option: actionstart @@ -40,7 +44,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois || echo missing whois program`\n\n + `%(_whois_command)s`\n\n Lines containing IP: in \n `grep -E '(^|[^0-9])([^0-9]|$)' `\n\n Regards,\n diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index e4c8450e6..fe81f499b 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -4,6 +4,10 @@ # # +[INCLUDES] + +before = mail-whois-common.conf + [Definition] # Option: actionstart @@ -40,7 +44,7 @@ actionban = printf %%b "Hi,\n The IP has just been banned by Fail2Ban after attempts against .\n\n Here is more information about :\n - `whois || echo missing whois program`\n + `%(_whois_command)s`\n Regards,\n Fail2Ban"|mail -s "[Fail2Ban] : banned from `uname -n`" From c0cf3daac81ff95bdb18f1293ccb9f72ac6d0ebc Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 18:20:25 +0100 Subject: [PATCH 2/8] Add myself to the changelog --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 11365ee64..2522b9a43 100644 --- a/ChangeLog +++ b/ChangeLog @@ -78,7 +78,8 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released * Absorbed DNSUtils.cidr into addr2bin in filter.py, added unittests * Added syslogsocket configuration to fail2ban.conf * Note in the jail.conf for the recidive jail to increase dbpurgeage (gh-964) - * Character detection heuristics for whois output via optional setting in mail-whois*.conf. When set by user, + * Character detection heuristics for whois output via optional setting in mail-whois*.conf. Thanks Thomas Mayer + When set by user, - detects character set of whois output (which is undefined by RFC 3912) via heuristics of the file command - converts whois data to UTF-8 character set with iconv - sends the whois output in UTF-8 character set to mail program From 80f11a4d28d3cdf91a217517f9130c3e6fb63473 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 18:36:09 +0100 Subject: [PATCH 3/8] Add empty Init Section to pass tests (issue #1003) --- config/action.d/mail-whois-common.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 77f490990..178081e42 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -20,5 +20,8 @@ _whois_convert_charset = whois | { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local +# or other *.local which include mail-whois-common.conf. _whois_command = %(_whois)s #_whois_command = %(_whois_convert_charset)s + +[Init] From 1c2e8f418575463ac9295d39e3f3743453bc68f8 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Fri, 27 Mar 2015 19:12:05 +0100 Subject: [PATCH 4/8] Add myself to the THANKS section --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 5482b21d2..2c1073941 100644 --- a/THANKS +++ b/THANKS @@ -108,6 +108,7 @@ Stefan Tatschner Stephen Gildea Steven Hiscocks TESTOVIK +Thomas Mayer Tom Pike Tomas Pihl Tony Lawrence From 4a598070c8eebb2062c241b3361742db1911c265 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sat, 28 Mar 2015 06:58:01 +0100 Subject: [PATCH 5/8] remove '-ne' option as it's not interpreted any way (issue #1003) --- config/action.d/mail-whois-common.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 178081e42..b0de00e04 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,9 +15,10 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program +# make sure you have 'file' and 'iconv' commands installed when using this _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(echo "$c" | file -b --mime-encoding -) ; echo "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From ac1e41ea703cda2bfd6be064ed469c65ea5579ae Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 17:54:25 +0200 Subject: [PATCH 6/8] Revert "remove '-ne' option as it's not interpreted any way (issue #1003)" This reverts commit 4a598070c8eebb2062c241b3361742db1911c265. --- config/action.d/mail-whois-common.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index b0de00e04..178081e42 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,10 +15,9 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program -# make sure you have 'file' and 'iconv' commands installed when using this _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo "$c" | file -b --mime-encoding -) ; echo "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From 675c3a7c95b7427f3f6517302d5306f488852637 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 18:08:47 +0200 Subject: [PATCH 7/8] use printf instead of echo for POSIX compatibility (issue #1003) --- config/action.d/mail-whois-common.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 178081e42..1da54286e 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -17,7 +17,7 @@ _whois = whois || echo "missing whois program" # character set before sending it to a mail program _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(echo -ne "$c" | file -b --mime-encoding -) ; echo -ne "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { c=$(cat) ; cs=$(printf %%b "$c" | file -b --mime-encoding -) ; printf %%b "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf. From 923d807ef866fc23656708588e5a04644f178f36 Mon Sep 17 00:00:00 2001 From: Thomas Mayer Date: Sun, 29 Mar 2015 18:18:30 +0200 Subject: [PATCH 8/8] use human-readable variable names (issue #1003) --- config/action.d/mail-whois-common.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index 1da54286e..b0d27afc3 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -15,9 +15,10 @@ _whois = whois || echo "missing whois program" # use heuristics to convert charset of whois output to a target # character set before sending it to a mail program +# make sure you have 'file' and 'iconv' commands installed when opting for that _whois_target_charset = UTF-8 _whois_convert_charset = whois | - { c=$(cat) ; cs=$(printf %%b "$c" | file -b --mime-encoding -) ; printf %%b "$c" | iconv -f $cs -t %(_whois_target_charset)s//TRANSLIT - ; } + { WHOIS_OUTPUT=$(cat) ; WHOIS_CHARSET=$(printf %%b "$WHOIS_OUTPUT" | file -b --mime-encoding -) ; printf %%b "$WHOIS_OUTPUT" | iconv -f $WHOIS_CHARSET -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local # or other *.local which include mail-whois-common.conf.