From ef504c869f0b00d3bbc80d837d00bc867305229d Mon Sep 17 00:00:00 2001 From: Andy Fragen Date: Mon, 26 Aug 2013 16:06:23 -0700 Subject: [PATCH 01/34] added osx specific ipfw action with random rulenum --- config/action.d/osx-ipfw.conf | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 config/action.d/osx-ipfw.conf diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf new file mode 100644 index 00000000..8cd36df6 --- /dev/null +++ b/config/action.d/osx-ipfw.conf @@ -0,0 +1,67 @@ +# Fail2Ban configuration file +# +# Author: Nick Munger +# Modified by: Andy Fragen +# +# Mod for OS X, using random rulenum +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionban = ipfw add set 10 deny log tcp from to + + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: IP address +# Values: CMD +# +actionunban = ipfw delete `ipfw list | grep -i | awk '{print $1;}'` + +[Init] + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] +# +port = ssh + +# Option: localhost +# Notes.: the local IP address of the network interface +# Values: IP +# +localhost = 127.0.0.1 + +# Option: number for ipfw rule +# Values: 1 - 65535 +# Random value between 10000 and 12000 +rulenum = "`echo $((RANDOM%%2000+10000))`" + From 908d4adf6f736bf2c6cf7a3a1dadbc8246287fbe Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 09:37:15 +1000 Subject: [PATCH 02/34] DOC: credits and thanks for Andy for osx-ipfw --- ChangeLog | 3 +++ THANKS | 1 + 2 files changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index c3811a47..e46e9e4f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests closes gh-343. - New Features: + Andy Fragen and Daniel Black + * filter.d/osx-ipfw.conf - ipfw action for OSX based on random rule + numbers. Daniel Black & ykimon * filter.d/3proxy.conf -- filter added Daniel Black diff --git a/THANKS b/THANKS index e3d2cd13..41780856 100644 --- a/THANKS +++ b/THANKS @@ -7,6 +7,7 @@ will be added Adrien Clerc ache Andrey G. Grozin +Andy Fragen Arturo 'Buanzo' Busleiman Axel Thimm Bill Heaton From 5741348f45df3fd061786ab102686b3742d1bac8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 09:38:18 +1000 Subject: [PATCH 03/34] ENH: more options and ruggedness to prevent unintensional consequences --- config/action.d/osx-ipfw.conf | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index 8cd36df6..b7f2f376 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -1,9 +1,9 @@ # Fail2Ban configuration file # # Author: Nick Munger -# Modified by: Andy Fragen +# Modified by: Andy Fragen and Daniel Black # -# Mod for OS X, using random rulenum +# Mod for OS X, using random rulenum as OSX ipfw doesn't include tables # [Definition] @@ -35,7 +35,7 @@ actioncheck = # Tags: IP address # Values: CMD # -actionban = ipfw add set 10 deny log tcp from to +actionban = ipfw add set log from to # Option: actionunban @@ -44,24 +44,41 @@ actionban = ipfw add set 10 deny log tcp from to IP address # Values: CMD # -actionunban = ipfw delete `ipfw list | grep -i | awk '{print $1;}'` +actionunban = ipfw delete `ipfw list | sed -n '/^\([0-9]*\) set log from to $/s//\1/p'` [Init] # Option: port -# Notes.: specifies port to monitor +# Notes.: specifies port to block # Values: [ NUM | STRING ] # port = ssh -# Option: localhost +# Option: dst # Notes.: the local IP address of the network interface -# Values: IP +# Values: IP, any, me or anything support by ipfw as a dst # -localhost = 127.0.0.1 +dst = me -# Option: number for ipfw rule -# Values: 1 - 65535 -# Random value between 10000 and 12000 -rulenum = "`echo $((RANDOM%%2000+10000))`" +# Option: block +# Notes: This is how much to block. +# Can be "ip", "tcp", "udp" or various other options. +# Values: STRING +block = tcp +# Option: blocktype +# Notes.: How to block the traffic. Use a action from man 8 ipfw +# Common values: deny, unreach port, reset +# Values: STRING +# +blocktype = unreach port + +# Option: set number +# Notes.: The ipset number this is added to. +# Values: 0-31 +setnum = 10 + +# Option: number for ipfw rule +# Notes: This is ment to be automaticly generated and not overwritten +# Values: Random value between 10000 and 12000 +rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`" From 808aa1a792302a07b6a0334c932d5f38a6461fd7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 09:39:21 +1000 Subject: [PATCH 04/34] ENH: added jail.conf example. closes gh-340 --- config/jail.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index 86c61911..7f1e40b5 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -416,3 +416,8 @@ filter = perdition action = iptables-multiport[name=perdition,port="110,143,993,995"] logpath = /var/log/maillog +[osx-ssh-ipfw] +enabled = false +filter = sshd +action = osx-ipfw +logpath = /var/log/secure.log From 8b22fa15b55a590d72f21b1d02533f46b0af43b3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 11:03:01 +1000 Subject: [PATCH 05/34] BF: reverted to simplier random rulenum. If your machine is handling 1000s of block the addition complexity isnt what you want --- config/action.d/osx-ipfw.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index b7f2f376..416344a7 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -79,6 +79,9 @@ blocktype = unreach port setnum = 10 # Option: number for ipfw rule -# Notes: This is ment to be automaticly generated and not overwritten +# Notes: This is meant to be automaticly generated and not overwritten # Values: Random value between 10000 and 12000 -rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`" +rulenum="`echo $((RANDOM%%2000+10000))`" + +# Duplicate prevention mechanism +#rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`" From 749f2150898710360789c502b55eebbf5486680c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 11:07:15 +1000 Subject: [PATCH 06/34] ENH: port optional --- config/action.d/osx-ipfw.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index 416344a7..2aadd626 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -44,12 +44,12 @@ actionban = ipfw add set log from to # Tags: IP address # Values: CMD # -actionunban = ipfw delete `ipfw list | sed -n '/^\([0-9]*\) set log from to $/s//\1/p'` +actionunban = ipfw delete `ipfw list | sed -n '/^\([0-9]*\) set log from to ?$/s//\1/p'` [Init] # Option: port -# Notes.: specifies port to block +# Notes.: specifies port to block. Can be blank however may require block="ip" # Values: [ NUM | STRING ] # port = ssh From f2bcf84893b13e1dd69b63074c0064b79978af1d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 31 Aug 2013 11:40:04 +1000 Subject: [PATCH 07/34] BF: action.d/bsd-ipfw - use blocktype instead of unused action for icmp rejecting blocked packets --- ChangeLog | 4 +++- config/action.d/bsd-ipfw.conf | 17 +++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index c3811a47..cadca394 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,7 +32,9 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across all platforms to ensure permissions are the same before and after a ban - closes gh-266. hostsdeny supports daemon_list now too. - * filter.d/roundcube-auth - timezone offset can be positive or negative + * filter.d/roundcube-auth - timezone offset can be positive or negative + * action.d/bsd-ipfw - action option unsed. Fixed to blocktype for + consistency. default to port unreach instead of deny Rolf Fokkens * action.d/dshield.conf and complain.conf -- reorder mailx arguements. https://bugzilla.redhat.com/show_bug.cgi?id=998020 diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 059de386..1285361d 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,7 @@ # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = ipfw show | fgrep -q 'table()' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num deny from table\(
\) to me ; echo $num > "" ) +actionstart = ipfw show | fgrep -q 'table(
)' || ( ipfw show | awk 'BEGIN { b = 1 } { if ($1 <= b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num from table\(
\) to me ; echo $num > "" ) # Option: actionstop @@ -68,15 +68,16 @@ port = # Values: STRING startstatefile = /var/run/fail2ban/ipfw-started-table_
-# Option: action -# Notes: This is the action to take for automaticly created rules. See the -# ACTION defination at the top of man ipfw for allowed values. -# "deny" and "unreach port" are probably the useful. -# Values: STRING -action = deny - # Option: block # Notes: This is how much to block. # Can be "ip", "tcp", "udp" or various other options. # Values: STRING block = ip + +# Option: blocktype +# Notes.: How to block the traffic. Use a action from man 5 ipfw +# Common values: deny, unreach port, reset +# ACTION defination at the top of man ipfw for allowed values. +# Values: STRING +# +blocktype = unreach port From a4884f82cd57c2047f8cf1f41c1769493fbfaa22 Mon Sep 17 00:00:00 2001 From: Andy Fragen Date: Sat, 31 Aug 2013 08:39:19 -0700 Subject: [PATCH 08/34] add mods from grooverdan and fix actionunban actionunban still not working in grooverdan's mod. I made this one grep both and . It should be more specific if the same is banned on multiple ports. --- config/action.d/osx-ipfw.conf | 45 +++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index 8cd36df6..4f421ffb 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -1,9 +1,9 @@ # Fail2Ban configuration file # # Author: Nick Munger -# Modified by: Andy Fragen +# Modified by: Andy Fragen and Daniel Black # -# Mod for OS X, using random rulenum +# Mod for OS X, using random rulenum as OSX ipfw doesn't include tables # [Definition] @@ -35,7 +35,7 @@ actioncheck = # Tags: IP address # Values: CMD # -actionban = ipfw add set 10 deny log tcp from to +actionban = ipfw add set log from to # Option: actionunban @@ -44,24 +44,45 @@ actionban = ipfw add set 10 deny log tcp from to IP address # Values: CMD # -actionunban = ipfw delete `ipfw list | grep -i | awk '{print $1;}'` +actionunban = ipfw delete `ipfw list | grep -i | grep -i | awk '{print $1;}'` +#actionunban = ipfw delete `ipfw list | sed -n '/^\([0-9]*\) set log from to ?$/s//\1/p'` [Init] # Option: port -# Notes.: specifies port to monitor +# Notes.: specifies port to block. Can be blank however may require block="ip" # Values: [ NUM | STRING ] # port = ssh -# Option: localhost +# Option: dst # Notes.: the local IP address of the network interface -# Values: IP +# Values: IP, any, me or anything support by ipfw as a dst # -localhost = 127.0.0.1 +dst = me -# Option: number for ipfw rule -# Values: 1 - 65535 -# Random value between 10000 and 12000 -rulenum = "`echo $((RANDOM%%2000+10000))`" +# Option: block +# Notes: This is how much to block. +# Can be "ip", "tcp", "udp" or various other options. +# Values: STRING +block = tcp +# Option: blocktype +# Notes.: How to block the traffic. Use a action from man 8 ipfw +# Common values: deny, unreach port, reset +# Values: STRING +# +blocktype = deny + +# Option: set number +# Notes.: The ipset number this is added to. +# Values: 0-31 +setnum = 10 + +# Option: number for ipfw rule +# Notes: This is meant to be automaticly generated and not overwritten +# Values: Random value between 10000 and 12000 +rulenum="`echo $((RANDOM%%2000+10000))`" + +# Duplicate prevention mechanism +#rulenum = "`a=$((RANDOM%%2000+10000)); while ipfw show | grep -q ^$a\ ; do a=$((RANDOM%%2000+10000)); done; echo $a`" \ No newline at end of file From fe557e5900b79af1e563b32a5d59425f5580a507 Mon Sep 17 00:00:00 2001 From: Andy Fragen Date: Sun, 1 Sep 2013 13:09:51 -0700 Subject: [PATCH 09/34] more specific actionunban --- config/action.d/osx-ipfw.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index 4f421ffb..b90a136e 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -44,8 +44,7 @@ actionban = ipfw add set log from to # Tags: IP address # Values: CMD # -actionunban = ipfw delete `ipfw list | grep -i | grep -i | awk '{print $1;}'` -#actionunban = ipfw delete `ipfw list | sed -n '/^\([0-9]*\) set log from to ?$/s//\1/p'` +actionunban = ipfw delete `ipfw -S list | grep -i 'set log from to ' | awk '{print $1;}'` [Init] From d258a51a23ea296ef43d8a78cbeb9bd7100af694 Mon Sep 17 00:00:00 2001 From: Andy Fragen Date: Wed, 4 Sep 2013 11:28:03 -0700 Subject: [PATCH 10/34] after some research it looks like setting to unreachable better than deny --- config/action.d/osx-ipfw.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/osx-ipfw.conf b/config/action.d/osx-ipfw.conf index b90a136e..fe0caa2a 100644 --- a/config/action.d/osx-ipfw.conf +++ b/config/action.d/osx-ipfw.conf @@ -71,7 +71,7 @@ block = tcp # Common values: deny, unreach port, reset # Values: STRING # -blocktype = deny +blocktype = unreach port # Option: set number # Notes.: The ipset number this is added to. From d0098b02137516cc89e46b871e916427e86d3f93 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 9 Sep 2013 03:37:59 +1000 Subject: [PATCH 11/34] ENH: add timezone offest and subsecond support to Datedetector --- ChangeLog | 1 + config/filter.d/apache-common.conf | 2 +- config/filter.d/named-refused.conf | 7 ++---- config/filter.d/roundcube-auth.conf | 2 +- server/datedetector.py | 32 +++++++++++++---------- server/datetemplate.py | 39 ++++++++++++++++++++++++++--- testcases/datedetectortestcase.py | 29 +++++++++++++++++---- testcases/files/logs/apache-badbots | 2 +- testcases/files/logs/php-url-fopen | 2 +- testcases/files/logs/roundcube-auth | 2 +- 10 files changed, 87 insertions(+), 31 deletions(-) diff --git a/ChangeLog b/ChangeLog index 633bebbb..e0343ede 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,6 +66,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests and extra failure examples in sample logs * filter.d/apache-auth - added expressions for mod_authz, mod_auth and mod_auth_digest failures. + * Support %z (Timezone offset) and %f (sub-seconds) support for datedetector Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index 134fad29..6955dae1 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -18,4 +18,4 @@ after = apache-common.local # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4] # 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652] # Reference: https://github.com/fail2ban/fail2ban/issues/268 -_apache_error_client = \[[^]]*\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\] +_apache_error_client = \[\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client (:\d{1,5})?\] diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index 1b6f4d4d..4c30a0fe 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -22,10 +22,7 @@ __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re) __line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? -# note - (\.\d+)? is a really ugly catch of the microseconds not captured in -# in the date detector -# -failregex = ^%(__line_prefix)s(\.\d+)?( error:)?\s*client #\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ - ^%(__line_prefix)s(\.\d+)?( error:)?\s*client #\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$ +failregex = ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ + ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$ ^%(__line_prefix)s(\.\d+)?( error:)?\s*client #\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ diff --git a/config/filter.d/roundcube-auth.conf b/config/filter.d/roundcube-auth.conf index d36f5fef..5af6431a 100644 --- a/config/filter.d/roundcube-auth.conf +++ b/config/filter.d/roundcube-auth.conf @@ -17,7 +17,7 @@ before = common.conf # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^\s*(\[(\s[+-][0-9]{4})?\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ +failregex = ^\s*(\[\])?(%(__hostname)s roundcube: IMAP Error)?: (FAILED login|Login failed) for .*? from (\. AUTHENTICATE .*)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/server/datedetector.py b/server/datedetector.py index ab2dd174..2d773113 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -48,8 +48,8 @@ class DateDetector: try: # asctime with subsecond template = DateStrptime() - template.setName("WEEKDAY MONTH Day Hour:Minute:Second[.subsecond] Year") - template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}\.\d+ \d{4}") + template.setName("WEEKDAY MONTH Day Hour:Minute:Second.Subsecond Year") + template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}\.(?P<_f>\d+) \d{4}") template.setPattern("%a %b %d %H:%M:%S.%f %Y") self._appendTemplate(template) # asctime without no subsecond @@ -64,7 +64,7 @@ class DateDetector: template.setRegex("\S{3} \S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") template.setPattern("%a %b %d %H:%M:%S") self._appendTemplate(template) - # standard - most loose from above 3 so by default follows after + # standard - most loose from above so by default follows after template = DateStrptime() template.setName("MONTH Day Hour:Minute:Second") template.setRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") @@ -89,11 +89,11 @@ class DateDetector: template.setRegex("\d{2}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}") template.setPattern("%d/%m/%y %H:%M:%S") self._appendTemplate(template) - # Apache format [31/Oct/2006:09:22:55 -0000] + # Apache format [31/Oct/2006:09:22:55 -0200] template = DateStrptime() - template.setName("Day/MONTH/Year:Hour:Minute:Second") - template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2}") - template.setPattern("%d/%b/%Y:%H:%M:%S") + template.setName("Day/MONTH/Year:Hour:Minute:Second ZoneOffset") + template.setRegex("\d{2}/\S{3}/\d{4}:\d{2}:\d{2}:\d{2} (?P<_z>[+-]\d{4})") + template.setPattern("%d/%b/%Y:%H:%M:%S %z") self._appendTemplate(template) # CPanel 05/20/2008:01:57:39 template = DateStrptime() @@ -115,9 +115,15 @@ class DateDetector: self._appendTemplate(template) # named 26-Jul-2007 15:20:52.252 template = DateStrptime() - template.setName("Day-MONTH-Year Hour:Minute:Second[.Millisecond]") - template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}") - template.setPattern("%d-%b-%Y %H:%M:%S") + template.setName("Day-MONTH-Year Hour:Minute:Second.Subsecond") + template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2}\.(?P<_f>\d+)") + template.setPattern("%d-%b-%Y %H:%M:%S.%f") + self._appendTemplate(template) + # roundcube 26-Jul-2007 15:20:52 +0200 + template = DateStrptime() + template.setName("Day-MONTH-Year Hour:Minute:Second ZoneOffset") + template.setRegex("\d{2}-\S{3}-\d{4} \d{2}:\d{2}:\d{2} (?P<_z>[+-]\d{4})") + template.setPattern("%d-%b-%Y %H:%M:%S %z") self._appendTemplate(template) # 17-07-2008 17:23:25 template = DateStrptime() @@ -127,9 +133,9 @@ class DateDetector: self._appendTemplate(template) # 01-27-2012 16:22:44.252 template = DateStrptime() - template.setName("Month-Day-Year Hour:Minute:Second[.Millisecond]") - template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}") - template.setPattern("%m-%d-%Y %H:%M:%S") + template.setName("Month-Day-Year Hour:Minute:Second.Subsecond") + template.setRegex("\d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}\.(?P<_f>\d+)") + template.setPattern("%m-%d-%Y %H:%M:%S.%f") self._appendTemplate(template) # TAI64N template = DateTai64n() diff --git a/server/datetemplate.py b/server/datetemplate.py index 0754391b..f4847640 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -114,9 +114,12 @@ class DateStrptime(DateTemplate): def __init__(self): DateTemplate.__init__(self) self.__pattern = "" + self.__unsupportedStrptimeBits = False def setPattern(self, pattern): - self.__pattern = pattern.strip() + self.__unsupported_f = not DateStrptime._f and re.search('%f', pattern) + self.__unsupported_z = not DateStrptime._z and re.search('%z', pattern) + self.__pattern = pattern def getPattern(self): return self.__pattern @@ -135,13 +138,23 @@ class DateStrptime(DateTemplate): def getDate(self, line): date = None dateMatch = self.matchDate(line) + if dateMatch: + datePattern = self.getPattern() + if self.__unsupported_f: + if dateMatch.group('_f'): + datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern) + logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern)) + if self.__unsupported_z: + if dateMatch.group('_z'): + datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern) + logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) try: # Try first with 'C' locale - date = list(time.strptime(dateMatch.group(), self.getPattern())) + date = list(time.strptime(dateMatch.group(), datePattern)) except ValueError: # Try to convert date string to 'C' locale - conv = self.convertLocale(dateMatch.group()) + conv = self.convertLocale(datePattern) try: date = list(time.strptime(conv, self.getPattern())) except (ValueError, re.error), e: @@ -179,8 +192,28 @@ class DateStrptime(DateTemplate): # NOTE: Possibly makes week/year day incorrect date[1] = MyTime.gmtime()[1] date[2] = MyTime.gmtime()[2] + if self.__unsupported_z: + z = dateMatch.group('_z') + if z: + date_sec = time.mktime(date) + date_sec -= (int(z[1:3]) * 60 + int(z[3:])) * int(z[0] + '60') + date = list(time.localtime(date_sec)) + #date[8] = 0 # dst + logSys.debug(u"After working with offset date now %r" % date) + return date +try: + time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") + DateStrptime._f = True +except ValueError: + DateTemplate._f = False + +try: + time.strptime("24/Mar/2013:08:58:32 -0500","%d/%b/%Y:%H:%M:%S %z") + DateStrptime._z = True +except ValueError: + DateStrptime._z = False class DateTai64n(DateTemplate): diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index bc6a3865..f60adcc6 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -65,16 +65,18 @@ class DateDetectorTest(unittest.TestCase): for sdate in ( "Jan 23 21:59:59", + "Sun Jan 23 21:59:59.011 2005", "Sun Jan 23 21:59:59 2005", "Sun Jan 23 21:59:59", "2005/01/23 21:59:59", "2005.01.23 21:59:59", "23/01/2005 21:59:59", "23/01/05 21:59:59", - "23/Jan/2005:21:59:59", + "23/Jan/2005:22:59:59 +0100", "01/23/2005:21:59:59", "2005-01-23 21:59:59", - "23-Jan-2005 21:59:59", + "23-Jan-2005 21:59:59.02", + "23-Jan-2005 22:59:59 +0100", "23-01-2005 21:59:59", "01-23-2005 21:59:59.252", # reported on f2b, causes Feb29 fix to break "@4000000041f4104f00000000", # TAI64N @@ -129,6 +131,23 @@ class DateDetectorTest(unittest.TestCase): for template in self.__datedetector.getTemplates() if hasattr(template, "getPattern")] + ZERO = datetime.timedelta(0) + HOUR = datetime.timedelta(hours=1) + + # A UTC class. to make %z formats work + + class UTC(datetime.tzinfo): + """UTC""" + + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return ZERO + year = 2008 # Leap year, 08 for %y can be confused with both %d and %m def iterDates(year): for month in xrange(1, 13): @@ -137,7 +156,7 @@ class DateDetectorTest(unittest.TestCase): for minute in xrange(0, 60, 15): for second in xrange(0, 60, 15): # Far enough? yield datetime.datetime( - year, month, day, hour, minute, second) + year, month, day, hour, minute, second, 300, UTC()) overlapedTemplates = set() for date in iterDates(year): @@ -156,12 +175,12 @@ class DateDetectorTest(unittest.TestCase): matchedTemplates = [template for template in self.__datedetector.getTemplates() if template.getHits() > 0] - assert matchedTemplates != [] # Should match at least one + self.assertNotEqual(matchedTemplates, [], "Date %r should match at least one template" % pattern) if len(matchedTemplates) > 1: overlapedTemplates.add((pattern, tuple(sorted(template.getName() for template in matchedTemplates)))) if overlapedTemplates: - print "WARNING: The following date templates overlap:" + print("WARNING: The following date templates overlap:") pprint.pprint(overlapedTemplates) # def testDefaultTempate(self): diff --git a/testcases/files/logs/apache-badbots b/testcases/files/logs/apache-badbots index 35669252..20134971 100644 --- a/testcases/files/logs/apache-badbots +++ b/testcases/files/logs/apache-badbots @@ -1,2 +1,2 @@ -# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2007-03-05T13:39:21", "match": true , "host": "1.2.3.4" } 1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" diff --git a/testcases/files/logs/php-url-fopen b/testcases/files/logs/php-url-fopen index f119a928..5168e299 100644 --- a/testcases/files/logs/php-url-fopen +++ b/testcases/files/logs/php-url-fopen @@ -1,2 +1,2 @@ -# failJSON: { "time": "2009-03-26T08:44:20", "match": true , "host": "66.185.212.172" } +# failJSON: { "time": "2009-03-26T13:44:20", "match": true , "host": "66.185.212.172" } 66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index 7c16efbd..ed71af75 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -1,4 +1,4 @@ -# failJSON: { "time": "2013-01-22T22:28:21", "match": true , "host": "192.0.43.10" } +# failJSON: { "time": "2013-01-22T20:28:21", "match": true , "host": "192.0.43.10" } [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 # failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" } May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 From 8c1b8284231932fc2f3e41ff66baf371c3c71001 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 9 Sep 2013 03:41:12 +1000 Subject: [PATCH 12/34] BF: capture of microseconds no longer needed. Closes gh-341 --- config/filter.d/named-refused.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index 4c30a0fe..031b8608 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -24,5 +24,5 @@ __line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? failregex = ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: (view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$ - ^%(__line_prefix)s(\.\d+)?( error:)?\s*client #\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ + ^%(__line_prefix)s( error:)?\s*client #\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ From b78507654853525b57c3375583ce40a57cf440c7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Sep 2013 23:37:04 +1000 Subject: [PATCH 13/34] TST: fix test data to match parsing of timezone in UTC --- testcases/files/logs/apache-badbots | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testcases/files/logs/apache-badbots b/testcases/files/logs/apache-badbots index 20134971..35669252 100644 --- a/testcases/files/logs/apache-badbots +++ b/testcases/files/logs/apache-badbots @@ -1,2 +1,2 @@ -# failJSON: { "time": "2007-03-05T13:39:21", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" } 1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02" From d875e8ca0d4083372aba0f5963bb51f8f4aa3d00 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Sep 2013 23:38:44 +1000 Subject: [PATCH 14/34] TST: fix test data - TAI64N timedata was in UTC+1 DST rather than UTC+1 --- testcases/files/logs/dovecot | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testcases/files/logs/dovecot b/testcases/files/logs/dovecot index 553ed621..b581baf1 100644 --- a/testcases/files/logs/dovecot +++ b/testcases/files/logs/dovecot @@ -1,12 +1,12 @@ -# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" } +# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "80.187.101.33" } @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS -# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" } +# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.224" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 -# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" } +# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "192.0.43.10" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" } +# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.225" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root # failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } From 4997b30193d9b154967c22ad0a744800dc7bbcf3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Sep 2013 23:39:41 +1000 Subject: [PATCH 15/34] TST: add datetime method on mytime for rework of datedetector --- server/mytime.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/mytime.py b/server/mytime.py index 8ae85184..554d3544 100644 --- a/server/mytime.py +++ b/server/mytime.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import time +import time, datetime ## # MyTime class. @@ -74,6 +74,14 @@ class MyTime: return time.gmtime(MyTime.myTime) gmtime = staticmethod(gmtime) + #@staticmethod + def now(): + if MyTime.myTime is None: + return datetime.now() + else: + return datetime.datetime.fromtimestamp(MyTime.myTime) + now = staticmethod(now) + def localtime(x=None): if MyTime.myTime is None or x is not None: return time.localtime(x) From 422e2527c4f18f3c77a8011246a78e6499f20d08 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Sep 2013 23:42:38 +1000 Subject: [PATCH 16/34] TST: correct failData - faildata in UTC+1 not UTC --- testcases/files/logs/php-url-fopen | 2 +- testcases/files/logs/roundcube-auth | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testcases/files/logs/php-url-fopen b/testcases/files/logs/php-url-fopen index 5168e299..cbeacf93 100644 --- a/testcases/files/logs/php-url-fopen +++ b/testcases/files/logs/php-url-fopen @@ -1,2 +1,2 @@ -# failJSON: { "time": "2009-03-26T13:44:20", "match": true , "host": "66.185.212.172" } +# failJSON: { "time": "2009-03-26T14:44:20", "match": true , "host": "66.185.212.172" } 66.185.212.172 - - [26/Mar/2009:08:44:20 -0500] "GET /index.php?n=http://eatmyfood.hostinginfive.com/pizza.htm? HTTP/1.1" 200 114 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)" diff --git a/testcases/files/logs/roundcube-auth b/testcases/files/logs/roundcube-auth index ed71af75..3046c63b 100644 --- a/testcases/files/logs/roundcube-auth +++ b/testcases/files/logs/roundcube-auth @@ -1,4 +1,4 @@ -# failJSON: { "time": "2013-01-22T20:28:21", "match": true , "host": "192.0.43.10" } +# failJSON: { "time": "2013-01-22T21:28:21", "match": true , "host": "192.0.43.10" } [22-Jan-2013 22:28:21 +0200]: FAILED login for user1 from 192.0.43.10 # failJSON: { "time": "2005-05-26T07:12:40", "match": true , "host": "10.1.1.47" } May 26 07:12:40 hamster roundcube: IMAP Error: Login failed for sales@example.com from 10.1.1.47 From d8f73c0205ae64976e95681f913d8c23d6448010 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 15 Sep 2013 23:44:30 +1000 Subject: [PATCH 17/34] ENH: full timezone support ISO8601 and %z based timezones now fully supported. Restructured so log lines are also only parsed once and return a unixtime and a pattern match. Fix all test cases to adjust for the change in return value. --- server/datedetector.py | 4 -- server/datetemplate.py | 80 ++++++++++++++++++------------- server/filter.py | 34 ++++++------- testcases/datedetectortestcase.py | 69 +++++++++++--------------- testcases/filtertestcase.py | 23 ++++----- testcases/samplestestcase.py | 20 +++++--- 6 files changed, 118 insertions(+), 112 deletions(-) diff --git a/server/datedetector.py b/server/datedetector.py index 2d773113..8d15d01c 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -208,10 +208,6 @@ class DateDetector: finally: self.__lock.release() - def getUnixTime(self, line): - date = self.getTime(line) - return date and time.mktime(date) - ## # Sort the template lists using the hits score. This method is not called # in this object and thus should be called from time to time. diff --git a/server/datetemplate.py b/server/datetemplate.py index f4847640..0d7aefb4 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -24,7 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import re, time +import re, time, calendar + +from datetime import datetime +from datetime import timedelta from mytime import MyTime import iso8601 @@ -82,12 +85,11 @@ class DateEpoch(DateTemplate): self.setRegex("^\d{10}(\.\d{6})?") def getDate(self, line): - date = None dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch - date = list(MyTime.localtime(float(dateMatch.group()))) - return date + return (float(dateMatch.group()), dateMatch) + return None ## @@ -136,7 +138,6 @@ class DateStrptime(DateTemplate): convertLocale = staticmethod(convertLocale) def getDate(self, line): - date = None dateMatch = self.matchDate(line) if dateMatch: @@ -151,12 +152,12 @@ class DateStrptime(DateTemplate): logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) try: # Try first with 'C' locale - date = list(time.strptime(dateMatch.group(), datePattern)) + date = datetime.strptime(dateMatch.group(), datePattern) except ValueError: # Try to convert date string to 'C' locale - conv = self.convertLocale(datePattern) + conv = self.convertLocale(dateMatch.group()) try: - date = list(time.strptime(conv, self.getPattern())) + date = datetime.strptime(conv, self.getPattern()) except (ValueError, re.error), e: # Try to add the current year to the pattern. Should fix # the "Feb 29" issue. @@ -165,7 +166,7 @@ class DateStrptime(DateTemplate): if not '%Y' in opattern: pattern = "%s %%Y" % opattern conv += " %s" % MyTime.gmtime()[0] - date = list(time.strptime(conv, pattern)) + date = datetime.strptime(conv, pattern) else: # we are helpless here raise ValueError( @@ -173,35 +174,48 @@ class DateStrptime(DateTemplate): "exception was %r and Feb 29 workaround could not " "be tested due to already present year mark in the " "pattern" % (opattern, e)) - if date[0] < 2000: + + if self.__unsupported_z: + z = dateMatch.group('_z') + if z: + delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:])) + direction = z[0] + logSys.debug(u"Altering %r by removing time zone offset (%s)%s" % (date, direction, delta)) + # here we reverse the effect of the timezone and force it to UTC + if direction == '+': + date -= delta + else: + date += delta + date = date.replace(tzinfo=iso8601.Utc()) + else: + logSys.warn("No _z group captured and %%z is not supported on current platform" + " - timezone ignored and assumed to be localtime. date: %s on line: %s" + % (date, line)) + + if date.year < 2000: # There is probably no year field in the logs # NOTE: Possibly makes week/year day incorrect - date[0] = MyTime.gmtime()[0] + date = date.replace(year=MyTime.gmtime()[0]) # Bug fix for #1241756 # If the date is greater than the current time, we suppose # that the log is not from this year but from the year before - if time.mktime(date) > MyTime.time(): + if date > MyTime.now(): logSys.debug( - u"Correcting deduced year from %d to %d since %f > %f" % - (date[0], date[0]-1, time.mktime(date), MyTime.time())) - # NOTE: Possibly makes week/year day incorrect - date[0] -= 1 - elif date[1] == 1 and date[2] == 1: + u"Correcting deduced year by one since %s > now (%s)" % + (date, MyTime.time())) + date = date.replace(year=date.year-1) + elif date.month == 1 and date.day == 1: # If it is Jan 1st, it is either really Jan 1st or there # is neither month nor day in the log. # NOTE: Possibly makes week/year day incorrect - date[1] = MyTime.gmtime()[1] - date[2] = MyTime.gmtime()[2] - if self.__unsupported_z: - z = dateMatch.group('_z') - if z: - date_sec = time.mktime(date) - date_sec -= (int(z[1:3]) * 60 + int(z[3:])) * int(z[0] + '60') - date = list(time.localtime(date_sec)) - #date[8] = 0 # dst - logSys.debug(u"After working with offset date now %r" % date) + date = date.replace(month=MyTime.gmtime()[1], day=1) + + if date.tzinfo: + return ( calendar.timegm(date.utctimetuple()), dateMatch ) + else: + return ( time.mktime(date.utctimetuple()), dateMatch ) - return date + return None try: time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") @@ -224,15 +238,14 @@ class DateTai64n(DateTemplate): self.setRegex("@[0-9a-f]{24}", wordBegin=False) def getDate(self, line): - date = None dateMatch = self.matchDate(line) if dateMatch: # extract part of format which represents seconds since epoch value = dateMatch.group() seconds_since_epoch = value[2:17] # convert seconds from HEX into local time stamp - date = list(MyTime.localtime(int(seconds_since_epoch, 16))) - return date + return (int(seconds_since_epoch, 16), dateMatch) + return None class DateISO8601(DateTemplate): @@ -245,11 +258,10 @@ class DateISO8601(DateTemplate): self.setRegex(date_re) def getDate(self, line): - date = None dateMatch = self.matchDate(line) if dateMatch: # Parses the date. value = dateMatch.group() - date = list(iso8601.parse_date(value).timetuple()) - return date + return (calendar.timegm(iso8601.parse_date(value).utctimetuple()), dateMatch) + return None diff --git a/server/filter.py b/server/filter.py index 2f88cbea..412dd2d1 100644 --- a/server/filter.py +++ b/server/filter.py @@ -295,18 +295,8 @@ class Filter(JailThread): l = l.rstrip('\r\n') logSys.log(7, "Working on line %r", l) - timeMatch = self.dateDetector.matchTime(l) - if timeMatch: - # Lets split into time part and log part of the line - timeLine = timeMatch.group() - # Lets leave the beginning in as well, so if there is no - # anchore at the beginning of the time regexp, we don't - # at least allow injection. Should be harmless otherwise - logLine = l[:timeMatch.start()] + l[timeMatch.end():] - else: - timeLine = l - logLine = l - return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) + + return self.findFailure(l, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager @@ -349,16 +339,28 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, timeLine, logLine, + def findFailure(self, logLine, returnRawHost=False, checkAllRegex=False): - logSys.log(5, "Date: %r, message: %r", timeLine, logLine) failList = list() # Checks if we must ignore this line. if self.ignoreLine(logLine) is not None: # The ignoreregex matched. Return. - logSys.log(7, "Matched ignoreregex and was ignored") + logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine) return failList - date = self.dateDetector.getUnixTime(timeLine) + dd = self.dateDetector.getTime(logLine) + + if dd is None: + return failList + date = dd[0] + timeMatch = dd[1] + if timeMatch: + # Lets split into time part and log part of the line + timeLine = timeMatch.group() + # Lets leave the beginning in as well, so if there is no + # anchore at the beginning of the time regexp, we don't + # at least allow injection. Should be harmless otherwise + logLine = logLine[:timeMatch.start()] + logLine[timeMatch.end():] + # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): failRegex.search(logLine) diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index f60adcc6..27a32c19 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -24,9 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import unittest, calendar, datetime, re, pprint +import unittest, calendar, time, datetime, re, pprint from server.datedetector import DateDetector from server.datetemplate import DateTemplate +from server.iso8601 import Utc class DateDetectorTest(unittest.TestCase): @@ -40,11 +41,12 @@ class DateDetectorTest(unittest.TestCase): def testGetEpochTime(self): log = "1138049999 [sshd] error: PAM: Authentication failure" - date = [2006, 1, 23, 21, 59, 59, 0, 23, 0] + #date = [2006, 1, 23, 21, 59, 59, 0, 23, 0] dateUnix = 1138049999.0 - self.assertEqual(self.__datedetector.getTime(log), date) - self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix) + ( datelog, matchlog ) = self.__datedetector.getTime(log) + self.assertEqual(datelog, dateUnix) + self.assertEqual(matchlog.group(), '1138049999') def testGetTime(self): log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure" @@ -54,8 +56,9 @@ class DateDetectorTest(unittest.TestCase): # is not correctly determined atm, since year is not present # in the log entry. Since this doesn't effect the operation # of fail2ban -- we just ignore incorrect day of the week - self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6]) - self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix) + ( datelog, matchlog ) = self.__datedetector.getTime(log) + self.assertEqual(datelog, dateUnix) + self.assertEqual(matchlog.group(), 'Jan 23 21:59:59') def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand @@ -72,16 +75,16 @@ class DateDetectorTest(unittest.TestCase): "2005.01.23 21:59:59", "23/01/2005 21:59:59", "23/01/05 21:59:59", - "23/Jan/2005:22:59:59 +0100", + "23/Jan/2005:21:59:59 +0100", "01/23/2005:21:59:59", "2005-01-23 21:59:59", "23-Jan-2005 21:59:59.02", - "23-Jan-2005 22:59:59 +0100", + "23-Jan-2005 21:59:59 +0100", "23-01-2005 21:59:59", "01-23-2005 21:59:59.252", # reported on f2b, causes Feb29 fix to break "@4000000041f4104f00000000", # TAI64N - "2005-01-23T21:59:59.252Z", #ISO 8601 - "2005-01-23T21:59:59-05:00Z", #ISO 8601 with TZ + "2005-01-23T20:59:59.252Z", #ISO 8601 + "2005-01-23T15:59:59-05:00", #ISO 8601 with TZ "<01/23/05@21:59:59>", "050123 21:59:59", # MySQL "Jan-23-05 21:59:59", # ASSP like @@ -92,8 +95,9 @@ class DateDetectorTest(unittest.TestCase): # yoh: on [:6] see in above test logtime = self.__datedetector.getTime(log) self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s" % sdate) - self.assertEqual(logtime[:6], date[:6], "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6])) - self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix, "getUnixTime failure for %s: \"%s\" is not \"%s\"" % (sdate, logtime[:6], date[:6])) + ( logUnix, logMatch ) = logtime + self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: \"%s\" is not \"%s\"" % (sdate, logUnix, dateUnix)) + self.assertEqual(logMatch.group(), sdate) def testStableSortTemplate(self): old_names = [x.getName() for x in self.__datedetector.getTemplates()] @@ -110,44 +114,29 @@ class DateDetectorTest(unittest.TestCase): # see https://github.com/fail2ban/fail2ban/pull/130 # yoh: unfortunately this test is not really effective to reproduce the # situation but left in place to assure consistent behavior - m1 = [2012, 10, 11, 2, 37, 17] - self.assertEqual( - self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6], - m1) + mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).utctimetuple()) + logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') + self.assertNotEqual(logdate, None) + ( logTime, logMatch ) = logdate + self.assertEqual(logTime, mu) + self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') self.__datedetector.sortTemplate() # confuse it with year being at the end for i in xrange(10): - self.assertEqual( - self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')[:6], - m1) + ( logTime, logMatch ) = self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') + self.assertEqual(logTime, mu) + self.assertEqual(logMatch.group(), '11/10/2012 02:37:17') self.__datedetector.sortTemplate() # and now back to the original - self.assertEqual( - self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6], - m1) + ( logTime, logMatch ) = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') + self.assertEqual(logTime, mu) + self.assertEqual(logMatch.group(), '2012/10/11 02:37:17') def testDateDetectorTemplateOverlap(self): patterns = [template.getPattern() for template in self.__datedetector.getTemplates() if hasattr(template, "getPattern")] - ZERO = datetime.timedelta(0) - HOUR = datetime.timedelta(hours=1) - - # A UTC class. to make %z formats work - - class UTC(datetime.tzinfo): - """UTC""" - - def utcoffset(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return ZERO - year = 2008 # Leap year, 08 for %y can be confused with both %d and %m def iterDates(year): for month in xrange(1, 13): @@ -156,7 +145,7 @@ class DateDetectorTest(unittest.TestCase): for minute in xrange(0, 60, 15): for second in xrange(0, 60, 15): # Far enough? yield datetime.datetime( - year, month, day, hour, minute, second, 300, UTC()) + year, month, day, hour, minute, second, 300, Utc()) overlapedTemplates = set() for date in iterDates(year): diff --git a/testcases/filtertestcase.py b/testcases/filtertestcase.py index 2061327a..a89e62c0 100644 --- a/testcases/filtertestcase.py +++ b/testcases/filtertestcase.py @@ -40,6 +40,7 @@ from server.failmanager import FailManagerEmpty # from utils import mtimesleep +from server.mytime import MyTime # yoh: per Steven Hiscocks's insight while troubleshooting # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 @@ -78,8 +79,8 @@ def _assert_equal_entries(utest, found, output, count=None): utest.assertEqual(found[0], output[0]) # IP utest.assertEqual(found[1], count or output[1]) # count found_time, output_time = \ - time.localtime(found[2]),\ - time.localtime(output[2]) + MyTime.localtime(found[2]),\ + MyTime.localtime(output[2]) utest.assertEqual(found_time, output_time) if len(output) > 3 and count is None: # match matches # do not check if custom count (e.g. going through them twice) @@ -560,7 +561,7 @@ class GetFailures(unittest.TestCase): FILENAME_USEDNS = "testcases/files/testcase-usedns.log" # so that they could be reused by other tests - FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, + FAILURES_01 = ('193.168.0.128', 3, 1124017199.0, ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) def setUp(self): @@ -604,7 +605,7 @@ class GetFailures(unittest.TestCase): def testGetFailures02(self): - output = ('141.3.81.106', 4, 1124013539.0, + output = ('141.3.81.106', 4, 1124017139.0, ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' % m for m in 53, 54, 57, 58]) @@ -614,7 +615,7 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 6, 1124013544.0) + output = ('203.162.223.135', 6, 1124017144.0) self.filter.addLogPath(GetFailures.FILENAME_03) self.filter.addFailRegex("error,relay=,.*550 User unknown") @@ -622,8 +623,8 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures04(self): - output = [('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 4, 1124013598.0)] + output = [('212.41.96.186', 4, 1124017200.0), + ('212.41.96.185', 4, 1124017198.0)] self.filter.addLogPath(GetFailures.FILENAME_04) self.filter.addFailRegex("Invalid user .* ") @@ -637,11 +638,11 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) - output_yes = ('93.184.216.119', 2, 1124013539.0, + output_yes = ('93.184.216.119', 2, 1124017139.0, ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', 'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) - output_no = ('93.184.216.119', 1, 1124013539.0, + output_no = ('93.184.216.119', 1, 1124017139.0, ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) # Actually no exception would be raised -- it will be just set to 'no' @@ -663,7 +664,7 @@ class GetFailures(unittest.TestCase): def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = ('141.3.81.106', 8, 1124017141.0) self.filter.addLogPath(GetFailures.FILENAME_02) self.filter.addFailRegex("Failed .* from ") @@ -672,7 +673,7 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresIgnoreRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = ('141.3.81.106', 8, 1124017141.0) self.filter.addLogPath(GetFailures.FILENAME_02) self.filter.addFailRegex("Failed .* from ") diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index c7ff0b9a..acd553c1 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -22,7 +22,9 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import unittest, sys, os, fileinput, re, datetime, inspect +import unittest, sys, os, fileinput, re, time, datetime, inspect +from server.mytime import MyTime + if sys.version_info >= (2, 6): import json @@ -110,15 +112,19 @@ def testSampleRegexsFactory(name): (map(lambda x: x[0], ret),logFile.filename(), logFile.filelineno())) # Verify timestamp and host as expected - failregex, host, time = ret[0] + failregex, host, fail2banTime = ret[0] self.assertEqual(host, faildata.get("host", None)) - fail2banTime = datetime.datetime.fromtimestamp(time) - jsonTime = datetime.datetime.strptime( - faildata.get("time", None), "%Y-%m-%dT%H:%M:%S") + + t = faildata.get("time", None) + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") + + jsonTime = time.mktime(jsonTimeLocal.utctimetuple()) self.assertEqual(fail2banTime, jsonTime, - "Time mismatch %s != %s on: %s:%i %r:" % - (fail2banTime, jsonTime, logFile.filename(), logFile.filelineno(), line ) ) + "UTC Time mismatch fail2ban %s (%s) != failJson %s (%s) (diff %i seconds) on: %s:%i %r:" % + (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), + jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), + fail2banTime - jsonTime, logFile.filename(), logFile.filelineno(), line ) ) regexsUsed.add(failregex) From 2cefce5ee0594527ec27f15c43035b3c1aadb6a9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 16 Sep 2013 00:09:41 +1000 Subject: [PATCH 18/34] TST: testDateDetectorTemplateOverlap fix for python-2.5 without %f in strftime --- testcases/datedetectortestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testcases/datedetectortestcase.py b/testcases/datedetectortestcase.py index 27a32c19..6eca3204 100644 --- a/testcases/datedetectortestcase.py +++ b/testcases/datedetectortestcase.py @@ -151,6 +151,7 @@ class DateDetectorTest(unittest.TestCase): for date in iterDates(year): for pattern in patterns: datestr = date.strftime(pattern) + datestr = re.sub(r'%f','300', datestr) # for python 2.5 where there is no %f datestrs = set([ datestr, re.sub(r"(\s)0", r"\1 ", datestr), From 50a6289f03a0f1540c0a965e85b07609b476e1ae Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 16 Sep 2013 18:50:19 +1000 Subject: [PATCH 19/34] BF: handle 2.4 and %f in strftime --- server/datetemplate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/datetemplate.py b/server/datetemplate.py index 0d7aefb4..174538e5 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -220,7 +220,7 @@ class DateStrptime(DateTemplate): try: time.strptime("26-Jul-2007 15:20:52.252","%d-%b-%Y %H:%M:%S.%f") DateStrptime._f = True -except ValueError: +except (ValueError, KeyError): DateTemplate._f = False try: From 0f283f8b6fdefa6da969fd734e664555da9826b7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 20 Sep 2013 21:16:39 +1000 Subject: [PATCH 20/34] BF: wrong variable name in previous merge --- fail2ban/server/filter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8e4c723b..68b27b32 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -350,9 +350,7 @@ class Filter(JailThread): line = line.rstrip('\r\n') logSys.log(7, "Working on line %r", line) - logSys.log(7, "Working on line %r", l) - - return self.findFailure(l, returnRawHost, checkAllRegex) + return self.findFailure(line, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager From d9f0438a8d8cc72f599b3a2c9fa94426019d4ca7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 20 Sep 2013 21:18:44 +1000 Subject: [PATCH 21/34] MRG: remerge in %z and %f datetime format --- fail2ban/server/datedetector.py | 10 +++++++--- fail2ban/server/datetemplate.py | 35 +++++++++++++++++---------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index b6dc39a5..9f119b3c 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -53,6 +53,8 @@ class DateDetector: def addDefaultTemplate(self): self.__lock.acquire() try: + # asctime with subsecond + self.appendTemplate("%a %b %d %H:%M:%S.%f %Y") # asctime self.appendTemplate("%a %b %d %H:%M:%S %Y") # asctime without year @@ -67,17 +69,19 @@ class DateDetector: # (See http://bugs.debian.org/537610) self.appendTemplate("%d/%m/%y %H:%M:%S") # Apache format [31/Oct/2006:09:22:55 -0000] - self.appendTemplate("%d/%b/%Y:%H:%M:%S") + self.appendTemplate("%d/%b/%Y:%H:%M:%S %z") # CPanel 05/20/2008:01:57:39 self.appendTemplate("%m/%d/%Y:%H:%M:%S") # custom for syslog-ng 2006.12.21 06:43:20 self.appendTemplate("%Y.%m.%d %H:%M:%S") # named 26-Jul-2007 15:20:52.252 - self.appendTemplate("%d-%b-%Y %H:%M:%S") + self.appendTemplate("%d-%b-%Y %H:%M:%S.%f") + # roundcube 26-Jul-2007 15:20:52 +0200 + self.appendTemplate("%d-%b-%Y %H:%M:%S %z") # 17-07-2008 17:23:25 self.appendTemplate("%d-%m-%Y %H:%M:%S") # 01-27-2012 16:22:44.252 - self.appendTemplate("%m-%d-%Y %H:%M:%S") + self.appendTemplate("%m-%d-%Y %H:%M:%S.%f") # TAI64N template = DateTai64n() template.setName("TAI64N") diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 8dbb0482..81b307eb 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -51,6 +51,7 @@ class DateTemplate: return self.__name def setRegex(self, regex, wordBegin=True): + #logSys.debug(u"setRegex for %s is %r" % (self.__name, regex)) regex = regex.strip() if (wordBegin and not re.search(r'^\^', regex)): regex = r'\b' + regex @@ -115,16 +116,16 @@ class DateStrptime(DateTemplate): def __init__(self): DateTemplate.__init__(self) - self.__pattern = "" - self.__unsupportedStrptimeBits = False + self._pattern = "" + self._unsupportedStrptimeBits = False def setPattern(self, pattern): - self.__unsupported_f = not DateStrptime._f and re.search('%f', pattern) - self.__unsupported_z = not DateStrptime._z and re.search('%z', pattern) - self.__pattern = pattern + self._unsupported_f = not DateStrptime._f and re.search('%f', pattern) + self._unsupported_z = not DateStrptime._z and re.search('%z', pattern) + self._pattern = pattern def getPattern(self): - return self.__pattern + return self._pattern #@staticmethod def convertLocale(date): @@ -142,11 +143,11 @@ class DateStrptime(DateTemplate): if dateMatch: datePattern = self.getPattern() - if self.__unsupported_f: + if self._unsupported_f: if dateMatch.group('_f'): datePattern = re.sub(r'%f', dateMatch.group('_f'), datePattern) logSys.debug(u"Replacing %%f with %r now %r" % (dateMatch.group('_f'), datePattern)) - if self.__unsupported_z: + if self._unsupported_z: if dateMatch.group('_z'): datePattern = re.sub(r'%z', dateMatch.group('_z'), datePattern) logSys.debug(u"Replacing %%z with %r now %r" % (dateMatch.group('_z'), datePattern)) @@ -175,7 +176,7 @@ class DateStrptime(DateTemplate): "be tested due to already present year mark in the " "pattern" % (opattern, e)) - if self.__unsupported_z: + if self._unsupported_z: z = dateMatch.group('_z') if z: delta = timedelta(hours=int(z[1:3]),minutes=int(z[3:])) @@ -231,21 +232,24 @@ except ValueError: class DatePatternRegex(DateStrptime): _reEscape = r"([\\.^$*+?\(\){}\[\]|])" - _patternRE = r"%(%|[aAbBdHIjmMpSUwWyY])" + _patternRE = r"%(%|[aAbBdfHIjmMpSUwWyYz])" _patternName = { 'a': "DAY", 'A': "DAYNAME", 'b': "MON", 'B': "MONTH", 'd': "Day", 'H': "24hour", 'I': "12hour", 'j': "Yearday", 'm': "Month", 'M': "Minute", 'p': "AMPM", 'S': "Second", 'U': "Yearweek", - 'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%"} + 'w': "Weekday", 'W': "Yearweek", 'y': 'Year2', 'Y': "Year", '%': "%", + 'z': "Zone offset", 'f': "Microseconds" } _patternRegex = { 'a': r"\w{3}", 'A': r"\w+", 'b': r"\w{3}", 'B': r"\w+", - 'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)", + 'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)", + 'f': r"\d{1,6}", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)", 'I': r"(?:1[0-2]|[ 0]?\d)", 'j': r"(?:36[0-6]3[0-5]\d|[1-2]\d\d|[ 0]?\d\d|[ 0]{0,2}\d)", 'm': r"(?:1[0-2]|[ 0]?[1-9])", 'M': r"[0-5]\d", 'p': r"[AP]M", 'S': r"(?:6[01]|[0-5]\d)", 'U': r"(?:5[0-3]|[1-4]\d|[ 0]?\d)", 'w': r"[0-6]", 'W': r"(?:5[0-3]|[ 0]?\d)", 'y': r"\d{2}", - 'Y': r"\d{4}", '%': "%"} + 'Y': r"\d{4}", + 'z': r"(?P<_z>[+-]\d{4})", '%': "%"} def __init__(self, pattern=None, **kwargs): DateStrptime.__init__(self) @@ -253,7 +257,7 @@ class DatePatternRegex(DateStrptime): self.setPattern(pattern, **kwargs) def setPattern(self, pattern, anchor=False, **kwargs): - self.__pattern = pattern.strip() + DateStrptime.setPattern(self, pattern.strip()) name = re.sub(self._patternRE, r'%(\1)s', pattern) % self._patternName DateStrptime.setName(self, name) @@ -266,9 +270,6 @@ class DatePatternRegex(DateStrptime): regex = r"^" + regex DateStrptime.setRegex(self, regex, **kwargs) - def getPattern(self): - return self.__pattern - def setRegex(self, line): raise NotImplementedError("Regex derived from pattern") From cfd9778f3c9d6dda670d2f5f8311bbdf65ca645e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 20 Sep 2013 21:43:27 +1000 Subject: [PATCH 22/34] TST: fix unicode on test strings --- fail2ban/tests/filtertestcase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 057d1a71..d57766e3 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -739,7 +739,7 @@ class GetFailures(unittest.TestCase): # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124017199.0, - ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) + [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128\n']*3) def setUp(self): """Call before every test case.""" @@ -785,7 +785,7 @@ class GetFailures(unittest.TestCase): def testGetFailures02(self): output = ('141.3.81.106', 4, 1124017139.0, - ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' + [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2\n' % m for m in 53, 54, 57, 58]) self.filter.addLogPath(GetFailures.FILENAME_02) @@ -818,11 +818,11 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ('93.184.216.119', 2, 1124017139.0, - ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', - 'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2\n', + u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) output_no = ('93.184.216.119', 1, 1124017139.0, - ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2\n']) # Actually no exception would be raised -- it will be just set to 'no' #self.assertRaises(ValueError, From 33aee14fcc80f8f72b10137789ebdb265af7134a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 09:16:31 +1000 Subject: [PATCH 23/34] DOC: comment examples of date formats --- fail2ban/server/datedetector.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 9f119b3c..6cbffeed 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -53,19 +53,19 @@ class DateDetector: def addDefaultTemplate(self): self.__lock.acquire() try: - # asctime with subsecond + # asctime with subsecond: Sun Jan 23 21:59:59.011 2005 self.appendTemplate("%a %b %d %H:%M:%S.%f %Y") - # asctime + # asctime: Sun Jan 23 21:59:59 2005 self.appendTemplate("%a %b %d %H:%M:%S %Y") - # asctime without year + # asctime without year: Sun Jan 23 21:59:59 self.appendTemplate("%a %b %d %H:%M:%S") - # standard + # standard: Jan 23 21:59:59 self.appendTemplate("%b %d %H:%M:%S") - # simple date + # simple date: 2005/01/23 21:59:59 self.appendTemplate("%Y/%m/%d %H:%M:%S") - # simple date too (from x11vnc) + # simple date too (from x11vnc): 23/01/2005 21:59:59 self.appendTemplate("%d/%m/%Y %H:%M:%S") - # previous one but with year given by 2 digits + # previous one but with year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) self.appendTemplate("%d/%m/%y %H:%M:%S") # Apache format [31/Oct/2006:09:22:55 -0000] From ec0670f6d575adecd32cad9add7b70924fe8f72b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 09:44:24 +1000 Subject: [PATCH 24/34] BF: fix MyTime imports --- fail2ban/tests/filtertestcase.py | 2 +- fail2ban/tests/samplestestcase.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index d57766e3..b948e33c 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -39,6 +39,7 @@ from fail2ban.server.filterpoll import FilterPoll from fail2ban.server.filter import FileFilter, DNSUtils from fail2ban.server.failmanager import FailManager from fail2ban.server.failmanager import FailManagerEmpty +from fail2ban.server.mytime import MyTime from fail2ban.tests.utils import setUpMyTime, tearDownMyTime TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -48,7 +49,6 @@ TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") # from utils import mtimesleep -from server.mytime import MyTime # yoh: per Steven Hiscocks's insight while troubleshooting # https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836 diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 34f99841..fb3ebbc8 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -23,8 +23,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import unittest, sys, os, fileinput, re, time, datetime, inspect -from server.mytime import MyTime - if sys.version_info >= (2, 6): import json From 0035c99d07736af2f2569c9d0ae8251934dd5ef8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 10:22:15 +1000 Subject: [PATCH 25/34] BF: remove unused function DateDetector.getUnixTime --- fail2ban/server/datedetector.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 6cbffeed..056cd2d8 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -140,10 +140,6 @@ class DateDetector: finally: self.__lock.release() - def getUnixTime(self, line): - date = self.getTime(line) - return date and time.mktime(tuple(date)) - ## # Sort the template lists using the hits score. This method is not called # in this object and thus should be called from time to time. From 855d802ba8c608309dc010262f71056963522724 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 10:31:26 +1000 Subject: [PATCH 26/34] ENH: more detail in debug messages in date/time detection --- fail2ban/server/datedetector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 056cd2d8..ee4621c9 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -132,7 +132,7 @@ class DateDetector: date = template.getDate(line) if date is None: continue - logSys.debug("Got time using template %s" % template.getName()) + logSys.debug("Got time %i for \"%r\" using template %s" % (date[0], date[1].group(), template.getName())) return date except ValueError: pass From a0676cdd1e3163bac6c30a1ffc707e95a4810d9f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 11:44:50 +1000 Subject: [PATCH 27/34] ENH: add date format 2005-01-23 21:59:59 (%Y-%m-%d %H:%M:%S) so ISO8601 doesnt handle it --- fail2ban/server/datedetector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index ee4621c9..62c892aa 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -61,6 +61,8 @@ class DateDetector: self.appendTemplate("%a %b %d %H:%M:%S") # standard: Jan 23 21:59:59 self.appendTemplate("%b %d %H:%M:%S") + # simple date: 2005-01-23 21:59:59 + self.appendTemplate("%Y-%m-%d %H:%M:%S") # simple date: 2005/01/23 21:59:59 self.appendTemplate("%Y/%m/%d %H:%M:%S") # simple date too (from x11vnc): 23/01/2005 21:59:59 From 37de5462beca411b01cff7b354778b64d75ba5a7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 21 Sep 2013 11:47:24 +1000 Subject: [PATCH 28/34] ENH: iso8601 - remove default_timezone set to UTC, isn't valid any more - calculate from local timezone. Enforce T date/time separator in iso8601 regex. Make minutes in timezone optional as per ISO8601. Use consistant regex for ISO8601 --- fail2ban/server/datetemplate.py | 5 +---- fail2ban/server/iso8601.py | 40 ++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 81b307eb..6309e8eb 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -300,10 +300,7 @@ class DateISO8601(DateTemplate): def __init__(self): DateTemplate.__init__(self) - date_re = "[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}" \ - ".[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?" \ - "(Z|(([-+])([0-9]{2}):([0-9]{2})))?" - self.setRegex(date_re) + self.setRegex(iso8601.ISO8601_REGEX_RAW) def getDate(self, line): dateMatch = self.matchDate(line) diff --git a/fail2ban/server/iso8601.py b/fail2ban/server/iso8601.py index a42989ea..f40d5e75 100644 --- a/fail2ban/server/iso8601.py +++ b/fail2ban/server/iso8601.py @@ -32,17 +32,17 @@ datetime.datetime(2007, 1, 25, 12, 0, tzinfo=) """ -from datetime import datetime, timedelta, tzinfo +from datetime import datetime, timedelta, tzinfo, time import re __all__ = ["parse_date", "ParseError"] # Adapted from http://delete.me.uk/2005/03/iso8601.html -ISO8601_REGEX = re.compile(r"(?P[0-9]{4})(-(?P[0-9]{1,2})(-(?P[0-9]{1,2})" - r"((?P.)(?P[0-9]{2}):(?P[0-9]{2})(:(?P[0-9]{2})(\.(?P[0-9]+))?)?" - r"(?PZ|(([-+])([0-9]{2}):([0-9]{2})))?)?)?)?" -) -TIMEZONE_REGEX = re.compile("(?P[+-])(?P[0-9]{2}).(?P[0-9]{2})") +ISO8601_REGEX_RAW = "(?P[0-9]{4})-(?P[0-9]{1,2})-(?P[0-9]{1,2})" \ + "T(?P[0-9]{2}):(?P[0-9]{2})(:(?P[0-9]{2})(\.(?P[0-9]+))?)?" \ + "(?PZ|(([-+])([0-9]{2}):([0-9]{2})))?" +ISO8601_REGEX = re.compile(ISO8601_REGEX_RAW) +TIMEZONE_REGEX = re.compile("(?P[+-])(?P[0-9]{2}):?(?P[0-9]{2})?") class ParseError(Exception): """Raised when there is a problem parsing a date string""" @@ -67,8 +67,8 @@ class FixedOffset(tzinfo): """Fixed offset in hours and minutes from UTC """ - def __init__(self, offset_hours, offset_minutes, name): - self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes) + def __init__(self, name, offset_hours, offset_minutes, offset_seconds=0): + self.__offset = timedelta(hours=offset_hours, minutes=offset_minutes, seconds=offset_seconds) self.__name = name def utcoffset(self, dt): @@ -83,26 +83,30 @@ class FixedOffset(tzinfo): def __repr__(self): return "" % self.__name -def parse_timezone(tzstring, default_timezone=UTC): +def parse_timezone(tzstring): """Parses ISO 8601 time zone specs into tzinfo offsets """ if tzstring == "Z": - return default_timezone - # This isn't strictly correct, but it's common to encounter dates without - # timezones so I'll assume the default (which defaults to UTC). - # Addresses issue 4. + return UTC + if tzstring is None: - return default_timezone + zone_sec = -time.timezone + return FixedOffset(name=time.tzname[0],hours=(zone_sec / 3600),minutes=(zone_sec % 3600)/60,seconds=zone_sec % 60) + m = TIMEZONE_REGEX.match(tzstring) prefix, hours, minutes = m.groups() - hours, minutes = int(hours), int(minutes) + if minutes is None: + minutes = 0 + else: + minutes = int(minutes) + hours = int(hours) if prefix == "-": hours = -hours minutes = -minutes - return FixedOffset(hours, minutes, tzstring) + return FixedOffset(tzstring, hours, minutes) -def parse_date(datestring, default_timezone=UTC): +def parse_date(datestring): """Parses ISO 8601 dates into datetime objects The timezone is parsed from the date string. However it is quite common to @@ -116,7 +120,7 @@ def parse_date(datestring, default_timezone=UTC): if not m: raise ParseError("Unable to parse date string %r" % datestring) groups = m.groupdict() - tz = parse_timezone(groups["timezone"], default_timezone=default_timezone) + tz = parse_timezone(groups["timezone"]) if groups["fraction"] is None: groups["fraction"] = 0 else: From b3cd5ca807f4366a570cfdd0bde7914eba7252aa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 22 Sep 2013 21:51:51 +1000 Subject: [PATCH 29/34] BF: correct scope for datetime.now --- fail2ban/server/mytime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 554d3544..96c7f8ab 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -77,7 +77,7 @@ class MyTime: #@staticmethod def now(): if MyTime.myTime is None: - return datetime.now() + return datetime.datetime.now() else: return datetime.datetime.fromtimestamp(MyTime.myTime) now = staticmethod(now) From 30d1f003e1c5f5f038f0887c1f6afcce0d7cd060 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Sep 2013 20:56:48 +1000 Subject: [PATCH 30/34] BF: add multiline support --- fail2ban/server/filter.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 68b27b32..41eb7505 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -74,6 +74,7 @@ class Filter(JailThread): self.__lineBuffer = [] ## Store last time stamp, applicable for multi-line self.__lastTimeLine = "" + self.__lastDate = None self.dateDetector = DateDetector() self.dateDetector.addDefaultTemplate() @@ -402,19 +403,28 @@ class Filter(JailThread): # The ignoreregex matched. Return. logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", logLine) return failList - dd = self.dateDetector.getTime(logLine) - - if dd is None: - return failList - date = dd[0] - timeMatch = dd[1] - if timeMatch: + + dateTimeMatch = self.dateDetector.getTime(logLine) + + if dateTimeMatch is not None: # Lets split into time part and log part of the line + date = dateTimeMatch[0] + timeMatch = dateTimeMatch[1] + timeLine = timeMatch.group() + self.__lastTimeLine = timeLine + self.__lastDate = date # Lets leave the beginning in as well, so if there is no # anchore at the beginning of the time regexp, we don't # at least allow injection. Should be harmless otherwise logLine = logLine[:timeMatch.start()] + logLine[timeMatch.end():] + else: + timeLine = self.__lastTimeLine or logLine + date = self.__lastDate + + self.__lineBuffer = (self.__lineBuffer + [logLine])[-self.__lineBufferSize:] + + logLine = "\n".join(self.__lineBuffer) + "\n" # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): From 6fd21797253006882cc54ac59704ffd3db087839 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Sep 2013 21:15:01 +1000 Subject: [PATCH 31/34] BF: timefix --- fail2ban/tests/filtertestcase.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b948e33c..643728d8 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -794,7 +794,7 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 6, 1124017144.0) + output = ('203.162.223.135', 7, 1124017144.0) self.filter.addLogPath(GetFailures.FILENAME_03) self.filter.addFailRegex("error,relay=,.*550 User unknown") @@ -864,8 +864,8 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [("192.0.43.10", 2, 1124017199.0), + ("192.0.43.11", 1, 1124017198.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) @@ -883,7 +883,7 @@ class GetFailures(unittest.TestCase): self.assertEqual(sorted(foundList), sorted(output)) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [("192.0.43.10", 2, 1124017199.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") @@ -897,9 +897,9 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + output = [("192.0.43.10", 2, 1124017199.0), + ("192.0.43.11", 1, 1124017198.0), + ("192.0.43.15", 1, 1124017198.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") From 7b52a578bd2781125226a618f297b29ce02f97d5 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Sep 2013 21:29:39 +1000 Subject: [PATCH 32/34] BF: group _f for %f in strptime for py2.5 compatibility --- fail2ban/server/datetemplate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 6309e8eb..34f1ed23 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -242,7 +242,7 @@ class DatePatternRegex(DateStrptime): _patternRegex = { 'a': r"\w{3}", 'A': r"\w+", 'b': r"\w{3}", 'B': r"\w+", 'd': r"(?:3[0-1]|[1-2]\d|[ 0]?\d)", - 'f': r"\d{1,6}", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)", + 'f': r"(?P<_f>\d{1,6})", 'H': r"(?:2[0-3]|1\d|[ 0]?\d)", 'I': r"(?:1[0-2]|[ 0]?\d)", 'j': r"(?:36[0-6]3[0-5]\d|[1-2]\d\d|[ 0]?\d\d|[ 0]{0,2}\d)", 'm': r"(?:1[0-2]|[ 0]?[1-9])", 'M': r"[0-5]\d", 'p': r"[AP]M", From 5cf25a63df6ffa244f82ba4cdbb184b5db78885e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Sep 2013 21:31:45 +1000 Subject: [PATCH 33/34] BF: remove duplicate ssh-pf in jail.conf --- config/jail.conf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 14fb694f..ff8bc625 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -532,15 +532,6 @@ bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 -# PF is a BSD based firewall -[ssh-pf] - -enabled=false -filter = sshd -action = pf -logpath = /var/log/sshd.log -maxretry=5 - [3proxy] enabled = false From 74434694dc46ce182ae404f51031397397adb9b7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Sep 2013 21:38:15 +1000 Subject: [PATCH 34/34] BF: more duplicate jail.conf entries - 3proxy exim{,-spam}, perdition --- config/jail.conf | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index ff8bc625..178a4c5f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -532,34 +532,6 @@ bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 -[3proxy] - -enabled = false -action = iptables[name=3proxy, port=3128, protocol=tcp] -logpath = /var/log/3proxy.log - - -[exim] - -enabled = false -action = iptables-multiport[name=exim,port="25,465,587"] -logpath = /var/log/exim/mainlog - - -[exim-spam] - -enabled = false -action = iptables-multiport[name=exim-spam,port="25,465,587"] -logpath = /var/log/exim/mainlog - - -[perdition] - -enabled = false -action = iptables-multiport[name=perdition,port="110,143,993,995"] -logpath = /var/log/maillog - - [osx-ssh-ipfw] enabled = false