From b0f26fa391ad2582804fd2b70689b8bb0bcdba3c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Aug 2014 12:45:50 -0400 Subject: [PATCH 01/17] Adjusting fail2ban logwatch script to match lines from 0.9 as well File itself includes additional log information about changes --- 3rdparty/logwatch/fail2ban | 86 ++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 087eb529..46dd11b5 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -3,6 +3,12 @@ # $Id: fail2ban 150 2013-06-18 22:19:38Z mtremaine $ ########################################################################## # $Log: fail2ban,v $ +# +# Revision 1.6 2014/08/11 16:07:46 yoh +# Patches from Yaroslav Halchenko to match adjusted in 0.9.x lines. +# Also reports now total number of hits (matches) along with Ban:Unban +# and relaxed regular expressions for matching any log level +# # Revision 1.5 2008/08/18 16:07:46 mike # Patches from Paul Gear -mgt # @@ -46,8 +52,8 @@ my $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0; my $IgnoreHost = $ENV{'sshd_ignore_host'} || ""; my $DebugCounter = 0; my $ReInitializations = 0; -my @IptablesErrors = (); -my @ActionErrors = (); +my @ActionsErrors = (); +my @CommandsErrors = (); my $NotValidIP = 0; # reported invalid IPs number my @OtherList = (); @@ -66,40 +72,55 @@ while (defined(my $ThisLine = )) { chomp($ThisLine); if ( ($ThisLine =~ /..,... DEBUG: /) or ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... INFO: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or - ($ThisLine =~ /INFO\s+Log rotation detected for/) or - ($ThisLine =~ /INFO\s+Jail.+(?:stopped|started|uses poller)/) or - ($ThisLine =~ /INFO\s+Changed logging target to/) or - ($ThisLine =~ /INFO\s+Creating new jail/) or + ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or + ($ThisLine =~ /\S+\s+rollover performed on/) or + ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or + ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or + ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or + ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or + ($ThisLine =~ /\S+\s+Log rotation detected for/) or + ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or + ($ThisLine =~ /\S+\s+Changed logging target to/) or + ($ThisLine =~ /\S+\s+Creating new jail/) or ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... WARNING: Verbose level is /) or - ($ThisLine =~ /..,... WARNING: Restoring firewall rules/) + ($ThisLine =~ /..,... \S+: Verbose level is /) or + ($ThisLine =~ /..,... \S+: Restoring firewall rules/) ) { if ( $Debug >= 6 ) { print STDERR "DEBUG($DebugCounter): line ignored\n"; } - } elsif ( my ($Service,$Action,$Host) = ($ThisLine =~ m/WARNING:?\s\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { + } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { if ( $Debug >= 6 ) { print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; } $ServicesBans{$Service}{$Host}{$Action}++; $ServicesBans{$Service}{"(all)"}{$Action}++; - } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/INFO: (\S+): (.+) has (\d+) login failure\(s\). Banned./)) { + } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{"Hit"}++; + $ServicesBans{$Service}{"(all)"}{"Hit"}++; + } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { if ($Debug >= 4) { print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; } push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ ERROR:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/WARNING\s*\[(.*)\]\s*(\S+)\s*already banned/)) { + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+\s*\[(.*)\]\s*(\S+)\s*already banned/)) { $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ WARNING:\s(.*):\sReBan (\S+)/)) { + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { $ServicesBans{$Service}{$Host}{'ReBan'}++; } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { - push @IptablesErrors, "$ThisLine\n"; + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { + push @CommandsErrors, "$ThisLine\n"; } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { - push @ActionErrors, "$ThisLine\n"; + push @ActionsErrors, "$ThisLine\n"; } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { $ReInitializations++; @@ -117,20 +138,22 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { - printf("\nBanned services with Fail2Ban: Bans:Unbans\n"); + printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%-3d]\n", "$service:", + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", $ServicesBans{$service}{'(all)'}{'Ban'}, - $ServicesBans{$service}{'(all)'}{'Unban'}); + $ServicesBans{$service}{'(all)'}{'Unban'}, + $ServicesBans{$service}{'(all)'}{'Hit'}); delete $ServicesBans{$service}{'(all)'}; my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); if ($Detail >= 5) { foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { my $name = LookupIP($ip); - printf(" %-53s %3d:%-3d\n", + printf(" %-53s %3d:%d:%-3d\n", $name, $ServicesBans{$service}{$ip}{'Ban'}, - $ServicesBans{$service}{$ip}{'Unban'}); + $ServicesBans{$service}{$ip}{'Unban'}, + $ServicesBans{$service}{$ip}{'Hit'}); if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { print " Failed "; foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { @@ -146,21 +169,20 @@ if (keys %ServicesBans) { } } - if ($Detail>0) { - if ($#IptablesErrors > 0) { - printf("\n%d faulty iptables invocation(s)", $#IptablesErrors); + if ($#ActionsErrors >= 0) { + printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); if ($Detail > 5) { print ":\n"; - print @IptablesErrors ; + print @ActionsErrors ; } } - if ($#ActionErrors > 0) { - printf("\n%d error(s) returned from actions", $#ActionErrors); - if ($Detail > 5) { - print ":\n"; - print @ActionErrors ; - } + if ($#CommandsErrors >= 0) { + printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @CommandsErrors ; + } } if ($ReInitializations > 0) { printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); From 3bd36ba40a67542a7201e86c8602927a23c7ee69 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:15:31 -0400 Subject: [PATCH 02/17] Sample logfiles to test logwatch services script --- 3rdparty/logwatch/fail2ban-0.8.log | 2 ++ 3rdparty/logwatch/fail2ban-0.9.log | 52 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 3rdparty/logwatch/fail2ban-0.8.log create mode 100644 3rdparty/logwatch/fail2ban-0.9.log diff --git a/3rdparty/logwatch/fail2ban-0.8.log b/3rdparty/logwatch/fail2ban-0.8.log new file mode 100644 index 00000000..f9d5f3d5 --- /dev/null +++ b/3rdparty/logwatch/fail2ban-0.8.log @@ -0,0 +1,2 @@ +2014-08-04 03:06:26,161 fail2ban.actions[4822]: WARNING [apache-badbots] Ban 37.152.91.34 +2014-08-05 03:06:26,448 fail2ban.actions[4822]: WARNING [apache-badbots] Unban 37.152.91.34 diff --git a/3rdparty/logwatch/fail2ban-0.9.log b/3rdparty/logwatch/fail2ban-0.9.log new file mode 100644 index 00000000..ad996a1c --- /dev/null +++ b/3rdparty/logwatch/fail2ban-0.9.log @@ -0,0 +1,52 @@ +2014-08-08 14:59:35,013 fail2ban.server.server[31122]: INFO Exiting Fail2ban +2014-08-08 14:59:36,041 fail2ban.server.server[21667]: INFO Changed logging target to /var/log/fail2ban.log for Fail2ban v0.9.0 +2014-08-08 14:59:36,043 fail2ban.server.database[21667]: INFO Connected to fail2ban persistent database '/var/lib/fail2ban/fail2ban.sqlite3' +2014-08-08 14:59:36,072 fail2ban.server.jail[21667]: INFO Creating new jail 'exim' +2014-08-08 14:59:36,137 fail2ban.server.jail[21667]: INFO Jail 'exim' uses pyinotify +2014-08-08 14:59:36,172 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,172 fail2ban.server.jail[21667]: INFO Initiated 'pyinotify' backend +2014-08-08 14:59:36,233 fail2ban.server.filter[21667]: INFO Added logfile = /var/log/exim4/mainlog +2014-08-08 14:59:36,249 fail2ban.server.filter[21667]: INFO Set maxRetry = 5 +2014-08-08 14:59:36,251 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,252 fail2ban.server.actions[21667]: INFO Set banTime = 600 +2014-08-08 14:59:36,254 fail2ban.server.filter[21667]: INFO Set findtime = 600 +2014-08-08 14:59:36,284 fail2ban.server.jail[21667]: INFO Creating new jail 'sshd' +2014-08-08 14:59:36,284 fail2ban.server.jail[21667]: INFO Jail 'sshd' uses pyinotify +2014-08-08 14:59:36,286 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,286 fail2ban.server.jail[21667]: INFO Initiated 'pyinotify' backend +2014-08-08 14:59:36,499 fail2ban.server.filter[21667]: INFO Added logfile = /var/log/auth.log +2014-08-08 14:59:36,510 fail2ban.server.filter[21667]: INFO Set maxRetry = 5 +2014-08-08 14:59:36,512 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,513 fail2ban.server.actions[21667]: INFO Set banTime = 600 +2014-08-08 14:59:36,514 fail2ban.server.filter[21667]: INFO Set findtime = 600 +2014-08-08 14:59:36,515 fail2ban.server.filter[21667]: INFO Set maxlines = 10 +2014-08-08 14:59:36,788 fail2ban.server.server[21667]: INFO Jail sshd is not a JournalFilter instance +2014-08-08 14:59:36,798 fail2ban.server.jail[21667]: INFO Jail 'exim' started +2014-08-08 14:59:36,802 fail2ban.server.jail[21667]: INFO Jail 'sshd' started +2014-08-08 15:01:30,120 fail2ban.server.transmitter[21667]: WARNING Command ['status', 'ssh'] has failed. Received UnknownJailException('ssh',) +2014-08-08 15:09:36,978 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 116.10.191.199 +2014-08-08 15:09:37,187 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- stdout: '' +2014-08-08 15:09:37,188 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- stderr: 'iptables: Bad rule (does a matching rule exist in that chain?).\n' +2014-08-08 15:09:37,188 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- returned 1 +2014-08-08 15:09:37,188 fail2ban.server.actions[21667]: ERROR Failed to execute unban jail 'sshd' action 'symbiosis-blacklist': Error unbanning 116.10.191.199 +2014-08-10 02:27:27,235 fail2ban.server.server[21667]: INFO rollover performed on /var/log/fail2ban.log +2014-08-10 02:27:28,109 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/exim4/mainlog +2014-08-10 02:28:01,747 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/auth.log +2014-08-10 02:33:29,500 fail2ban.server.filter[21667]: INFO [sshd] Found 86.101.234.57 +2014-08-10 02:46:06,846 fail2ban.server.filter[21667]: INFO [sshd] Found 220.130.163.247 +2014-08-10 03:10:43,794 fail2ban.server.filter[21667]: INFO [sshd] Found 220.130.163.247 +2014-08-10 06:49:27,446 fail2ban.server.actions[21667]: NOTICE [sshd] Ban 116.10.191.181 +2014-08-10 06:59:28,375 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 116.10.191.181 +2014-08-10 20:06:41,576 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 50.30.34.7 +2014-08-13 17:55:50,401 fail2ban.server.actions[17436]: NOTICE [sshd] 144.0.0.25 already banned +2014-08-10 20:06:41,785 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- stdout: '' +2014-08-10 20:06:41,785 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- stderr: 'iptables: Bad rule (does a matching rule exist in that chain?).\n' +2014-08-10 20:06:41,786 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- returned 1 +2014-08-10 20:06:41,786 fail2ban.server.actions[21667]: ERROR Failed to execute unban jail 'sshd' action 'symbiosis-blacklist': Error unbanning 50.30.34.7 +2014-08-11 02:27:35,433 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/exim4/mainlog From 8b62353ab06369023f8e71bdc1d3228ad1f988f6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:24:38 -0400 Subject: [PATCH 03/17] BF: logwatch -- fixing up regex for 'already banned' --- 3rdparty/logwatch/fail2ban | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 46dd11b5..117c7521 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -108,11 +108,14 @@ while (defined(my $ThisLine = )) { } push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+\s*\[(.*)\]\s*(\S+)\s*already banned/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; + } + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { - $ServicesBans{$Service}{$Host}{'ReBan'}++; + $ServicesBans{$Service}{$Host}{'ReBan'}++; } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { push @ActionsErrors, "$ThisLine\n"; } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { From decea64cf998d7252616625bfb8857f422cb46b8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:28:03 -0400 Subject: [PATCH 04/17] ENH: untabified and reindented entire script for sane formatting (no functional changes) --- 3rdparty/logwatch/fail2ban | 222 ++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 117c7521..2b028d63 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -54,87 +54,87 @@ my $DebugCounter = 0; my $ReInitializations = 0; my @ActionsErrors = (); my @CommandsErrors = (); -my $NotValidIP = 0; # reported invalid IPs number +my $NotValidIP = 0; # reported invalid IPs number my @OtherList = (); my %ServicesBans = (); if ( $Debug >= 5 ) { - print STDERR "\n\nDEBUG: Inside Fail2Ban Filter \n\n"; - $DebugCounter = 1; + print STDERR "\n\nDEBUG: Inside Fail2Ban Filter \n\n"; + $DebugCounter = 1; } while (defined(my $ThisLine = )) { - if ( $Debug >= 5 ) { - print STDERR "DEBUG($DebugCounter): $ThisLine"; - $DebugCounter++; - } - chomp($ThisLine); - if ( ($ThisLine =~ /..,... DEBUG: /) or - ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or - ($ThisLine =~ /\S+\s+rollover performed on/) or - ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or - ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or - ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or - ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or - ($ThisLine =~ /\S+\s+Log rotation detected for/) or - ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or - ($ThisLine =~ /\S+\s+Changed logging target to/) or - ($ThisLine =~ /\S+\s+Creating new jail/) or - ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... \S+: Verbose level is /) or - ($ThisLine =~ /..,... \S+: Restoring firewall rules/) + if ( $Debug >= 5 ) { + print STDERR "DEBUG($DebugCounter): $ThisLine"; + $DebugCounter++; + } + chomp($ThisLine); + if ( ($ThisLine =~ /..,... DEBUG: /) or + ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban + ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or + ($ThisLine =~ /\S+\s+rollover performed on/) or + ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or + ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or + ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or + ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or + ($ThisLine =~ /\S+\s+Log rotation detected for/) or + ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or + ($ThisLine =~ /\S+\s+Changed logging target to/) or + ($ThisLine =~ /\S+\s+Creating new jail/) or + ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban + ($ThisLine =~ /..,... \S+: Verbose level is /) or + ($ThisLine =~ /..,... \S+: Restoring firewall rules/) ) - { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): line ignored\n"; - } - } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; - } - $ServicesBans{$Service}{$Host}{$Action}++; - $ServicesBans{$Service}{"(all)"}{$Action}++; - } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; - } - $ServicesBans{$Service}{$Host}{"Hit"}++; - $ServicesBans{$Service}{"(all)"}{"Hit"}++; - } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { - if ($Debug >= 4) { - print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; - } - push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; - } - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { - $ServicesBans{$Service}{$Host}{'ReBan'}++; - } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { - push @CommandsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or - ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { - $ReInitializations++; - } elsif ($ThisLine =~ /..,... WARNING: is not a valid IP address/) { - # just ignore - this will be fixed within fail2ban and is harmless warning - } - else - { - # Report any unmatched entries... - push @OtherList, "$ThisLine\n"; - } + { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): line ignored\n"; + } + } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{$Action}++; + $ServicesBans{$Service}{"(all)"}{$Action}++; + } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{"Hit"}++; + $ServicesBans{$Service}{"(all)"}{"Hit"}++; + } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { + if ($Debug >= 4) { + print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; + } + push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; + } + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { + $ServicesBans{$Service}{$Host}{'ReBan'}++; + } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { + push @CommandsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or + ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { + $ReInitializations++; + } elsif ($ThisLine =~ /..,... WARNING: is not a valid IP address/) { + # just ignore - this will be fixed within fail2ban and is harmless warning + } + else + { + # Report any unmatched entries... + push @OtherList, "$ThisLine\n"; + } } ########################################################### @@ -143,56 +143,56 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%d:%-3d]\n", "$service:", - $ServicesBans{$service}{'(all)'}{'Ban'}, - $ServicesBans{$service}{'(all)'}{'Unban'}, - $ServicesBans{$service}{'(all)'}{'Hit'}); - delete $ServicesBans{$service}{'(all)'}; - my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); - if ($Detail >= 5) { - foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { - my $name = LookupIP($ip); - printf(" %-53s %3d:%d:%-3d\n", - $name, - $ServicesBans{$service}{$ip}{'Ban'}, - $ServicesBans{$service}{$ip}{'Unban'}, - $ServicesBans{$service}{$ip}{'Hit'}); - if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { - print " Failed "; - foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { - print " $fails"; - } - print " times"; - printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; - printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; - print "\n"; - } - } - } + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", + $ServicesBans{$service}{'(all)'}{'Ban'}, + $ServicesBans{$service}{'(all)'}{'Unban'}, + $ServicesBans{$service}{'(all)'}{'Hit'}); + delete $ServicesBans{$service}{'(all)'}; + my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); + if ($Detail >= 5) { + foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { + my $name = LookupIP($ip); + printf(" %-53s %3d:%d:%-3d\n", + $name, + $ServicesBans{$service}{$ip}{'Ban'}, + $ServicesBans{$service}{$ip}{'Unban'}, + $ServicesBans{$service}{$ip}{'Hit'}); + if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { + print " Failed "; + foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { + print " $fails"; + } + print " times"; + printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; + printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; + print "\n"; + } + } + } } } if ($Detail>0) { if ($#ActionsErrors >= 0) { - printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); - if ($Detail > 5) { - print ":\n"; - print @ActionsErrors ; - } + printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @ActionsErrors ; + } } if ($#CommandsErrors >= 0) { - printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); - if ($Detail > 5) { - print ":\n"; - print @CommandsErrors ; - } + printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @CommandsErrors ; + } } if ($ReInitializations > 0) { - printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); + printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); } if ($#OtherList >= 0) { - print "\n**Unmatched Entries**\n"; - print @OtherList; + print "\n**Unmatched Entries**\n"; + print @OtherList; } } From b1c04f5fa2b2ee2641588ecef005dd86105f7554 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:37:17 -0400 Subject: [PATCH 05/17] ENH: print rebans stats even if no "Failures" are logged, and reduce indentation in output --- 3rdparty/logwatch/fail2ban | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 2b028d63..56ae070b 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -143,7 +143,7 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%d:%-3d]\n", "$service:", + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", $ServicesBans{$service}{'(all)'}{'Ban'}, $ServicesBans{$service}{'(all)'}{'Unban'}, $ServicesBans{$service}{'(all)'}{'Hit'}); @@ -152,7 +152,7 @@ if (keys %ServicesBans) { if ($Detail >= 5) { foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { my $name = LookupIP($ip); - printf(" %-53s %3d:%d:%-3d\n", + printf(" %-53s %3d:%d:%-3d\n", $name, $ServicesBans{$service}{$ip}{'Ban'}, $ServicesBans{$service}{$ip}{'Unban'}, @@ -162,10 +162,13 @@ if (keys %ServicesBans) { foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { print " $fails"; } - print " times"; - printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; - printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; - print "\n"; + print " times\n"; + } + if ($ServicesBans{$service}{$ip}{'AlreadyInTheList'}>0) { + printf(" %d Duplicate Ban attempt(s)\n", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; + } + if ($ServicesBans{$service}{$ip}{'ReBan'}>0) { + printf(" %d ReBan(s) due to rules reinitilizations\n", $ServicesBans{$service}{$ip}{'ReBan'}) ; } } } From 544cfaff2c582945abb188e0e32e6ed249694fa3 Mon Sep 17 00:00:00 2001 From: Paul Traina Date: Sat, 6 Sep 2014 10:23:38 -0700 Subject: [PATCH 06/17] Add support for postfix/submission/smtpd matching. --- config/filter.d/postfix-sasl.conf | 2 +- config/filter.d/postfix.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index c5f8e3bc..52cff8c8 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -7,7 +7,7 @@ before = common.conf [Definition] -_daemon = postfix/smtpd +_daemon = postfix/(submission/)?smtpd failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$ diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index ff58c58c..9e34d9cc 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -10,7 +10,7 @@ before = common.conf [Definition] -_daemon = postfix/smtpd +_daemon = postfix/(submission/)?smtpd failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ From 249e169d8e36ae7b5b15164918304d7d04d831cf Mon Sep 17 00:00:00 2001 From: Paul Traina Date: Mon, 8 Sep 2014 11:53:51 -0700 Subject: [PATCH 07/17] Update test cases and also suport smtps per request. --- config/filter.d/postfix-sasl.conf | 2 +- config/filter.d/postfix.conf | 4 +++- fail2ban/tests/files/logs/postfix | 6 ++++++ fail2ban/tests/files/logs/postfix-sasl | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index 52cff8c8..7e70a423 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -7,7 +7,7 @@ before = common.conf [Definition] -_daemon = postfix/(submission/)?smtpd +_daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$ diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 9e34d9cc..1d7c63ac 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -10,11 +10,13 @@ before = common.conf [Definition] -_daemon = postfix/(submission/)?smtpd +_daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ + ^%(__prefix_line)swarning: Connection concurrency limit exceeded: \d+ from \S+\[\] for service.*$ + ^%(__prefix_line)slost connection after AUTH from \S+\[\]$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index ccf2f8bc..9cb4fa79 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -20,3 +20,9 @@ Dec 25 02:35:54 platypus postfix/smtpd[9144]: improper command pipelining after # failJSON: { "time": "2004-12-18T02:05:46", "match": true , "host": "216.245.198.245" } Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after NOOP from unknown[216.245.198.245] + +# failJSON: { "time": "2004-09-07T08:24:30", "match": true , "host": "83.110.98.15" } +Sep 7 08:24:30 umkc postfix/smtpd[2282]: lost connection after AUTH from bba453913.alshamil.net.ae[83.110.98.15] + +# failJSON: { "time": "2004-09-05T12:45:53", "match": true , "host": "213.123.185.20" } +Sep 5 12:45:53 umkc postfix/smtpd[20286]: warning: Connection concurrency limit exceeded: 6 from mail.baileysbodybuilders.co.uk[213.123.185.20] for service smtp diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl index ca306098..697ac424 100644 --- a/fail2ban/tests/files/logs/postfix-sasl +++ b/fail2ban/tests/files/logs/postfix-sasl @@ -5,3 +5,6 @@ Dec 2 22:24:22 hel postfix/smtpd[7676]: warning: 114-44-142-233.dynamic.hinet.n # failJSON: { "time": "2005-03-10T13:33:30", "match": true , "host": "1.1.1.1" } Mar 10 13:33:30 gandalf postfix/smtpd[3937]: warning: HOSTNAME[1.1.1.1]: SASL LOGIN authentication failed: authentication failure +#3 Example from postfix post-debian changes to rename to add "submission" to syslog name +# failJSON: { "time": "2004-09-06T00:44:56", "match": true , "host": "82.221.106.233" } +Sep 6 00:44:56 trianon postfix/submission/smtpd[11538]: warning: unknown[82.221.106.233]: SASL LOGIN authentication failed: UGFzc3dvcmQ6 From c58c4de9bc05f6ee73fa3af0ea2942a917b990db Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 13 Sep 2014 10:18:37 -0400 Subject: [PATCH 08/17] ENH: add empty ignoreregex to avoid a warning (Close #805) --- config/filter.d/postfix-sasl.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index c5f8e3bc..9cbcb324 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -11,6 +11,8 @@ _daemon = postfix/smtpd failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL (?:LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/]*={0,2})?\s*$ +ignoreregex = + [Init] journalmatch = _SYSTEMD_UNIT=postfix.service From 0e1f8f7f39e0a700e8ebc8cd103e5a7ade701682 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 13 Sep 2014 10:25:27 -0400 Subject: [PATCH 09/17] RF: remove those two additional failregexes for the postfix see comment https://github.com/fail2ban/fail2ban/pull/804\#discussion_r17512426 --- config/filter.d/postfix.conf | 2 -- fail2ban/tests/files/logs/postfix | 6 ------ 2 files changed, 8 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 1d7c63ac..a7a05e47 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,8 +15,6 @@ _daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ - ^%(__prefix_line)swarning: Connection concurrency limit exceeded: \d+ from \S+\[\] for service.*$ - ^%(__prefix_line)slost connection after AUTH from \S+\[\]$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 9cb4fa79..ccf2f8bc 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -20,9 +20,3 @@ Dec 25 02:35:54 platypus postfix/smtpd[9144]: improper command pipelining after # failJSON: { "time": "2004-12-18T02:05:46", "match": true , "host": "216.245.198.245" } Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after NOOP from unknown[216.245.198.245] - -# failJSON: { "time": "2004-09-07T08:24:30", "match": true , "host": "83.110.98.15" } -Sep 7 08:24:30 umkc postfix/smtpd[2282]: lost connection after AUTH from bba453913.alshamil.net.ae[83.110.98.15] - -# failJSON: { "time": "2004-09-05T12:45:53", "match": true , "host": "213.123.185.20" } -Sep 5 12:45:53 umkc postfix/smtpd[20286]: warning: Connection concurrency limit exceeded: 6 from mail.baileysbodybuilders.co.uk[213.123.185.20] for service smtp From 8f521b85516b2e51cd86878e89a1481b518cba8e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 13 Sep 2014 10:27:37 -0400 Subject: [PATCH 10/17] DOC: Changelog and THANKS for previous changes --- ChangeLog | 4 +++- THANKS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6a9bf03f..0d2ffcc6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,7 +48,9 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * cyrus-imap -- also catch also failed logins via secured (imaps/pop3s). Regression was introduced while strengthening failregex in 0.8.11 (bd175f) Debian bug #755173 - * postfix-sasl -- added journalmatch. Thanks Luc Maisonobe + * postfix-sasl - added journalmatch. Thanks Luc Maisonobe + * postfix* - match with a new daemon string (postfix/submission/smtpd). + Closes gh-804 . Thanks Paul Traina - New features: - New filters: diff --git a/THANKS b/THANKS index b9288d4d..42887a05 100644 --- a/THANKS +++ b/THANKS @@ -81,6 +81,7 @@ Mika (mkl) Nick Munger onorua Paul Marrapese +Paul Traina Noel Butler Patrick Börjesson Pressy From caa851e5c8f432088f933399e92a166d08fa0278 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 14 Sep 2014 09:48:14 -0400 Subject: [PATCH 11/17] RF: moving logwatch setup/sample logs under files/logwatch --- {3rdparty => files}/logwatch/fail2ban | 0 {3rdparty => files}/logwatch/fail2ban-0.8.log | 0 {3rdparty => files}/logwatch/fail2ban-0.9.log | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {3rdparty => files}/logwatch/fail2ban (100%) rename {3rdparty => files}/logwatch/fail2ban-0.8.log (100%) rename {3rdparty => files}/logwatch/fail2ban-0.9.log (100%) diff --git a/3rdparty/logwatch/fail2ban b/files/logwatch/fail2ban similarity index 100% rename from 3rdparty/logwatch/fail2ban rename to files/logwatch/fail2ban diff --git a/3rdparty/logwatch/fail2ban-0.8.log b/files/logwatch/fail2ban-0.8.log similarity index 100% rename from 3rdparty/logwatch/fail2ban-0.8.log rename to files/logwatch/fail2ban-0.8.log diff --git a/3rdparty/logwatch/fail2ban-0.9.log b/files/logwatch/fail2ban-0.9.log similarity index 100% rename from 3rdparty/logwatch/fail2ban-0.9.log rename to files/logwatch/fail2ban-0.9.log From 2c158fe16890b9e9f98a3428cc009f792618ba75 Mon Sep 17 00:00:00 2001 From: Nick Weeds Date: Sat, 13 Sep 2014 21:43:39 +0100 Subject: [PATCH 12/17] Add apache filter for AH01630 client denied by server configuration --- ChangeLog | 1 + config/filter.d/apache-auth.conf | 2 +- fail2ban/tests/files/logs/apache-auth | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 0d2ffcc6..1a98c1a0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -51,6 +51,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * postfix-sasl - added journalmatch. Thanks Luc Maisonobe * postfix* - match with a new daemon string (postfix/submission/smtpd). Closes gh-804 . Thanks Paul Traina + * apache - added filter for AH01630 client denied by server configuration. - New features: - New filters: diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index f4213487..8a63858d 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -10,7 +10,7 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (AH01797: )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$ +failregex = ^%(_apache_error_client)s (AH(01797|01630): )?client denied by server configuration: (uri )?\S*(, referer: \S+)?\s*$ ^%(_apache_error_client)s (AH01617: )?user .*? authentication failure for "\S*": Password Mismatch(, referer: \S+)?$ ^%(_apache_error_client)s (AH01618: )?user .*? not found(: )?\S*(, referer: \S+)?\s*$ ^%(_apache_error_client)s (AH01614: )?client used wrong authentication scheme: \S*(, referer: \S+)?\s*$ diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth index a01d2c76..5b7b3c48 100644 --- a/fail2ban/tests/files/logs/apache-auth +++ b/fail2ban/tests/files/logs/apache-auth @@ -19,6 +19,9 @@ # failJSON: { "time": "2013-07-20T21:34:49", "match": true , "host": "127.0.0.1" } [Sat Jul 20 21:34:49.453232 2013] [access_compat:error] [pid 17512:tid 140123104306944] [client 127.0.0.1:51380] AH01797: client denied by server configuration: /var/www/html/noentry/cant_get_me.html +# failJSON: { "time": "2014-09-14T21:44:43", "match": true , "host": "192.3.9.178" } +[Sun Sep 14 21:44:43.008606 2014] [authz_core:error] [pid 10691] [client 192.3.9.178:44271] AH01630: client denied by server configuration: /var/www/html/noentry/cant_get_me.html + # wget --http-user='' --http-password='' http://localhost/basic/file/cant_get_me.html -O /dev/null # failJSON: { "time": "2013-07-17T23:14:37", "match": true , "host": "127.0.0.1" } [Wed Jul 17 23:14:37 2013] [error] [client 127.0.0.1] user not found: /basic/anon/cant_get_me.html From 4f636eb0e3295ee1e8a35f1398926e95ab8b39ba Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Fri, 26 Sep 2014 16:25:07 +0100 Subject: [PATCH 13/17] adding filter to detect Shellshock attack attempts against bash scripts through apache. See http://seclists.org/oss-sec/2014/q3/650 --- config/filter.d/apache-shellshock.conf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 config/filter.d/apache-shellshock.conf diff --git a/config/filter.d/apache-shellshock.conf b/config/filter.d/apache-shellshock.conf new file mode 100644 index 00000000..39df1704 --- /dev/null +++ b/config/filter.d/apache-shellshock.conf @@ -0,0 +1,26 @@ +# Fail2Ban filter to block web requests containing custom headers attempting to exploit the shellshock bug +# +# + +[INCLUDES] + +# overwrite with apache-common.local if _apache_error_client is incorrect. +before = apache-common.conf + +[Definition] + +failregex = ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: warning: HTTP_.*?: ignoring function definition attempt(, referer: \S+)?\s*$ + ^%(_apache_error_client)s (AH01215: )?/bin/(ba)?sh: error importing function definition for `HTTP_.*?'(, referer: \S+)?\s*$ + +ignoreregex = + + +# DEV Notes: +# +# https://wiki.apache.org/httpd/ListOfErrors for apache error IDs +# +# example log lines: +# [Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt +# [Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST' +# +# Author: Eugene Hopkinson (riot@riot.so) From fc5f729f01dfaa8aae21e7f7a9603caf2e6fa626 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Fri, 26 Sep 2014 16:37:50 +0100 Subject: [PATCH 14/17] adding jail conf for shellshock filter --- config/jail.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index c48e6a7b..99729350 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -283,6 +283,11 @@ port = http,https logpath = %(apache_error_log)s maxretry = 2 +[apache-shellshock] + +port = http,https +logpath = $(apache_error_log)s +maxretry = 1 [nginx-http-auth] From 7b5dc9f24f5624ebe6d42048b3e81920a214c7fe Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Fri, 26 Sep 2014 18:48:56 +0100 Subject: [PATCH 15/17] adding test case, changelog and thanks entries for apache shellshock filter --- ChangeLog | 1 + THANKS | 1 + 2 files changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 1a98c1a0..d92aec4a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -57,6 +57,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - New filters: - monit Thanks Jason H Martin - directadmin Thanks niorg + - apache-shellshock Thanks Eugene Hopkinson (SlowRiot) - New actions: - symbiosis-blacklist-allports for Bytemark symbiosis firewall - fail2ban-client can fetch the running server version diff --git a/THANKS b/THANKS index 42887a05..0433f7ed 100644 --- a/THANKS +++ b/THANKS @@ -34,6 +34,7 @@ David Nutter Derek Atkins Eric Gerbier Enrico Labedzki +Eugene Hopkinson (SlowRiot) ftoppi François Boulogne Frédéric From 5d526bbeb15d484039c86ce655e42ce31b462714 Mon Sep 17 00:00:00 2001 From: SlowRiot Date: Mon, 29 Sep 2014 00:49:22 +0100 Subject: [PATCH 16/17] forgot to add test case to last commit --- fail2ban/tests/files/logs/apache-shellshock | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 fail2ban/tests/files/logs/apache-shellshock diff --git a/fail2ban/tests/files/logs/apache-shellshock b/fail2ban/tests/files/logs/apache-shellshock new file mode 100644 index 00000000..0acf4546 --- /dev/null +++ b/fail2ban/tests/files/logs/apache-shellshock @@ -0,0 +1,4 @@ +# failJSON: { "time": "2014-09-25T09:27:18", "match": true , "host": "89.207.132.76" } +[Thu Sep 25 09:27:18.813902 2014] [cgi:error] [pid 16860] [client 89.207.132.76:59635] AH01215: /bin/bash: warning: HTTP_TEST: ignoring function definition attempt +# failJSON: { "time": "2014-09-25T09:29:56", "match": true , "host": "162.247.73.206" } +[Thu Sep 25 09:29:56.141832 2014] [cgi:error] [pid 16864] [client 162.247.73.206:41273] AH01215: /bin/bash: error importing function definition for `HTTP_TEST' From 270ea363d35dd71fc2ca378e1606aaffe1863ec3 Mon Sep 17 00:00:00 2001 From: Daniel Schaal Date: Sun, 14 Sep 2014 08:38:39 +0200 Subject: [PATCH 17/17] tests: define CONFIG_DIR in utils. --- fail2ban/tests/action_d/test_badips.py | 6 +----- fail2ban/tests/action_d/test_smtp.py | 5 +---- fail2ban/tests/clientreadertestcase.py | 4 +++- fail2ban/tests/samplestestcase.py | 6 +----- fail2ban/tests/utils.py | 9 +++++++++ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py index 30e016ea..a7f148b1 100644 --- a/fail2ban/tests/action_d/test_badips.py +++ b/fail2ban/tests/action_d/test_badips.py @@ -22,11 +22,7 @@ import unittest import sys from ..dummyjail import DummyJail - -if os.path.exists('config/fail2ban.conf'): - CONFIG_DIR = "config" -else: - CONFIG_DIR='/etc/fail2ban' +from ..utils import CONFIG_DIR if sys.version_info >= (2,7): class BadIPsActionTest(unittest.TestCase): diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index a8a8a1a9..440db55c 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -30,10 +30,7 @@ else: from ..dummyjail import DummyJail -if os.path.exists('config/fail2ban.conf'): - CONFIG_DIR = "config" -else: - CONFIG_DIR='/etc/fail2ban' +from ..utils import CONFIG_DIR class TestSMTPServer(smtpd.SMTPServer): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index ce19a50e..0ad3c66e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -32,8 +32,10 @@ from ..client.configurator import Configurator from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + +from .utils import CONFIG_DIR + STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) -CONFIG_DIR='config' if STOCK else '/etc/fail2ban' IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config') diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 132ade7b..e0831184 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -32,13 +32,9 @@ else: from ..server.filter import Filter from ..client.filterreader import FilterReader -from .utils import setUpMyTime, tearDownMyTime +from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") -if os.path.exists('config/fail2ban.conf'): - CONFIG_DIR = "config" -else: - CONFIG_DIR='/etc/fail2ban' class FilterSamplesRegex(unittest.TestCase): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 0cb8d5b8..912c5a90 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -34,6 +34,15 @@ from ..helpers import getLogger logSys = getLogger(__name__) +CONFIG_DIR = os.environ.get('FAIL2BAN_CONFIG_DIR', None) + +if not CONFIG_DIR: +# Use heuristic to figure out where configuration files are + if os.path.exists(os.path.join('config','fail2ban.conf')): + CONFIG_DIR = 'config' + else: + CONFIG_DIR = '/etc/fail2ban' + def mtimesleep(): # no sleep now should be necessary since polling tracks now not only # mtime but also ino and size