From b0f26fa391ad2582804fd2b70689b8bb0bcdba3c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 11 Aug 2014 12:45:50 -0400 Subject: [PATCH 1/9] Adjusting fail2ban logwatch script to match lines from 0.9 as well File itself includes additional log information about changes --- 3rdparty/logwatch/fail2ban | 86 ++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 087eb529..46dd11b5 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -3,6 +3,12 @@ # $Id: fail2ban 150 2013-06-18 22:19:38Z mtremaine $ ########################################################################## # $Log: fail2ban,v $ +# +# Revision 1.6 2014/08/11 16:07:46 yoh +# Patches from Yaroslav Halchenko to match adjusted in 0.9.x lines. +# Also reports now total number of hits (matches) along with Ban:Unban +# and relaxed regular expressions for matching any log level +# # Revision 1.5 2008/08/18 16:07:46 mike # Patches from Paul Gear -mgt # @@ -46,8 +52,8 @@ my $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0; my $IgnoreHost = $ENV{'sshd_ignore_host'} || ""; my $DebugCounter = 0; my $ReInitializations = 0; -my @IptablesErrors = (); -my @ActionErrors = (); +my @ActionsErrors = (); +my @CommandsErrors = (); my $NotValidIP = 0; # reported invalid IPs number my @OtherList = (); @@ -66,40 +72,55 @@ while (defined(my $ThisLine = )) { chomp($ThisLine); if ( ($ThisLine =~ /..,... DEBUG: /) or ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... INFO: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or - ($ThisLine =~ /INFO\s+Log rotation detected for/) or - ($ThisLine =~ /INFO\s+Jail.+(?:stopped|started|uses poller)/) or - ($ThisLine =~ /INFO\s+Changed logging target to/) or - ($ThisLine =~ /INFO\s+Creating new jail/) or + ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or + ($ThisLine =~ /\S+\s+rollover performed on/) or + ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or + ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or + ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or + ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or + ($ThisLine =~ /\S+\s+Log rotation detected for/) or + ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or + ($ThisLine =~ /\S+\s+Changed logging target to/) or + ($ThisLine =~ /\S+\s+Creating new jail/) or ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... WARNING: Verbose level is /) or - ($ThisLine =~ /..,... WARNING: Restoring firewall rules/) + ($ThisLine =~ /..,... \S+: Verbose level is /) or + ($ThisLine =~ /..,... \S+: Restoring firewall rules/) ) { if ( $Debug >= 6 ) { print STDERR "DEBUG($DebugCounter): line ignored\n"; } - } elsif ( my ($Service,$Action,$Host) = ($ThisLine =~ m/WARNING:?\s\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { + } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { if ( $Debug >= 6 ) { print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; } $ServicesBans{$Service}{$Host}{$Action}++; $ServicesBans{$Service}{"(all)"}{$Action}++; - } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/INFO: (\S+): (.+) has (\d+) login failure\(s\). Banned./)) { + } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{"Hit"}++; + $ServicesBans{$Service}{"(all)"}{"Hit"}++; + } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { if ($Debug >= 4) { print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; } push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ ERROR:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/WARNING\s*\[(.*)\]\s*(\S+)\s*already banned/)) { + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+\s*\[(.*)\]\s*(\S+)\s*already banned/)) { $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ WARNING:\s(.*):\sReBan (\S+)/)) { + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { $ServicesBans{$Service}{$Host}{'ReBan'}++; } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { - push @IptablesErrors, "$ThisLine\n"; + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { + push @CommandsErrors, "$ThisLine\n"; } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { - push @ActionErrors, "$ThisLine\n"; + push @ActionsErrors, "$ThisLine\n"; } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { $ReInitializations++; @@ -117,20 +138,22 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { - printf("\nBanned services with Fail2Ban: Bans:Unbans\n"); + printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%-3d]\n", "$service:", + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", $ServicesBans{$service}{'(all)'}{'Ban'}, - $ServicesBans{$service}{'(all)'}{'Unban'}); + $ServicesBans{$service}{'(all)'}{'Unban'}, + $ServicesBans{$service}{'(all)'}{'Hit'}); delete $ServicesBans{$service}{'(all)'}; my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); if ($Detail >= 5) { foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { my $name = LookupIP($ip); - printf(" %-53s %3d:%-3d\n", + printf(" %-53s %3d:%d:%-3d\n", $name, $ServicesBans{$service}{$ip}{'Ban'}, - $ServicesBans{$service}{$ip}{'Unban'}); + $ServicesBans{$service}{$ip}{'Unban'}, + $ServicesBans{$service}{$ip}{'Hit'}); if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { print " Failed "; foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { @@ -146,21 +169,20 @@ if (keys %ServicesBans) { } } - if ($Detail>0) { - if ($#IptablesErrors > 0) { - printf("\n%d faulty iptables invocation(s)", $#IptablesErrors); + if ($#ActionsErrors >= 0) { + printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); if ($Detail > 5) { print ":\n"; - print @IptablesErrors ; + print @ActionsErrors ; } } - if ($#ActionErrors > 0) { - printf("\n%d error(s) returned from actions", $#ActionErrors); - if ($Detail > 5) { - print ":\n"; - print @ActionErrors ; - } + if ($#CommandsErrors >= 0) { + printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @CommandsErrors ; + } } if ($ReInitializations > 0) { printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); From 3bd36ba40a67542a7201e86c8602927a23c7ee69 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:15:31 -0400 Subject: [PATCH 2/9] Sample logfiles to test logwatch services script --- 3rdparty/logwatch/fail2ban-0.8.log | 2 ++ 3rdparty/logwatch/fail2ban-0.9.log | 52 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 3rdparty/logwatch/fail2ban-0.8.log create mode 100644 3rdparty/logwatch/fail2ban-0.9.log diff --git a/3rdparty/logwatch/fail2ban-0.8.log b/3rdparty/logwatch/fail2ban-0.8.log new file mode 100644 index 00000000..f9d5f3d5 --- /dev/null +++ b/3rdparty/logwatch/fail2ban-0.8.log @@ -0,0 +1,2 @@ +2014-08-04 03:06:26,161 fail2ban.actions[4822]: WARNING [apache-badbots] Ban 37.152.91.34 +2014-08-05 03:06:26,448 fail2ban.actions[4822]: WARNING [apache-badbots] Unban 37.152.91.34 diff --git a/3rdparty/logwatch/fail2ban-0.9.log b/3rdparty/logwatch/fail2ban-0.9.log new file mode 100644 index 00000000..ad996a1c --- /dev/null +++ b/3rdparty/logwatch/fail2ban-0.9.log @@ -0,0 +1,52 @@ +2014-08-08 14:59:35,013 fail2ban.server.server[31122]: INFO Exiting Fail2ban +2014-08-08 14:59:36,041 fail2ban.server.server[21667]: INFO Changed logging target to /var/log/fail2ban.log for Fail2ban v0.9.0 +2014-08-08 14:59:36,043 fail2ban.server.database[21667]: INFO Connected to fail2ban persistent database '/var/lib/fail2ban/fail2ban.sqlite3' +2014-08-08 14:59:36,072 fail2ban.server.jail[21667]: INFO Creating new jail 'exim' +2014-08-08 14:59:36,137 fail2ban.server.jail[21667]: INFO Jail 'exim' uses pyinotify +2014-08-08 14:59:36,172 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,172 fail2ban.server.jail[21667]: INFO Initiated 'pyinotify' backend +2014-08-08 14:59:36,233 fail2ban.server.filter[21667]: INFO Added logfile = /var/log/exim4/mainlog +2014-08-08 14:59:36,249 fail2ban.server.filter[21667]: INFO Set maxRetry = 5 +2014-08-08 14:59:36,251 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,252 fail2ban.server.actions[21667]: INFO Set banTime = 600 +2014-08-08 14:59:36,254 fail2ban.server.filter[21667]: INFO Set findtime = 600 +2014-08-08 14:59:36,284 fail2ban.server.jail[21667]: INFO Creating new jail 'sshd' +2014-08-08 14:59:36,284 fail2ban.server.jail[21667]: INFO Jail 'sshd' uses pyinotify +2014-08-08 14:59:36,286 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,286 fail2ban.server.jail[21667]: INFO Initiated 'pyinotify' backend +2014-08-08 14:59:36,499 fail2ban.server.filter[21667]: INFO Added logfile = /var/log/auth.log +2014-08-08 14:59:36,510 fail2ban.server.filter[21667]: INFO Set maxRetry = 5 +2014-08-08 14:59:36,512 fail2ban.server.filter[21667]: INFO Set jail log file encoding to UTF-8 +2014-08-08 14:59:36,513 fail2ban.server.actions[21667]: INFO Set banTime = 600 +2014-08-08 14:59:36,514 fail2ban.server.filter[21667]: INFO Set findtime = 600 +2014-08-08 14:59:36,515 fail2ban.server.filter[21667]: INFO Set maxlines = 10 +2014-08-08 14:59:36,788 fail2ban.server.server[21667]: INFO Jail sshd is not a JournalFilter instance +2014-08-08 14:59:36,798 fail2ban.server.jail[21667]: INFO Jail 'exim' started +2014-08-08 14:59:36,802 fail2ban.server.jail[21667]: INFO Jail 'sshd' started +2014-08-08 15:01:30,120 fail2ban.server.transmitter[21667]: WARNING Command ['status', 'ssh'] has failed. Received UnknownJailException('ssh',) +2014-08-08 15:09:36,978 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 116.10.191.199 +2014-08-08 15:09:37,187 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- stdout: '' +2014-08-08 15:09:37,188 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- stderr: 'iptables: Bad rule (does a matching rule exist in that chain?).\n' +2014-08-08 15:09:37,188 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/116.10.191.199.auto +iptables -D INPUT -s 116.10.191.199 -j DROP -- returned 1 +2014-08-08 15:09:37,188 fail2ban.server.actions[21667]: ERROR Failed to execute unban jail 'sshd' action 'symbiosis-blacklist': Error unbanning 116.10.191.199 +2014-08-10 02:27:27,235 fail2ban.server.server[21667]: INFO rollover performed on /var/log/fail2ban.log +2014-08-10 02:27:28,109 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/exim4/mainlog +2014-08-10 02:28:01,747 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/auth.log +2014-08-10 02:33:29,500 fail2ban.server.filter[21667]: INFO [sshd] Found 86.101.234.57 +2014-08-10 02:46:06,846 fail2ban.server.filter[21667]: INFO [sshd] Found 220.130.163.247 +2014-08-10 03:10:43,794 fail2ban.server.filter[21667]: INFO [sshd] Found 220.130.163.247 +2014-08-10 06:49:27,446 fail2ban.server.actions[21667]: NOTICE [sshd] Ban 116.10.191.181 +2014-08-10 06:59:28,375 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 116.10.191.181 +2014-08-10 20:06:41,576 fail2ban.server.actions[21667]: NOTICE [sshd] Unban 50.30.34.7 +2014-08-13 17:55:50,401 fail2ban.server.actions[17436]: NOTICE [sshd] 144.0.0.25 already banned +2014-08-10 20:06:41,785 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- stdout: '' +2014-08-10 20:06:41,785 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- stderr: 'iptables: Bad rule (does a matching rule exist in that chain?).\n' +2014-08-10 20:06:41,786 fail2ban.server.action[21667]: ERROR rm -f /etc/symbiosis/firewall/blacklist.d/50.30.34.7.auto +iptables -D INPUT -s 50.30.34.7 -j DROP -- returned 1 +2014-08-10 20:06:41,786 fail2ban.server.actions[21667]: ERROR Failed to execute unban jail 'sshd' action 'symbiosis-blacklist': Error unbanning 50.30.34.7 +2014-08-11 02:27:35,433 fail2ban.server.filter[21667]: INFO Log rotation detected for /var/log/exim4/mainlog From 8b62353ab06369023f8e71bdc1d3228ad1f988f6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:24:38 -0400 Subject: [PATCH 3/9] BF: logwatch -- fixing up regex for 'already banned' --- 3rdparty/logwatch/fail2ban | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 46dd11b5..117c7521 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -108,11 +108,14 @@ while (defined(my $ThisLine = )) { } push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+\s*\[(.*)\]\s*(\S+)\s*already banned/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; + } + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { - $ServicesBans{$Service}{$Host}{'ReBan'}++; + $ServicesBans{$Service}{$Host}{'ReBan'}++; } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { push @ActionsErrors, "$ThisLine\n"; } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { From decea64cf998d7252616625bfb8857f422cb46b8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:28:03 -0400 Subject: [PATCH 4/9] ENH: untabified and reindented entire script for sane formatting (no functional changes) --- 3rdparty/logwatch/fail2ban | 222 ++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 117c7521..2b028d63 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -54,87 +54,87 @@ my $DebugCounter = 0; my $ReInitializations = 0; my @ActionsErrors = (); my @CommandsErrors = (); -my $NotValidIP = 0; # reported invalid IPs number +my $NotValidIP = 0; # reported invalid IPs number my @OtherList = (); my %ServicesBans = (); if ( $Debug >= 5 ) { - print STDERR "\n\nDEBUG: Inside Fail2Ban Filter \n\n"; - $DebugCounter = 1; + print STDERR "\n\nDEBUG: Inside Fail2Ban Filter \n\n"; + $DebugCounter = 1; } while (defined(my $ThisLine = )) { - if ( $Debug >= 5 ) { - print STDERR "DEBUG($DebugCounter): $ThisLine"; - $DebugCounter++; - } - chomp($ThisLine); - if ( ($ThisLine =~ /..,... DEBUG: /) or - ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or - ($ThisLine =~ /\S+\s+rollover performed on/) or - ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or - ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or - ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or - ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or - ($ThisLine =~ /\S+\s+Log rotation detected for/) or - ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or - ($ThisLine =~ /\S+\s+Changed logging target to/) or - ($ThisLine =~ /\S+\s+Creating new jail/) or - ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban - ($ThisLine =~ /..,... \S+: Verbose level is /) or - ($ThisLine =~ /..,... \S+: Restoring firewall rules/) + if ( $Debug >= 5 ) { + print STDERR "DEBUG($DebugCounter): $ThisLine"; + $DebugCounter++; + } + chomp($ThisLine); + if ( ($ThisLine =~ /..,... DEBUG: /) or + ($ThisLine =~ /..,... \S*\s*: DEBUG /) or # syntax of 0.7.? fail2ban + ($ThisLine =~ /..,... \S+: (Fail2Ban v.* is running|Exiting|Enabled sections:)/) or + ($ThisLine =~ /\S+\s+rollover performed on/) or + ($ThisLine =~ /\S+\s+Connected to .* persistent database/) or + ($ThisLine =~ /\S+\s+Jail '.*' uses .*/) or + ($ThisLine =~ /\S+\s+Initiated '.*' backend/) or + ($ThisLine =~ /\S+\s+Jail .* is not a JournalFilter instance/) or + ($ThisLine =~ /\S+\s+Log rotation detected for/) or + ($ThisLine =~ /\S+\s+Jail.+(?:stopped|started|uses poller)/) or + ($ThisLine =~ /\S+\s+Changed logging target to/) or + ($ThisLine =~ /\S+\s+Creating new jail/) or + ($ThisLine =~ /..,... \S+\s*: INFO\s+(Set |Socket|Exiting|Gamin|Created|Added|Using)/) or # syntax of 0.7.? fail2ban + ($ThisLine =~ /..,... \S+: Verbose level is /) or + ($ThisLine =~ /..,... \S+: Restoring firewall rules/) ) - { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): line ignored\n"; - } - } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; - } - $ServicesBans{$Service}{$Host}{$Action}++; - $ServicesBans{$Service}{"(all)"}{$Action}++; - } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; - } - $ServicesBans{$Service}{$Host}{"Hit"}++; - $ServicesBans{$Service}{"(all)"}{"Hit"}++; - } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { - if ($Debug >= 4) { - print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; - } - push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { - if ( $Debug >= 6 ) { - print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; - } - $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; - } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { - $ServicesBans{$Service}{$Host}{'ReBan'}++; - } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { - push @CommandsErrors, "$ThisLine\n"; - } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { - push @ActionsErrors, "$ThisLine\n"; - } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or - ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { - $ReInitializations++; - } elsif ($ThisLine =~ /..,... WARNING: is not a valid IP address/) { - # just ignore - this will be fixed within fail2ban and is harmless warning - } - else - { - # Report any unmatched entries... - push @OtherList, "$ThisLine\n"; - } + { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): line ignored\n"; + } + } elsif ( my ($LogLevel,$Service,$Action,$Host) = ($ThisLine =~ m/(WARNING|NOTICE):?\s+\[?(.*?)[]:]?\s(Ban|Unban)[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found $Action for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{$Action}++; + $ServicesBans{$Service}{"(all)"}{$Action}++; + } elsif ( my ($LogLevel,$Service,$Host) = ($ThisLine =~ m/(INFO|WARNING|NOTICE):?\s+\[?(.*?)[]:]?\sFound[^\.]* (\S+)/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for $Service from $Host\n"; + } + $ServicesBans{$Service}{$Host}{"Hit"}++; + $ServicesBans{$Service}{"(all)"}{"Hit"}++; + } elsif ( my ($Service,$Host,$NumFailures) = ($ThisLine =~ m/\S+:\s+(\S+): (.+) has (\d+) login failure\(s\). Banned./)) { + if ($Debug >= 4) { + print STDERR "DEBUG: Found host $Host trying to access $Service - failed $NumFailures times\n"; + } + push @{$ServicesBans{$Service}{$Host}{'Failures'}}, $NumFailures; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\s(\S+)\salready in ban list/)) { + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/\S+:?\s+\[?([^[]*?)[]:]?\s+(\S+)\salready banned/)) { + if ( $Debug >= 6 ) { + print STDERR "DEBUG($DebugCounter): Found hit for already banned $Host against $Service\n"; + } + $ServicesBans{$Service}{$Host}{'AlreadyInTheList'}++; + } elsif ( my ($Service,$Host) = ($ThisLine =~ m/ \S+:\s(.*):\sReBan (\S+)/)) { + $ServicesBans{$Service}{$Host}{'ReBan'}++; + } elsif ($ThisLine =~ / ERROR:?\s*(Execution of command )?\'?iptables/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / ERROR\s*Failed to execute.*action/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ / WARNING Command \[.*\] has failed. Received/) { + push @CommandsErrors, "$ThisLine\n"; + } elsif ($ThisLine =~ /ERROR.*returned \d+$/) { + push @ActionsErrors, "$ThisLine\n"; + } elsif (($ThisLine =~ /..,... WARNING: \#\S+ reinitialization of firewalls/) or + ($ThisLine =~ / ERROR\s*Invariant check failed. Trying to restore a sane environment/)) { + $ReInitializations++; + } elsif ($ThisLine =~ /..,... WARNING: is not a valid IP address/) { + # just ignore - this will be fixed within fail2ban and is harmless warning + } + else + { + # Report any unmatched entries... + push @OtherList, "$ThisLine\n"; + } } ########################################################### @@ -143,56 +143,56 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%d:%-3d]\n", "$service:", - $ServicesBans{$service}{'(all)'}{'Ban'}, - $ServicesBans{$service}{'(all)'}{'Unban'}, - $ServicesBans{$service}{'(all)'}{'Hit'}); - delete $ServicesBans{$service}{'(all)'}; - my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); - if ($Detail >= 5) { - foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { - my $name = LookupIP($ip); - printf(" %-53s %3d:%d:%-3d\n", - $name, - $ServicesBans{$service}{$ip}{'Ban'}, - $ServicesBans{$service}{$ip}{'Unban'}, - $ServicesBans{$service}{$ip}{'Hit'}); - if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { - print " Failed "; - foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { - print " $fails"; - } - print " times"; - printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; - printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; - print "\n"; - } - } - } + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", + $ServicesBans{$service}{'(all)'}{'Ban'}, + $ServicesBans{$service}{'(all)'}{'Unban'}, + $ServicesBans{$service}{'(all)'}{'Hit'}); + delete $ServicesBans{$service}{'(all)'}; + my $totalSort = TotalCountOrder(%{$ServicesBans{$service}}, \&SortIP); + if ($Detail >= 5) { + foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { + my $name = LookupIP($ip); + printf(" %-53s %3d:%d:%-3d\n", + $name, + $ServicesBans{$service}{$ip}{'Ban'}, + $ServicesBans{$service}{$ip}{'Unban'}, + $ServicesBans{$service}{$ip}{'Hit'}); + if (($Detail >= 10) and ($ServicesBans{$service}{$ip}{'Failures'}>0)) { + print " Failed "; + foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { + print " $fails"; + } + print " times"; + printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; + printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; + print "\n"; + } + } + } } } if ($Detail>0) { if ($#ActionsErrors >= 0) { - printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); - if ($Detail > 5) { - print ":\n"; - print @ActionsErrors ; - } + printf("\n%d faulty action invocation(s)", $#ActionsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @ActionsErrors ; + } } if ($#CommandsErrors >= 0) { - printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); - if ($Detail > 5) { - print ":\n"; - print @CommandsErrors ; - } + printf("\n%d faulty command invocation(s) from client(s)", $#CommandsErrors+1); + if ($Detail > 5) { + print ":\n"; + print @CommandsErrors ; + } } if ($ReInitializations > 0) { - printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); + printf("\n%d fail2ban rules reinitialization(s)", $ReInitializations); } if ($#OtherList >= 0) { - print "\n**Unmatched Entries**\n"; - print @OtherList; + print "\n**Unmatched Entries**\n"; + print @OtherList; } } From b1c04f5fa2b2ee2641588ecef005dd86105f7554 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 13 Aug 2014 23:37:17 -0400 Subject: [PATCH 5/9] ENH: print rebans stats even if no "Failures" are logged, and reduce indentation in output --- 3rdparty/logwatch/fail2ban | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/3rdparty/logwatch/fail2ban b/3rdparty/logwatch/fail2ban index 2b028d63..56ae070b 100755 --- a/3rdparty/logwatch/fail2ban +++ b/3rdparty/logwatch/fail2ban @@ -143,7 +143,7 @@ while (defined(my $ThisLine = )) { if (keys %ServicesBans) { printf("\nBanned services with Fail2Ban: Bans:Unbans:Hits\n"); foreach my $service (sort {$a cmp $b} keys %ServicesBans) { - printf(" %-55s [%3d:%d:%-3d]\n", "$service:", + printf(" %-55s [%3d:%d:%-3d]\n", "$service:", $ServicesBans{$service}{'(all)'}{'Ban'}, $ServicesBans{$service}{'(all)'}{'Unban'}, $ServicesBans{$service}{'(all)'}{'Hit'}); @@ -152,7 +152,7 @@ if (keys %ServicesBans) { if ($Detail >= 5) { foreach my $ip (sort $totalSort keys %{$ServicesBans{$service}}) { my $name = LookupIP($ip); - printf(" %-53s %3d:%d:%-3d\n", + printf(" %-53s %3d:%d:%-3d\n", $name, $ServicesBans{$service}{$ip}{'Ban'}, $ServicesBans{$service}{$ip}{'Unban'}, @@ -162,10 +162,13 @@ if (keys %ServicesBans) { foreach my $fails (@{$ServicesBans{$service}{$ip}{'Failures'}}) { print " $fails"; } - print " times"; - printf("\n %d Duplicate Ban attempts", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; - printf("\n %d ReBans due to rules reinitilizations", $ServicesBans{$service}{$ip}{'ReBan'}) ; - print "\n"; + print " times\n"; + } + if ($ServicesBans{$service}{$ip}{'AlreadyInTheList'}>0) { + printf(" %d Duplicate Ban attempt(s)\n", $ServicesBans{$service}{$ip}{'AlreadyInTheList'}) ; + } + if ($ServicesBans{$service}{$ip}{'ReBan'}>0) { + printf(" %d ReBan(s) due to rules reinitilizations\n", $ServicesBans{$service}{$ip}{'ReBan'}) ; } } } From caa851e5c8f432088f933399e92a166d08fa0278 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 14 Sep 2014 09:48:14 -0400 Subject: [PATCH 6/9] RF: moving logwatch setup/sample logs under files/logwatch --- {3rdparty => files}/logwatch/fail2ban | 0 {3rdparty => files}/logwatch/fail2ban-0.8.log | 0 {3rdparty => files}/logwatch/fail2ban-0.9.log | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {3rdparty => files}/logwatch/fail2ban (100%) rename {3rdparty => files}/logwatch/fail2ban-0.8.log (100%) rename {3rdparty => files}/logwatch/fail2ban-0.9.log (100%) diff --git a/3rdparty/logwatch/fail2ban b/files/logwatch/fail2ban similarity index 100% rename from 3rdparty/logwatch/fail2ban rename to files/logwatch/fail2ban diff --git a/3rdparty/logwatch/fail2ban-0.8.log b/files/logwatch/fail2ban-0.8.log similarity index 100% rename from 3rdparty/logwatch/fail2ban-0.8.log rename to files/logwatch/fail2ban-0.8.log diff --git a/3rdparty/logwatch/fail2ban-0.9.log b/files/logwatch/fail2ban-0.9.log similarity index 100% rename from 3rdparty/logwatch/fail2ban-0.9.log rename to files/logwatch/fail2ban-0.9.log From 86a5f42f736ffe2b776deff6c76d9a78dd70312c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 12 Oct 2014 16:40:29 -0400 Subject: [PATCH 7/9] BF: made tests util digest.py friendly to python3 --- .../tests/files/config/apache-auth/digest.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py index baac36d2..9906c652 100755 --- a/fail2ban/tests/files/config/apache-auth/digest.py +++ b/fail2ban/tests/files/config/apache-auth/digest.py @@ -1,14 +1,21 @@ #!/bin/env python import requests -import md5 +try: + import hashlib + md5sum = hashlib.md5 +except ImportError: # pragma: no cover + # hashlib was introduced in Python 2.5. For compatibility with those + # elderly Pythons, import from md5 + import md5 + md5sum = md5.new def auth(v): - ha1 = md5.new(username + ':' + realm + ':' + password).hexdigest() - ha2 = md5.new("GET:" + url).hexdigest() + ha1 = md5sum(username + ':' + realm + ':' + password).hexdigest() + ha2 = md5sum("GET:" + url).hexdigest() - #response = md5.new(ha1 + ':' + v['nonce'][1:-1] + ':' + v['nc'] + ':' + v['cnonce'][1:-1] + #response = md5sum(ha1 + ':' + v['nonce'][1:-1] + ':' + v['nc'] + ':' + v['cnonce'][1:-1] # + ':' + v['qop'][1:-1] + ':' + ha2).hexdigest() nonce = v['nonce'][1:-1] @@ -17,7 +24,7 @@ def auth(v): #opaque = v.get('opaque') or '' qop = v['qop'][1:-1] algorithm = v['algorithm'] - response = md5.new(ha1 + ':' + nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2).hexdigest() + response = md5sum(ha1 + ':' + nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2).hexdigest() p = requests.Request('GET', host + url).prepare() #p.headers['Authentication-Info'] = response @@ -33,13 +40,13 @@ def auth(v): response="%s" """ % ( username, algorithm, realm, url, nonce, qop, response ) # opaque="%s", - print p.method, p.url, p.headers + print(p.method, p.url, p.headers) s = requests.Session() return s.send(p) def preauth(): r = requests.get(host + url) - print r + print(r) r.headers['www-authenticate'].split(', ') return dict([ a.split('=',1) for a in r.headers['www-authenticate'].split(', ') ]) @@ -51,7 +58,7 @@ v = preauth() username="username" password = "password" -print v +print(v) realm = 'so far away' r = auth(v) @@ -67,18 +74,18 @@ r = auth(v) # [Sun Jul 28 21:41:20 2013] [error] [client 127.0.0.1] Digest: unknown algorithm `super funky chicken' received: /digest/ -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) v['algorithm'] = algorithm r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) nonce = v['nonce'] v['nonce']=v['nonce'][5:-5] r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) # [Sun Jul 28 21:05:31.178340 2013] [auth_digest:error] [pid 24224:tid 139895539455744] [client 127.0.0.1:56906] AH01793: invalid qop `auth' received: /digest/qop_none/ @@ -86,7 +93,7 @@ print r.status_code,r.headers, r.text v['nonce']=nonce[0:11] + 'ZZZ' + nonce[14:] r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) #[Sun Jul 28 21:18:11.769228 2013] [auth_digest:error] [pid 24752:tid 139895505884928] [client 127.0.0.1:56964] AH01776: invalid nonce b9YAiJDiBAZZZ1b1abe02d20063ea3b16b544ea1b0d981c1bafe received - hash is not d42d824dee7aaf50c3ba0a7c6290bd453e3dd35b @@ -98,7 +105,7 @@ import time time.sleep(1) r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) # Obtained by putting the following code in modules/aaa/mod_auth_digest.c # in the function initialize_secret @@ -128,7 +135,7 @@ s = sha.sha(apachesecret) v=preauth() -print v['nonce'] +print(v['nonce']) realm = v['Digest realm'][1:-1] (t,) = struct.unpack('l',base64.b64decode(v['nonce'][1:13])) @@ -143,17 +150,17 @@ s.update(timepac) v['nonce'] = v['nonce'][0] + timepac + s.hexdigest() + v['nonce'][-1] -print v +print(v) r = auth(v) #[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) url='/digest_onetime/' v=preauth() # Need opaque header handling in auth r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) r = auth(v) -print r.status_code,r.headers, r.text +print(r.status_code,r.headers, r.text) From 5ac496d0309c0901fef00efdea39fa1eec40b4f4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 12 Oct 2014 17:28:35 -0400 Subject: [PATCH 8/9] We better check that installation doesn't cause any errors as well --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9a92a7f6..bd2d294c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,8 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi script: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi +# test installation + - sudo python setup.py install after_success: # Coverage config file must be .coveragerc for coveralls - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cp -v .travis_coveragerc .coveragerc; fi From e2f49b7334c20f01ddc3f88a1a4fc50799d7775b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 23 Oct 2014 14:34:17 -0400 Subject: [PATCH 9/9] DOC: very minor (tabs/spaces) --- ChangeLog | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0515823a..ad92a55e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,8 +16,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger provides defaults for the chain, port, protocol and name tags - Fixes: - * start of file2ban aborted (on slow hosts, systemd considers the server has been - timed out and kills him), see gh-824 + * start of file2ban aborted (on slow hosts, systemd considers the server has + been timed out and kills him), see gh-824 * UTF-8 fixes in pure-ftp thanks to Johannes Weberhofer. Closes gh-806. * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request @@ -67,8 +67,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Enhancements * Start performance of fail2ban-client (and tests) increased, start time - and cpu usage rapidly reduced. Introduced a shared storage logic, to bypass - reading lots of config files (see gh-824). + and cpu usage rapidly reduced. Introduced a shared storage logic, to + bypass reading lots of config files (see gh-824). Thanks to Joost Molenaar for good catch (reported gh-820). * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options