From 415f1876440272b2077019824af96a32f5d7664a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Mar 2014 07:12:12 +1100 Subject: [PATCH 1/5] ENH: sendmail-reject for all smtp ports. --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 5ba1c7c4..effd0381 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -169,7 +169,7 @@ logpath = /var/log/mail.log enabled = false filter = sendmail-reject -action = iptables[name=sendmail-reject, port=smtp, protocol=tcp] +action = iptables-multiport[name=sendmail-auth, port="submission,465,smtp", protocol=tcp] logpath = /var/log/mail.log From 27dafea2811ddf13bc16ffcb98dae9c8ce250e79 Mon Sep 17 00:00:00 2001 From: Hank Leininger Date: Thu, 13 Mar 2014 22:47:28 -0400 Subject: [PATCH 2/5] Fix the --print-all-{missed,ignored} options. Previously, fail2ban-regex always printed both regardless of whether those options were present or not. This commit duplicates the patch I submitted at https://sourceforge.net/p/fail2ban/patches/25/ --- fail2ban-regex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 18e2a5df..22fbbd67 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -354,8 +354,10 @@ class Fail2banRegex(object): print "\nLines: %s" % self._line_stats - self.printLines('ignored') - self.printLines('missed') + if self._print_all_ignored: + self.printLines('ignored') + if self._print_all_missed: + self.printLines('missed') return True From aa7af6f9c2b2feae17be7aafd5f3290c84494f8b Mon Sep 17 00:00:00 2001 From: Hank Leininger Date: Thu, 13 Mar 2014 23:18:08 -0400 Subject: [PATCH 3/5] Only remember log lines we need to print later. When testing proccessing a large logfile (430 MB, 2M lines) without --print-all-missed or --print-all-ignored, fail2ban-regex consumes just over 2 GB of RAM before completing. Even though it does not need to retain any of the missed or ignored log lines, it does anyway. With this patch, memory use never grows beyond about 11 MB (unless --print-all-* are enabled). --- fail2ban-regex | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 22fbbd67..941d0074 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -156,22 +156,16 @@ class LineStats(object): """ def __init__(self): self.tested = self.matched = 0 + self.missed = 0 self.missed_lines = [] self.missed_lines_timeextracted = [] + self.ignored = 0 self.ignored_lines = [] self.ignored_lines_timeextracted = [] def __str__(self): return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self - @property - def ignored(self): - return len(self.ignored_lines) - - @property - def missed(self): - return self.tested - (self.ignored + self.matched) - # just for convenient str def __getitem__(self, key): return getattr(self, key) @@ -267,16 +261,20 @@ class Fail2banRegex(object): line_datetimestripped, ret = fail2banRegex.testRegex(line) if is_ignored: - self._line_stats.ignored_lines.append(line) - self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) + self._line_stats.ignored += 1 + if self._print_all_ignored: + self._line_stats.ignored_lines.append(line) + self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) if len(ret) > 0: assert(not is_ignored) self._line_stats.matched += 1 else: if not is_ignored: - self._line_stats.missed_lines.append(line) - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + self._line_stats.missed += 1 + if self._print_all_missed: + self._line_stats.missed_lines.append(line) + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 if line_no % 10 == 0: From d4fb48fd142f2123f73679ed032a2c223619f215 Mon Sep 17 00:00:00 2001 From: Hank Leininger Date: Fri, 14 Mar 2014 01:57:55 -0400 Subject: [PATCH 4/5] Add --print-no-{missed,ignored} and restore -all. Realized --print-all-{missed,ignored} aren't meant to be simple on/off, but are meant to enable printing all lines even when there are more than 20. This restores the behavior of --print-all-*, but preserves the memory-savings when --print-all-* are not specified. Also adds --print-no-{missed,ignored} which can be used to suppress printing of a line type entirely. --- fail2ban-regex | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/fail2ban-regex b/fail2ban-regex index 941d0074..da5bcf88 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -111,10 +111,14 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues help="Be verbose in output"), Option("-D", "--debuggex", action='store_true', help="Produce debuggex.com urls for debugging there"), + Option("--print-no-missed", action='store_true', + help="Do not print any missed lines"), + Option("--print-no-ignored", action='store_true', + help="Do not print any ignored lines"), Option("--print-all-missed", action='store_true', - help="Either to print all missed lines"), + help="Print all missed lines, no matter how many"), Option("--print-all-ignored", action='store_true', - help="Either to print all ignored lines"), + help="Print all ignored lines, no matter how many"), Option("-t", "--log-traceback", action='store_true', help="Enrich log-messages with compressed tracebacks"), Option("--full-traceback", action='store_true', @@ -178,6 +182,9 @@ class Fail2banRegex(object): def __init__(self, opts): self._verbose = opts.verbose self._debuggex = opts.debuggex + self._maxlines = 20 + self._print_no_missed = opts.print_no_missed + self._print_no_ignored = opts.print_no_ignored self._print_all_missed = opts.print_all_missed self._print_all_ignored = opts.print_all_ignored @@ -262,7 +269,7 @@ class Fail2banRegex(object): if is_ignored: self._line_stats.ignored += 1 - if self._print_all_ignored: + if not self._print_no_ignored and (self._print_all_ignored or self._line_stats.ignored <= self._maxlines + 1): self._line_stats.ignored_lines.append(line) self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) @@ -272,7 +279,7 @@ class Fail2banRegex(object): else: if not is_ignored: self._line_stats.missed += 1 - if self._print_all_missed: + if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): self._line_stats.missed_lines.append(line) self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 @@ -284,9 +291,10 @@ class Fail2banRegex(object): def printLines(self, ltype): lstats = self._line_stats - assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) + assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) + lines = lstats[ltype] l = lstats[ltype + '_lines'] - if len(l): + if lines: header = "%s line(s):" % (ltype.capitalize(),) if self._debuggex: if ltype == 'missed': @@ -294,21 +302,20 @@ class Fail2banRegex(object): else: regexlist = self._ignoreregex l = lstats[ltype + '_lines_timeextracted'] - lines = len(l)*len(regexlist) - if lines < 20 or getattr(self, '_print_all_' + ltype): + if lines < self._maxlines or getattr(self, '_print_all_' + ltype): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: - print "%s: too many to print. Use --print-all-%s " \ + print "%s too many to print. Use --print-all-%s " \ "to print all %d lines" % (header, ltype, lines) - elif len(l) < 20 or getattr(self, '_print_all_' + ltype): + elif lines < self._maxlines or getattr(self, '_print_all_' + ltype): pprint_list([x.rstrip() for x in l], header) else: - print "%s: too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, len(l)) + print "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) def printStats(self): print @@ -352,9 +359,9 @@ class Fail2banRegex(object): print "\nLines: %s" % self._line_stats - if self._print_all_ignored: + if not self._print_no_ignored: self.printLines('ignored') - if self._print_all_missed: + if not self._print_no_missed: self.printLines('missed') return True From 2deb1b71a19cde27051264c35d1bb2821cc2b7bd Mon Sep 17 00:00:00 2001 From: Hank Leininger Date: Fri, 14 Mar 2014 02:10:00 -0400 Subject: [PATCH 5/5] Sanity-check print-all-* vs print-no-* options. Also, fixed a trivial whitespace issue. --- fail2ban-regex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fail2ban-regex b/fail2ban-regex index da5bcf88..9055360d 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -182,7 +182,7 @@ class Fail2banRegex(object): def __init__(self, opts): self._verbose = opts.verbose self._debuggex = opts.debuggex - self._maxlines = 20 + self._maxlines = 20 self._print_no_missed = opts.print_no_missed self._print_no_ignored = opts.print_no_ignored self._print_all_missed = opts.print_all_missed @@ -371,6 +371,14 @@ if __name__ == "__main__": parser = get_opt_parser() (opts, args) = parser.parse_args() + if opts.print_no_missed and opts.print_all_missed: + sys.stderr.write("ERROR: --print-no-missed and --print-all-missed are mutually exclusive.\n\n") + parser.print_help() + sys.exit(-1) + if opts.print_no_ignored and opts.print_all_ignored: + sys.stderr.write("ERROR: --print-no-ignored and --print-all-ignored are mutually exclusive.\n\n") + parser.print_help() + sys.exit(-1) fail2banRegex = Fail2banRegex(opts)