diff --git a/ChangeLog b/ChangeLog index 22671ec9..a575d4d1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,7 +8,7 @@ Fail2Ban (version 0.9.0a1) 20??/??/?? ================================================================================ -ver. 0.9.0 (2013/04/??) - alpha +ver. 0.9.0 (2013/??/??) - alpha ---------- Carries all fixes in 0.8.9 and new features and enhancements. Nearly @@ -41,6 +41,25 @@ code-review and minor additions from Yaroslav Halchenko. * [..e019ab7] Multiple instances of the same action are allowed in the same jail -- use actname option to disambiguate. +ver. 0.8.11 (2013/XX/XXX) - wanna-be-released +----------- + +- Fixes: + +- New Features: + Daniel Black & ykimon + * filter.d/3proxy.conf -- filter added +- Enhancements: + Daniel Black + * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening + and extra failure examples in sample logs + Daniel Black & Georgiy Mernov + * filter.d/exim.conf -- regex hardening and extra failure examples in + sample logs + Yaroslav Halchenko + * fail2ban-regex -- refactored to provide more details (missing and + ignored lines, control over logging, etc) while maintaining look&feel. + ver. 0.8.10 (2013/06/12) - wanna-be-secure ----------- diff --git a/THANKS b/THANKS index ba33b766..47c3e999 100644 --- a/THANKS +++ b/THANKS @@ -18,6 +18,7 @@ Daniel Black David Nutter Eric Gerbier Enrico Labedzki +Georgiy Mernov Guillaume Delvit Hanno 'Rince' Wagner Iain Lea @@ -48,5 +49,6 @@ Tyler Vaclav Misek Vincent Deffontaines Yaroslav Halchenko +ykimon Yehuda Katz zugeschmiert diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 9302fd6a..3790fe62 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -17,12 +17,24 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +Report bugs to https://github.com/fail2ban/fail2ban/issues +""" __author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" import getopt, sys, time, logging, os, locale + +from optparse import OptionParser, Option + +from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from fail2ban.version import version @@ -30,184 +42,171 @@ from fail2ban.client.configparserinc import SafeConfigParserWithIncludes from fail2ban.server.filter import Filter from fail2ban.server.failregex import RegexException +from fail2ban.tests.utils import FormatterWithTraceBack # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.regex") +logSys = logging.getLogger("fail2ban") -class RegexStat: +def shortstr(s, l=53): + """Return shortened string + """ + if len(s) > l: + return s[:l-3] + '...' + return s + +def pprint_list(l, header=None): + if not len(l): + return + if header: + s = "|- %s\n" % header + else: + s = '' + print s + "| " + "\n| ".join(l) + '\n`-' + +def get_opt_parser(): + # use module docstring for help output + p = OptionParser( + usage="%s [OPTIONS] [IGNOREREGEX]\n" % sys.argv[0] + __doc__ + + """ +LOG: + string a string representing a log line + filename path to a log file (/var/log/auth.log) + +REGEX: + string a string representing a 'failregex' + filename path to a filter file (filter.d/sshd.conf) + +IGNOREREGEX: + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +""", + version="%prog " + version) + + p.add_options([ + Option("-e", "--encoding", + help="File encoding. Default: system locale"), + Option("-L", "--maxlines", type=int, default=0, + help="maxlines for multi-line regex"), + Option("-v", "--verbose", action='store_true', + help="Be verbose in output"), + + Option('-l', "--log-level", type="choice", + dest="log_level", + choices=('heavydebug', 'debug', 'info', 'warning', 'error', 'fatal'), + default=None, + help="Log level for the Fail2Ban logger to use"), + Option("--print-all-missed", action='store_true', + help="Either to print all missed lines"), + Option("--print-all-ignored", action='store_true', + help="Either to print all ignored lines"), + Option("-t", "--log-traceback", action='store_true', + help="Enrich log-messages with compressed tracebacks"), + Option("--full-traceback", action='store_true', + help="Either to make the tracebacks full, not compressed (as by default)"), + + ]) + + return p + + +class RegexStat(object): def __init__(self, failregex): - self.__stats = 0 - self.__failregex = failregex - self.__ipList = list() + self._stats = 0 + self._failregex = failregex + self._ipList = list() def __str__(self): return "%s(%r) %d failed: %s" \ - % (self.__class__, self.__failregex, self.__stats, self.__ipList) + % (self.__class__, self._failregex, self._stats, self._ipList) def inc(self): - self.__stats += 1 + self._stats += 1 def getStats(self): - return self.__stats + return self._stats def getFailRegex(self): - return self.__failregex + return self._failregex def appendIP(self, value): - self.__ipList.extend(value) + self._ipList.extend(value) def getIPList(self): - return self.__ipList + return self._ipList -class Fail2banRegex: - test = None +class LineStats(object): + """Just a convenience container for stats + """ + def __init__(self): + self.tested = self.matched = 0 + self.missed_lines = [] + self.ignored_lines = [] + + def __str__(self): + return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self + + @property + def ignored(self): + return len(self.ignored_lines) + + @property + def missed(self): + return self.tested - (self.ignored + self.matched) + + # just for convenient str + def __getitem__(self, key): + return getattr(self, key) + + +class Fail2banRegex(object): CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"} - def __init__(self): - self.__filter = Filter(None) - self.__ignoreregex = list() - self.__failregex = list() - self.__verbose = False - self.__maxlines_set = False # so we allow to override maxlines in cmdline - self.encoding = locale.getpreferredencoding() - # Setup logging - logging.getLogger("fail2ban").handlers = [] - self.__hdlr = logging.StreamHandler(Fail2banRegex.test) - # set a format which is simpler for console use - formatter = logging.Formatter("%(message)s") - # tell the handler to use this format - self.__hdlr.setFormatter(formatter) - self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN - logging.getLogger("fail2ban").addHandler(self.__hdlr) - logging.getLogger("fail2ban").setLevel(logging.ERROR) + def __init__(self, opts): + self._verbose = opts.verbose + self._print_all_missed = opts.print_all_missed + self._print_all_ignored = opts.print_all_ignored + self._maxlines_set = False # so we allow to override maxlines in cmdline + if opts.encoding: + self.encoding = opts.encoding + else: + self.encoding = locale.getpreferredencoding() - #@staticmethod - def dispVersion(): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - dispVersion = staticmethod(dispVersion) + self._filter = Filter(None) + self._ignoreregex = list() + self._failregex = list() + self._line_stats = LineStats() + + if opts.maxlines: + self.setMaxLines(opts.maxlines) - #@staticmethod - def dispUsage(): - print "Usage: "+sys.argv[0]+" [OPTIONS] [IGNOREREGEX]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "This tools can test regular expressions for \"fail2ban\"." - print - print "Options:" - print " -e ENCODING, --encoding=ENCODING" - print " set the file encoding. default:system locale" - print " -h, --help display this help message" - print " -V, --version print the version" - print " -v, --verbose verbose output" - print " -l INT, --maxlines=INT set maxlines for multi-line regex default: 1" - print - print "Log:" - print " string a string representing a log line" - print " filename path to a log file (/var/log/auth.log)" - print - print "Regex:" - print " string a string representing a 'failregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "IgnoreRegex:" - print " string a string representing an 'ignoreregex'" - print " filename path to a filter file (filter.d/sshd.conf)" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" - dispUsage = staticmethod(dispUsage) def setMaxLines(self, v): - if not self.__maxlines_set: - self.__filter.setMaxLines(int(v)) - self.__maxlines_set = True + if not self._maxlines_set: + self._filter.setMaxLines(int(v)) + self._maxlines_set = True - def getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) - elif opt[0] in ["-v", "--verbose"]: - self.__verbose = True - elif opt[0] in ["-e", "--encoding"]: - self.encoding = opt[1] - elif opt[0] in ["-l", "--maxlines"]: - try: - self.setMaxLines(opt[1]) - except ValueError: - print "Invlaid value for maxlines: %s" % ( - opt[1]) - fail2banRegex.dispUsage() - sys.exit(-1) - #@staticmethod - def logIsFile(value): - return os.path.isfile(value) - logIsFile = staticmethod(logIsFile) - - def readIgnoreRegex(self, value): + def readRegex(self, value, regextype): + assert(regextype in ('fail', 'ignore')) + regex = regextype + 'regex' if os.path.isfile(value): reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) try: reader.read(value) - print "Use ignoreregex file : " + value - self.__ignoreregex = [RegexStat(m) - for m in reader.get("Definition", "ignoreregex").split('\n')] + print "Use %11s file : %s" % (regex, value) + # TODO: reuse functionality in client + regex_values = [RegexStat(m) + for m in reader.get("Definition", regex).split('\n')] except NoSectionError: - print "No [Definition] section in " + value - print + print "No [Definition] section in %s" % value return False except NoOptionError: - print "No failregex option in " + value - print + print "No %s option in %s" % (regex, value) return False except MissingSectionHeaderError: - print "No section headers in " + value - print - return False - else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use ignoreregex line : " + stripReg - self.__ignoreregex = [RegexStat(value)] - return True - - def readRegex(self, value): - if os.path.isfile(value): - reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS) - try: - reader.read(value) - print "Use regex file : " + value - self.__failregex = [RegexStat(m) - for m in reader.get("Definition", "failregex").split('\n')] - except NoSectionError: - print "No [Definition] section in " + value - print - return False - except NoOptionError: - print "No failregex option in " + value - print - return False - except MissingSectionHeaderError: - print "No section headers in " + value - print + print "No section headers in %s" % value return False # Read out and set possible value of maxlines @@ -219,49 +218,45 @@ class Fail2banRegex: else: try: self.setMaxLines(maxlines) + print "Use maxlines : %d" % self._filter.getMaxLines() except ValueError: print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ "read from %(value)s" % locals() return False else: - if len(value) > 53: - stripReg = value[0:50] + "..." - else: - stripReg = value - print "Use regex line : " + stripReg - self.__failregex = [RegexStat(value)] + print "Use %11s line : %s" % (regex, shortstr(value)) + regex_values = [RegexStat(value)] - print "Use maxlines : %d" % self.__filter.getMaxLines() + setattr(self, "_" + regex, regex_values) return True def testIgnoreRegex(self, line): found = False - for regex in self.__ignoreregex: - logging.getLogger("fail2ban").setLevel(self.__logging_level) + for regex in self._ignoreregex: try: - self.__filter.addIgnoreRegex(regex.getFailRegex()) + self._filter.addIgnoreRegex(regex.getFailRegex()) try: - ret = self.__filter.ignoreLine(line) + ret = self._filter.ignoreLine(line) if ret: + found = True regex.inc() except RegexException, e: print e return False finally: - self.__filter.delIgnoreRegex(0) - logging.getLogger("fail2ban").setLevel(self.__logging_level) + self._filter.delIgnoreRegex(0) + return found def testRegex(self, line): found = False - for regex in self.__ignoreregex: - self.__filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self.__failregex: - logging.getLogger("fail2ban").setLevel(logging.DEBUG) + for regex in self._ignoreregex: + self._filter.addIgnoreRegex(regex.getFailRegex()) + for regex in self._failregex: try: - self.__filter.addFailRegex(regex.getFailRegex()) + self._filter.addFailRegex(regex.getFailRegex()) try: - ret = self.__filter.processLine(line) - if not len(ret) == 0: + ret = self._filter.processLine(line) + if len(ret): if found == True: ret[0].append(True) else: @@ -276,21 +271,51 @@ class Fail2banRegex: print "Sorry, but no found in regex" return False finally: - self.__filter.delFailRegex(0) + self._filter.delFailRegex(0) try: - del self.__filter._Filter__lineBuffer[-1] + del self._filter._Filter__lineBuffer[-1] except IndexError: pass - logging.getLogger("fail2ban").setLevel(logging.CRITICAL) - self.__filter.processLine(line) - for regex in self.__ignoreregex: - self.__filter.delIgnoreRegex(0) + self._filter.processLine(line) + for regex in self._ignoreregex: + self._filter.delIgnoreRegex(0) + return found + + + def process(self, test_lines): + + for line in test_lines: + if line.startswith('# ') or not line.strip(): + # skip comment and empty lines + continue + is_ignored = fail2banRegex.testIgnoreRegex(line) + if is_ignored: + self._line_stats.ignored_lines.append(line) + + if fail2banRegex.testRegex(line): + assert(not is_ignored) + self._line_stats.matched += 1 + else: + if not is_ignored: + self._line_stats.missed_lines.append(line) + self._line_stats.tested += 1 + + def printLines(self, ltype): + lstats = self._line_stats + assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) + l = lstats[ltype + '_lines'] + if len(l): + header = "%s line(s):" % (ltype.capitalize(),) + if len(l) < 20 or getattr(self, '_print_all_' + ltype): + pprint_list([x.rstrip() for x in l], header) + else: + print "%s: too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, len(l)) def printStats(self): print print "Results" print "=======" - print def print_failregexes(title, failregexes): # Print title @@ -298,112 +323,115 @@ class Fail2banRegex: for cnt, failregex in enumerate(failregexes): match = failregex.getStats() total += match - if (match or self.__verbose): - out.append("| %d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - print "%s: %d total" % (title, total) - if len(out): - print "|- #) [# of hits] regular expression" - print '\n'.join(out) - print '`-' - print - return total + if (match or self._verbose): + out.append("%2d) [%d] %s" % (cnt+1, match, failregex.getFailRegex())) - # Print title - total = print_failregexes("Failregex", self.__failregex) - _ = print_failregexes("Ignoreregex", self.__ignoreregex) - - print "Summary" - print "=======" - print - - if total == 0: - print "Sorry, no match" - print - print "Look at the above section 'Running tests' which could contain important" - print "information." - return False - else: - # Print stats - print "Addresses found:" - for cnt, failregex in enumerate(self.__failregex): - if self.__verbose or len(failregex.getIPList()): - print "[%d]" % (cnt+1) + if self._verbose and len(failregex.getIPList()): for ip in failregex.getIPList(): timeTuple = time.localtime(ip[1]) timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - print " %s (%s)%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "") - print + out.append(" %s %s%s" % ( + ip[0], timeString, ip[2] and " (already matched)" or "")) - print "Date template hits:" - for template in self.__filter.dateDetector.getTemplates(): - if self.__verbose or template.getHits(): - print `template.getHits()` + " hit(s): " + template.getName() - print + print "\n%s: %d total" % (title, total) + pprint_list(out, " #) [# of hits] regular expression") + return total - print "Success, the total number of match is " + str(total) - print - print "However, look at the above section 'Running tests' which could contain important" - print "information." - return True + # Print title + total = print_failregexes("Failregex", self._failregex) + _ = print_failregexes("Ignoreregex", self._ignoreregex) + + + print "\nDate template hits:" + out = [] + for template in self._filter.dateDetector.getTemplates(): + if self._verbose or template.getHits(): + out.append("[%d] %s" % (template.getHits(), template.getName())) + pprint_list(out, "[# of hits] date format") + + print "\nLines: %s" % self._line_stats + + self.printLines('ignored') + self.printLines('missed') + + return True if __name__ == "__main__": - fail2banRegex = Fail2banRegex() - # Reads the command line options. - try: - cmdOpts = 'hVcvl:e:' - cmdLongOpts = ['help', 'version', 'verbose', 'maxlines=', 'encoding='] - optList, args = getopt.getopt(sys.argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - fail2banRegex.dispUsage() - sys.exit(-1) - # Process command line - fail2banRegex.getCmdLineOptions(optList) + + parser = get_opt_parser() + (opts, args) = parser.parse_args() + + fail2banRegex = Fail2banRegex(opts) # We need 2 or 3 parameters if not len(args) in (2, 3): - fail2banRegex.dispUsage() + sys.stderr.write("ERROR: provide both and .\n\n") + parser.print_help() sys.exit(-1) + + # TODO: taken from -testcases -- move common functionality somewhere + if opts.log_level is not None: # pragma: no cover + # so we had explicit settings + logSys.setLevel(getattr(logging, opts.log_level.upper())) + else: # pragma: no cover + # suppress the logging but it would leave unittests' progress dots + # ticking, unless like with '-l fatal' which would be silent + # unless error occurs + logSys.setLevel(getattr(logging, 'FATAL')) + + # Add the default logging handler + stdout = logging.StreamHandler(sys.stdout) + + fmt = 'D: %(message)s' + + if opts.log_traceback: + Formatter = FormatterWithTraceBack + fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt else: - print - print "Running tests" - print "=============" - print + Formatter = logging.Formatter - cmd_log, cmd_regex = args[:2] + # Custom log format for the verbose tests runs + if opts.verbose > 1: # pragma: no cover + stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) + else: # pragma: no cover + # just prefix with the space + stdout.setFormatter(Formatter(fmt)) + logSys.addHandler(stdout) - if len(args) == 3: - fail2banRegex.readIgnoreRegex(args[2]) or sys.exit(-1) + print + print "Running tests" + print "=============" + print - fail2banRegex.readRegex(cmd_regex) or sys.exit(-1) + cmd_log, cmd_regex = args[:2] - if fail2banRegex.logIsFile(cmd_log): - try: - hdlr = open(cmd_log, 'rb') - print "Use log file : " + cmd_log - print "Use encoding : " + fail2banRegex.encoding - print - for line in hdlr: - try: - line = line.decode(fail2banRegex.encoding, 'strict') - except UnicodeDecodeError: - if sys.version_info >= (3,): # Python 3 must be decoded - line = line.decode(fail2banRegex.encoding, 'ignore') - fail2banRegex.testIgnoreRegex(line) - fail2banRegex.testRegex(line) - except IOError, e: - print e - print - sys.exit(-1) - else: - if len(sys.argv[1]) > 53: - stripLog = cmd_log[0:50] + "..." - else: - stripLog = cmd_log - print "Use single line: " + stripLog - print - fail2banRegex.testIgnoreRegex(cmd_log) - fail2banRegex.testRegex(cmd_log) + if len(args) == 3: + fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log, 'rb') + print "Use log file : %s" % cmd_log + print "Use encoding : %s" % fail2banRegex.encoding + test_lines = [] + for line in hdlr: + try: + line = line.decode(fail2banRegex.encoding, 'strict') + except UnicodeDecodeError: + if sys.version_info >= (3,): # Python 3 must be decoded + line = line.decode(fail2banRegex.encoding, 'ignore') + test_lines.append(line) + except IOError, e: + print e + sys.exit(-1) + else: + print "Use single line : %s" % shortstr(cmd_log) + test_lines = [ cmd_log ] + print + + fail2banRegex.process(test_lines) + + fail2banRegex.printStats() or sys.exit(-1) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 15eb085a..d65bbb4d 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -48,7 +48,7 @@ def get_opt_parser(): p.add_options([ Option('-l', "--log-level", type="choice", dest="log_level", - choices=('debug', 'info', 'warn', 'error', 'fatal'), + choices=('heavydebug', 'debug', 'info', 'warn', 'error', 'fatal'), default=None, help="Log level for the logger to use during running tests"), Option('-n', "--no-network", action="store_true", @@ -72,7 +72,8 @@ parser = get_opt_parser() logSys = logging.getLogger("fail2ban") # Numerical level of verbosity corresponding to a log "level" -verbosity = {'debug': 3, +verbosity = {'heavydebug': 3, + 'debug': 3, 'info': 2, 'warn': 1, 'error': 1, diff --git a/config/filter.d/3proxy.conf b/config/filter.d/3proxy.conf new file mode 100644 index 00000000..b22b4588 --- /dev/null +++ b/config/filter.d/3proxy.conf @@ -0,0 +1,18 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# +# Requested by ykimon in https://github.com/fail2ban/fail2ban/issues/246 +# + +[Definition] + +# Option: failregex +# Notes.: http://www.3proxy.ru/howtoe.asp#ERRORS indicates that 01-09 are +# all authentication problems (%E field) +# Log format is: "L%d-%m-%Y %H:%M:%S %z %N.%p %E %U %C:%c %R:%r %O %I %h %T" +# Values: TEXT +# +failregex = ^\s[+-]\d{4} \S+ \d{3}0[1-9] \S+ :\d+ [\d.]+:\d+ \d+ \d+ \d+\s + +ignoreregex = diff --git a/config/filter.d/assp.conf b/config/filter.d/assp.conf index b1bfc082..2854d898 100644 --- a/config/filter.d/assp.conf +++ b/config/filter.d/assp.conf @@ -18,11 +18,11 @@ # Examples: Apr-27-13 02:33:09 Blocking 217.194.197.97 - too much AUTH errors (41); # Dec-29-12 17:10:31 [SSL-out] 200.247.87.82 SSL negotiation with client failed: SSL accept attempt failed with unknown errorerror:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol; # Dec-30-12 04:01:47 [SSL-out] 81.82.232.66 max sender authentication errors (5) exceeded -__assp_actions = (dropping|refusing) +__assp_actions = (?:dropping|refusing) -failregex = max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: [a-zA-Z0-9]+;$ - SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ - Blocking - too much AUTH errors \(\d{,3}\);$ +failregex = ^(:? \[SSL-out\])? max sender authentication errors \(\d{,3}\) exceeded -- %(__assp_actions)s connection - after reply: \d{3} \d{1}\.\d{1}.\d{1} Error: authentication failed: \w+;$ + ^(?: \[SSL-out\])? SSL negotiation with client failed: SSL accept attempt failed with unknown error.*:unknown protocol;$ + ^ Blocking - too much AUTH errors \(\d{,3}\);$ # Option: ignoreregex diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index c1b3dcab..a8f65e09 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -14,25 +14,22 @@ before = common.conf [Definition] # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P\S+) +# Notes.: regex to match the password failures messages in the logfile. # Values: TEXT # -failregex = NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Wrong password$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - No matching peer found$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Username/auth name mismatch$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Device does not match ACL$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Peer is not supposed to register$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - ACL error \(permit/deny\)$ - NOTICE%(__pid_re)s [^:]+: Registration from '[^']*' failed for '(:[0-9]+)?' - Not a local domain$ - NOTICE%(__pid_re)s\[[^:]+\] [^:]+: Call from '[^']*' \(:[0-9]+\) to extension '[0-9]+' rejected because extension not found in context 'default'.$ - NOTICE%(__pid_re)s [^:]+: Host failed to authenticate as '[^']*'$ - NOTICE%(__pid_re)s [^:]+: No registration for peer '[^']*' \(from \)$ - NOTICE%(__pid_re)s [^:]+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ - NOTICE%(__pid_re)s [^:]+: Failed to authenticate user [^@]+@\S*$ - SECURITY%(__pid_re)s [^:]+: SecurityEvent="InvalidAccountID",EventTV="[0-9-]+",Severity="[a-zA-Z]+",Service="[a-zA-Z]+",EventVersion="[0-9]+",AccountID="[0-9]+",SessionID="0x[0-9a-f]+",LocalAddress="IPV[46]/(UD|TC)P/[0-9a-fA-F:.]+/[0-9]+",RemoteAddress="IPV[46]/(UD|TC)P//[0-9]+"$ +failregex = ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Wrong password$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - No matching peer found$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Username/auth name mismatch$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Device does not match ACL$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Peer is not supposed to register$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - ACL error \(permit/deny\)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Registration from '[^']*' failed for '(:\d+)?' - Not a local domain$ + ^\[\]\s*NOTICE%(__pid_re)s\[\S+\] \S+: Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed to authenticate as '[^']*'$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: No registration for peer '[^']*' \(from \)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^\[\]\s*NOTICE%(__pid_re)s \S+: Failed to authenticate user [^@]+@\S*$ + ^\[\]\s*SECURITY%(__pid_re)s \S+: SecurityEvent="InvalidAccountID",EventTV="[\d-]+",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="\d+",SessionID="0x[\da-f]+",LocalAddress="IPV[46]/(UD|TC)P/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UD|TC)P//\d+"$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index d7fb6e6d..dd4c35ba 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -1,20 +1,23 @@ -# Fail2Ban configuration file for dovcot +# Fail2Ban configuration file for dovecot # # Author: Martin Waschbuesch -# -# +# Daniel Black (rewrote with begin and end anchors) + +[INCLUDES] + +before = common.conf [Definition] +_daemon = dovecot(-auth)? + # Option: failregex -# Notes.: regex to match the password failures messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Notes.: regex to match the password failures messages in the logfile. +# first regex is essentially a copy of pam-generic.conf # Values: TEXT # -failregex = .*(?:pop3-login|imap-login):.*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*\s+rip=(?P\S*),.* - pam.*dovecot.*(?:authentication failure).*\s+rhost=(?:\s+user=.*)?\s*$ +failregex = ^%(__prefix_line)s(pam_unix(?:\(\S+\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(\s+user=\S*)?\s*$ + ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \((no auth attempts|auth failed, \d+ attempts|tried to use disabled \S+ auth)\):( user=<\S+>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3},( TLS( handshaking)?(: Disconnected)?)?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/exim.conf b/config/filter.d/exim.conf index b846e992..d335c9fc 100644 --- a/config/filter.d/exim.conf +++ b/config/filter.d/exim.conf @@ -13,8 +13,8 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = \[\] .*(?:rejected by local_scan|Unrouteable address) - login authenticator failed for .* \[\]: 535 Incorrect authentication data \(set_id=.*\)\s*$ +failregex = ^ H=\S+ \(\S+\) \[\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$ + ^ login authenticator failed for (\S+ )?\(\S+\) \[\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index f28e2d4b..13080fcc 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -1,8 +1,15 @@ # Fail2Ban configuration file # # Author: Yaroslav Halchenko +# Daniel Black - hardening of regex # -# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + [Definition] @@ -13,10 +20,10 @@ # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = \(\S+\[\]\)[: -]+ USER \S+: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - \(\S+\[\]\)[: -]+ USER \S+ \(Login failed\): .*$ - \(\S+\[\]\)[: -]+ SECURITY VIOLATION: \S+ login attempted\. *$ - \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ +failregex = ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): .*$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^ %(__hostname)s %(__daemon_re)s%(__pid_re)s %(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. diff --git a/config/jail.conf b/config/jail.conf index 414cc101..af982f3c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -519,4 +519,10 @@ action = pf logpath = /var/log/sshd.log maxretry=5 +[3proxy] + +enabled = false +filter = 3proxy +action = iptables-multiport[name=3proxy, port=3128, protocol=tcp] +logpath = /var/log/3proxy.log diff --git a/fail2ban/client/__init__.py b/fail2ban/client/__init__.py index 2b76f4b6..3eae8ee3 100644 --- a/fail2ban/client/__init__.py +++ b/fail2ban/client/__init__.py @@ -23,3 +23,8 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" + +import logging + +# Custom debug level +logging.HEAVYDEBUG = 5 diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index c4c4bc22..d173f015 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -311,6 +311,8 @@ class Filter(JailThread): """Split the time portion from log msg and return findFailures on them """ line = line.rstrip('\r\n') + logSys.log(5, "Working on line %r", line) + timeMatch = self.dateDetector.matchTime(line) if timeMatch: # Lets split into time part and log part of the line @@ -380,6 +382,8 @@ class Filter(JailThread): continue # The failregex matched. date = self.dateDetector.getUnixTime(timeLine) + logSys.log(7, "Date: %r, message: %r", + timeLine, logLine) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" diff --git a/fail2ban/tests/files/logs/3proxy b/fail2ban/tests/files/logs/3proxy new file mode 100644 index 00000000..2967c9bf --- /dev/null +++ b/fail2ban/tests/files/logs/3proxy @@ -0,0 +1,3 @@ +11-06-2013 02:09:40 +0300 PROXY.3128 00004 - 1.2.3.4:28783 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +11-06-2013 02:09:43 +0300 PROXY.3128 00005 ewr 1.2.3.4:28788 0.0.0.0:0 0 0 0 GET http://www.yandex.ua/?ncrnd=2169807731 HTTP/1.1 +13-06-2013 01:39:34 +0300 PROXY.3128 00508 - 1.2.3.4:28938 0.0.0.0:0 0 0 0 diff --git a/testcases/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth similarity index 100% rename from testcases/files/logs/apache-auth rename to fail2ban/tests/files/logs/apache-auth diff --git a/testcases/files/logs/apache-noscript b/fail2ban/tests/files/logs/apache-noscript similarity index 100% rename from testcases/files/logs/apache-noscript rename to fail2ban/tests/files/logs/apache-noscript diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 434acade..f904a8fe 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -3,6 +3,13 @@ @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 @e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.224 user=root +@e040c6d8a3bfa62d358083300119c259cd44dcd0 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 + +Jun 13 16:30:54 platypus dovecot: imap-login: Disconnected (auth failed, 2 attempts): user=, method=PLAIN, rip=49.176.98.87, lip=113.212.99.194, TLS +Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attempts): method=PLAIN, rip=59.167.242.100, lip=113.212.99.194, TLS: Disconnected +Jun 13 20:48:11 platypus dovecot: pop3-login: Disconnected (no auth attempts): rip=121.44.24.254, lip=113.212.99.194, TLS: Disconnected +Jun 13 21:48:06 platypus dovecot: pop3-login: Disconnected: Inactivity (no auth attempts): rip=180.200.180.81, lip=113.212.99.194, TLS +Jun 13 20:20:21 platypus dovecot: imap-login: Disconnected (no auth attempts): rip=180.189.168.166, lip=113.212.99.194, TLS handshaking: Disconnected + diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index dd507379..d86b6455 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -1,2 +1,6 @@ # From IRC 2013-01-04 2013-01-04 17:03:46 login authenticator failed for rrcs-24-106-174-74.se.biz.rr.com ([192.168.2.33]) [24.106.174.74]: 535 Incorrect authentication data (set_id=brian) +# From IRC 2013-06-13 XATRIX (Georgiy Mernov) +2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s) +2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data +2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for : Unrouteable address diff --git a/fail2ban/tests/files/logs/proftpd b/fail2ban/tests/files/logs/proftpd index def8a83e..aaaf0295 100644 --- a/fail2ban/tests/files/logs/proftpd +++ b/fail2ban/tests/files/logs/proftpd @@ -1,5 +1,9 @@ Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 +Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. +Jun 09 11:15:43 platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (::ffff:101.71.143.238[::ffff:101.71.143.238]): USER god: no such user found from ::ffff:101.71.143.238 [::ffff:101.71.143.238] to ::ffff:123.212.99.194:21 +Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): SECURITY VIOLATION: root login attempted. +Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21