- Added "full line failregex" patch. Thanks to Yaroslav Halchenko. It will be possible to create stronger failregex against log injection

git-svn-id: https://fail2ban.svn.sourceforge.net/svnroot/fail2ban/branches/FAIL2BAN-0_8@621 a942ae1a-1317-0410-a47c-b1dcaea8d605

CHANGELOG

@@ -14,6 +14,9 @@ ver. 0.8.2 (2007/??/??) - stable
   Vincent Deffontaines
 - Fixed timezone bug with epoch date template. Thanks to
   Michael Hanselmann
+- Added "full line failregex" patch. Thanks to Yaroslav
+  Halchenko. It will be possible to create stronger failregex
+  against log injection
 
 ver. 0.8.1 (2007/08/14) - stable
 ----------

MANIFEST

@@ -7,6 +7,7 @@ fail2ban-server
 fail2ban-testcases
 fail2ban-regex
 client/configreader.py
+client/configparserinc.py
 client/jailreader.py
 client/fail2banreader.py
 client/jailsreader.py
@@ -60,6 +61,7 @@ common/__init__.py
 common/version.py
 common/protocol.py
 config/jail.conf
+config/filter.d/common.conf
 config/filter.d/apache-auth.conf
 config/filter.d/apache-badbots.conf
 config/filter.d/apache-noscript.conf

client/configparserinc.py

@@ -0,0 +1,97 @@
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+# Author: Yaroslav Halchenko
+# $Revision$
+
+__author__ = 'Yaroslav Halhenko'
+__revision__ = '$Revision: $'
+__date__ = '$Date:  $'
+__copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko'
+__license__ = 'GPL'
+
+from ConfigParser import SafeConfigParser
+from ConfigParser import NoOptionError, NoSectionError
+
+class SafeConfigParserWithIncludes(SafeConfigParser):
+	"""
+	Class adds functionality to SafeConfigParser to handle included
+	other configuration files (or may be urls, whatever in the future)
+
+	File should have section [includes] and only 2 options implemented
+	are 'files_before' and 'files_after' where files are listed 1 per
+	line.
+
+	Example:
+
+[INCLUDES]
+files_before = 1.conf
+			   3.conf
+
+files_after = 1.conf
+
+	It is a simple implementation, so just basic care is taken about
+	recursion. Includes preserve right order, ie new files are
+	inserted to the list of read configs before original, and their
+	includes correspondingly so the list should follow the leaves of
+	the tree.
+
+	I wasn't sure what would be the right way to implement generic (aka c++
+    template) so we could base at any *configparser class... so I will
+    leave it for the future
+
+	"""
+
+	@staticmethod
+	def getIncludedFiles(filename, sectionName='INCLUDES',
+						 defaults={}, seen=[]):
+		"""
+		Given 1 config filename returns list of included files
+		(recursively) with the original one as well
+		Simple loops are taken care about
+		"""
+		filenames = []
+		#print "Opening file " + filename
+		d = defaults.copy()		# so that we do not poison our defaults
+		parser = SafeConfigParser(defaults = d)
+		parser.read(filename)
+		newFiles = [ ('files_before', []), ('files_after', []) ]
+		if sectionName in parser.sections():
+			for option_name, option_list in newFiles:
+				if option_name in parser.options(sectionName):
+					newFileNames = parser.get(sectionName, option_name)
+					for newFileName in newFileNames.split('\n'):
+						if newFileName in seen: continue
+						option_list += SafeConfigParserWithIncludes.\
+									   getIncludedFiles(newFileName,
+														defaults=defaults,
+														seen=seen + [filename])
+		# combine lists
+		filenames = newFiles[0][1] + [filename] + newFiles[1][1]
+		#print "Includes list for " + filename + " is " + `filenames`
+		return filenames
+
+
+	def read(self, filenames):
+		fileNamesFull = []
+		if not isinstance(filenames, list):
+			filenames = [ filenames ]
+		for filename in filenames:
+			fileNamesFull += SafeConfigParserWithIncludes.\
+							 getIncludedFiles(filename, defaults=self._defaults)
+		#print "Opening config files " + `fileNamesFull`
+		return SafeConfigParser.read(self, fileNamesFull)
+

client/configreader.py

@@ -15,7 +15,7 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 # Author: Cyril Jaquier
-# 
+# Modified by: Yaroslav Halchenko (SafeConfigParserWithIncludes)
 # $Revision$
 
 __author__ = "Cyril Jaquier"
@@ -25,18 +25,20 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 import logging, os
-from ConfigParser import SafeConfigParser
+from configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError
 
 # Gets the instance of the logger.
 logSys = logging.getLogger("fail2ban.client.config")
 
-class ConfigReader(SafeConfigParser):
+class ConfigReader(SafeConfigParserWithIncludes):
 	
 	BASE_DIRECTORY = "/etc/fail2ban/"
 	
 	def __init__(self):
-		SafeConfigParser.__init__(self)
+		SafeConfigParserWithIncludes.__init__(self,
+											  {'configpath' : \
+											   ConfigReader.BASE_DIRECTORY} )
 		self.__opts = None
 	
 	@staticmethod
@@ -54,7 +56,7 @@ class ConfigReader(SafeConfigParser):
 		bConf = basename + ".conf"
 		bLocal = basename + ".local"
 		if os.path.exists(bConf) or os.path.exists(bLocal):
-			SafeConfigParser.read(self, [bConf, bLocal])
+			SafeConfigParserWithIncludes.read(self, [bConf, bLocal])
 			return True
 		else:
 			logSys.error(bConf + " and " + bLocal + " do not exist")

config/filter.d/common.conf

@@ -0,0 +1,41 @@
+# Generic configuration items (to be used as interpolations) in other
+# filters  or actions configurations
+#
+# Author: Yaroslav Halchenko
+#
+# $Revision:  $
+#
+
+[INCLUDES]
+
+# Load customizations if any available
+files_after = %(configpath)s/filter.d/common.local
+
+
+[DEFAULT]
+
+# Daemon definition is to be specialized (if needed) in .conf file
+_daemon = \S*
+
+#
+# Shortcuts for easier comprehension of the failregex
+#
+# PID.
+# EXAMPLES: [123]
+__pid_re = (?:\[\d+\])
+
+# Daemon name (with optional source_file:line or whatever)
+# EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix)
+__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:?
+
+# Combinations of daemon name and PID
+# EXAMPLES: sshd[31607], pop(pam_unix)[4920]
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:)
+
+#
+# Common line prefixes (beginnings) which could be used in filters
+#
+#       [hostname] [vserver tag] daemon_id spaces
+# this can be optional (for instance if we match named native log files)
+__prefix_line = \s*(?:\S+ )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s*
+

config/filter.d/sshd.conf

@@ -5,8 +5,17 @@
 # $Revision$
 #
 
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+files_before = %(configpath)s/filter.d/common.conf
+
+
 [Definition]
 
+_daemon = sshd
+
 # Option:  failregex
 # Notes.:  regex to match the password failures messages in the logfile. The
 #          host must be matched by a group named "host". The tag "<HOST>" can
@@ -14,12 +23,11 @@
 #          (?:::f{4,6}:)?(?P<host>\S+)
 # Values:  TEXT
 #
-failregex = (?:error: PAM: )?Authentication failure for .* from <HOST>\s*$
+failregex = ^%(__prefix_line)s(?:error: PAM: )?Authentication failure for .* from <HOST>\s*$
-            Failed [-/\w]+ for .* from <HOST>(?: port \d*)?(?: ssh\d*)?\s*$
+            ^%(__prefix_line)sFailed [-/\w]+ for .* from <HOST>(?: port \d*)?(?: ssh\d*)?$
-            ROOT LOGIN REFUSED.* FROM <HOST>\s*$
+            ^%(__prefix_line)sROOT LOGIN REFUSED.* FROM <HOST>\s*$
-            [iI](?:llegal|nvalid) user .* from <HOST>\s*$
+            ^%(__prefix_line)s[iI](?:llegal|nvalid) user .* from <HOST>\s*$
-            User .+ from <HOST> not allowed because not listed in AllowUsers\s*$
+            ^%(__prefix_line)sUser \S+ from <HOST> not allowed because not listed in AllowUsers$
-            User .+ from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

fail2ban-regex

@@ -31,7 +31,7 @@ import getopt, sys, time, logging, os
 # fix for bug #343821
 sys.path.insert(1, "/usr/share/fail2ban")
 
-from ConfigParser import SafeConfigParser
+from client.configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
 from common.version import version
 from server.filter import Filter
@@ -66,6 +66,8 @@ class Fail2banRegex:
 	
 	test = None
 
+	CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"}
+
 	def __init__(self):
 		self.__filter = Filter(None)
 		self.__ignoreregex = list()
@@ -135,7 +137,7 @@ class Fail2banRegex:
 
 	def readIgnoreRegex(self, value):
 		if os.path.isfile(value):
-			reader = SafeConfigParser()
+			reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS)
 			try:
 				reader.read(value)
 				print "Use ignoreregex file : " + value
@@ -164,7 +166,7 @@ class Fail2banRegex:
 
 	def readRegex(self, value):
 		if os.path.isfile(value):
-			reader = SafeConfigParser()
+			reader = SafeConfigParserWithIncludes(defaults=self.CONFIG_DEFAULTS)
 			try:
 				reader.read(value)
 				print "Use regex file : " + value

server/filter.py

@@ -383,7 +383,7 @@ class Filter(JailThread):
 			logSys.error("Unable to get failures in " + filename)
 			return False
 		self.__setFilePos()
-		lastLine = None
+		lastTimeLine = None
 		for line in self.__crtHandler:
 			if not self._isActive():
 				# The jail has been stopped
@@ -393,11 +393,18 @@ class Filter(JailThread):
 				line = line.decode('utf-8')
 			except UnicodeDecodeError:
 				pass
-			if not self.dateDetector.matchTime(line):
+			timeMatch = self.dateDetector.matchTime(line)
+			if not timeMatch:
 				# There is no valid time in this line
 				continue
-			lastLine = line
+			# Lets split into time part and log part of the line
-			for element in self.findFailure(line):
+			timeLine = timeMatch.group()
+			# Lets leave the beginning in as well, so if there is no
+			# anchore at the beginning of the time regexp, we don't
+			# at least allow injection. Should be harmless otherwise
+			logLine  = line[:timeMatch.start()] + line[timeMatch.end():]
+			lastTimeLine = timeLine
+			for element in self.findFailure(timeLine, logLine):
 				ip = element[0]
 				unixTime = element[1]
 				if unixTime < MyTime.time()-self.__findTime:
@@ -408,8 +415,8 @@ class Filter(JailThread):
 				logSys.debug("Found "+ip)
 				self.failManager.addFailure(FailTicket(ip, unixTime))
 		self.__lastPos[filename] = self.__getFilePos()
-		if lastLine:
+		if lastTimeLine:
-			self.__lastDate[filename] = self.dateDetector.getUnixTime(lastLine)
+			self.__lastDate[filename] = self.dateDetector.getUnixTime(lastTimeLine)
 		self.__closeLogFile()
 		return True
 
@@ -428,27 +435,28 @@ class Filter(JailThread):
 		return False
 
 	##
-	# Finds the failure in a line.
+	# Finds the failure in a line given split into time and log parts.
 	#
 	# Uses the failregex pattern to find it and timeregex in order
 	# to find the logging time.
 	# @return a dict with IP and timestamp.
 
-	def findFailure(self, line):
+	def findFailure(self, timeLine, logLine):
 		failList = list()
 		# Checks if we must ignore this line.
-		if self.ignoreLine(line):
+		if self.ignoreLine(logLine):
 			# The ignoreregex matched. Return.
 			return failList
 		# Iterates over all the regular expressions.
 		for failRegex in self.__failRegex:
-			failRegex.search(line)
+			failRegex.search(logLine)
 			if failRegex.hasMatched():
 				# The failregex matched.
-				date = self.dateDetector.getUnixTime(line)
+				date = self.dateDetector.getUnixTime(timeLine)
 				if date == None:
-					logSys.debug("Found a match but no valid date/time found "
+					logSys.debug("Found a match for '" + logLine +"' but no "
-								 + "for " + line + ". Please contact the "
+								 + "valid date/time found for '"
+								 + timeLine + "'. Please contact the "
 								 + "author in order to get support for this "
 								 + "format")
 				else: