- Added wildcards support for "logpath"

- Added "set <jail> addlogpath <path>" and "set <jail> dellogpath <path>"
- Adapted pyunit test

git-svn-id: https://fail2ban.svn.sourceforge.net/svnroot/fail2ban/trunk@354 a942ae1a-1317-0410-a47c-b1dcaea8d605

CHANGELOG

@@ -10,6 +10,7 @@ Fail2Ban (version 0.7.3)                           2006/??/??
 ver. 0.7.3 (2006/??/??) - beta
 ----------
 - Added man pages. Thanks to Yaroslav Halchenko
+- Added wildcard support for "logpath"
 
 ver. 0.7.2 (2006/09/10) - beta
 ----------

TODO

@@ -92,10 +92,11 @@ Legend:
 
 * remove most of the command lines options if possible.
 
-- add the possibility to specify wildcard in log files.
+* add the possibility to specify wildcard in log files.
   Example: logfile = /var/log/apache2/access-*.log
   Should we start one thread per file or just one thread per
   serivce?
+  We use just one thread per service
 
 # autodetect date format in log file. Match the most popular
   format and sort them using the hit ratio. Should avoid

client/beautifier.py

@@ -50,10 +50,11 @@ class Beautifier:
 		
 	def beautify(self, response):
 		logSys.debug("Beautify " + `response` + " with " + `self.inputCmd`)
+		inC = self.inputCmd
 		msg = response
-		if self.inputCmd[0:1] == ['status']:
-			if len(self.inputCmd) > 1:
-				msg = "Status for the jail: " + self.inputCmd[1] + "\n"
+		if inC[0:1] == ['status']:
+			if len(inC) > 1:
+				msg = "Status for the jail: " + inC[1] + "\n"
 				msg = msg + "|- " + response[0][0] + "\n"
 				msg = msg + "|  |- " + response[0][1][0][0] + ":\t\t" + `response[0][1][0][1]` + "\n"
 				msg = msg + "|  `- " + response[0][1][1][0] + ":\t\t" + `response[0][1][1][1]` + "\n"
@@ -64,7 +65,7 @@ class Beautifier:
 				msg = "Status\n"
 				msg = msg + "|- " + response[0][0] + ":\t" + `response[0][1]` + "\n"
 				msg = msg + "`- " + response[1][0] + ":\t\t" + response[1][1]
-		elif self.inputCmd[1:2] == ['loglevel']:
+		elif inC[1:2] == ['loglevel']:
 			msg = "Current logging level is "
 			if response == 1:
 				msg = msg + "ERROR"
@@ -76,9 +77,14 @@ class Beautifier:
 				msg = msg + "DEBUG"
 			else:
 				msg = msg + `response`
-		elif self.inputCmd == ["stop"]:
+		elif inC == ["stop"]:
 			if response == None:
 				msg = "Shutdown successful"
+		elif inC[2] == "logpath" or inC[2] == "addlogpath" or inC[2] == "dellogpath":
+			msg = "Current monitored log file(s):\n"
+			for path in response[:-1]:
+				msg = msg + "|- " + path + "\n"
+			msg = msg + "`- " + response[len(response)-1]
 		return msg
 
 	def beautifyError(self, response):

client/jailreader.py

@@ -24,7 +24,8 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import logging, re
+import logging, re, glob
+
 from configreader import ConfigReader
 from filterreader import FilterReader
 from actionreader import ActionReader
@@ -95,7 +96,11 @@ class JailReader(ConfigReader):
 		stream = [["add", self.name]]
 		for opt in self.opts:
 			if opt == "logpath":
-				stream.append(["set", self.name, "logpath", self.opts[opt]])
+				pathList = glob.glob(self.opts[opt])
+				if len(pathList) == 0:
+					logSys.error("No file found for " + self.opts[opt])
+				for path in pathList:
+					stream.append(["set", self.name, "addlogpath", path])
 			elif opt == "maxretry":
 				stream.append(["set", self.name, "maxretry", self.opts[opt]])
 			elif opt == "maxtime":

server/actions.py

@@ -128,7 +128,6 @@ class Actions(JailThread):
 	# @return True if an IP address get banned
 	
 	def checkBan(self):
-		logSys.debug("Check for IP address to ban")
 		ticket = self.jail.getFailTicket()
 		if ticket != False:
 			aInfo = dict()
@@ -148,7 +147,6 @@ class Actions(JailThread):
 	# Unban IP address which are outdated.
 	
 	def checkUnBan(self):
-		logSys.debug("Check for IP address to unban")
 		for ticket in self.banManager.unBanList(time.time()):
 			aInfo = dict()
 			aInfo["ip"] = ticket.getIP()

server/filter.py

@@ -57,9 +57,10 @@ class Filter(JailThread):
 		## The failures manager.
 		self.failManager = FailManager()
 		## The log file handler.
-		self.fileHandler = None
+		self.crtHandler = None
+		self.crtFilename = None
 		## The log file path.
-		self.logPath = ''
+		self.logPath = []
 		## The regular expression matching the failure.
 		self.failRegex = ''
 		self.failRegexObj = None
@@ -68,26 +69,49 @@ class Filter(JailThread):
 		## The ignore IP list.
 		self.ignoreIpList = []
 		## The time of the last modification of the file.
-		self.lastModTime = 0
+		self.lastModTime = dict()
 		## The last position of the file.
-		self.lastPos = 0
+		self.lastPos = dict()
 		## The last date in tht log file.
-		self.lastDate = 0
-		## The file statistics.
-		self.logStats = None
+		self.lastDate = dict()
+		self.file404Cnt = dict()
+		
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
-		self.fileNotFoundCnt = 0
 		logSys.info("Created Filter")
 
 	##
-	# Set the log file path
+	# Add a log file path
 	#
-	# @param value log file path
+	# @param path log file path
 
-	def setLogPath(self, value):
-		self.logPath = value
-		logSys.info("Set logfile = %s" % value)
+	def addLogPath(self, path):
+		try:
+			self.logPath.index(path)
+			logSys.error(path + " already exists")
+		except ValueError:
+			self.logPath.append(path)
+			# Initialize default values
+			self.lastDate[path] = 0
+			self.lastModTime[path] = 0
+			self.lastPos[path] = 0
+			self.file404Cnt[path] = 0
+			logSys.info("Added logfile = %s" % path)
+	
+	##
+	# Delete a log path
+	#
+	# @param path the log file to delete
+	
+	def delLogPath(self, path):
+		try:
+			self.logPath.remove(path)
+			del self.lastDate[path]
+			del self.lastModTime[path]
+			del self.lastPos[path]
+			logSys.info("Removed logfile = %s" % path)
+		except ValueError:
+			logSys.error(path + " is not monitored")
 
 	##
 	# Get the log file path
@@ -213,11 +237,16 @@ class Filter(JailThread):
 
 	def run(self):
 		self.setActive(True)
+		prevModified = False
 		while self.isActive():
 			if not self.isIdle:
-				if self.isModified():
-					self.getFailures()
+				if prevModified:
 					self.dateDetector.sortTemplate()
+				prevModified = False
+				for file in self.logPath:
+					if self.isModified(file):
+						self.getFailures(file)
+						prevModified = True
 				try:
 					ticket = self.failManager.toBan()
 					self.jail.putFailTicket(ticket)
@@ -267,19 +296,21 @@ class Filter(JailThread):
 	##
 	# Open the log file.
 	
-	def openLogFile(self):
+	def openLogFile(self, filename):
 		""" Opens the log file specified on init.
 		"""
 		try:
-			self.fileHandler = open(self.logPath)
+			self.crtFilename = filename
+			self.crtHandler = open(filename)
 		except OSError:
-			logSys.error("Unable to open "+self.logPath)
+			logSys.error("Unable to open " + filename)
 	
 	##
 	# Close the log file.
 	
 	def closeLogFile(self):
-		self.fileHandler.close()
+		self.crtFilename = None
+		self.crtHandler.close()
 	
 	##
 	# Checks if the log file has been modified.
@@ -287,23 +318,23 @@ class Filter(JailThread):
 	# Checks if the log file has been modified using os.stat().
 	# @return True if log file has been modified
 	
-	def isModified(self):
+	def isModified(self, filename):
 		try:
-			self.logStats = os.stat(self.logPath)
-			self.fileNotFoundCnt = 0
-			if self.lastModTime == self.logStats.st_mtime:
+			logStats = os.stat(filename)
+			self.file404Cnt[filename] = 0
+			if self.lastModTime[filename] == logStats.st_mtime:
 				return False
 			else:
-				logSys.debug(self.logPath + " has been modified")
-				self.lastModTime = self.logStats.st_mtime
+				logSys.debug(filename + " has been modified")
+				self.lastModTime[filename] = logStats.st_mtime
 				return True
 		except OSError:
-			logSys.error("Unable to get stat on " + self.logPath)
-			self.fileNotFoundCnt = self.fileNotFoundCnt + 1
-			if self.fileNotFoundCnt > 2:
+			logSys.error("Unable to get stat on " + filename)
+			self.file404Cnt[filename] = self.file404Cnt[filename] + 1
+			if self.file404Cnt[filename] > 2:
 				logSys.warn("Too much read error. Set the jail idle")
 				self.jail.setIdle(True)
-				self.fileNotFoundCnt = 0
+				self.file404Cnt[filename] = 0
 			return False
 
 	##
@@ -314,22 +345,23 @@ class Filter(JailThread):
 	# timestamp in order to detect this.
 	
 	def setFilePos(self):
-		line = self.fileHandler.readline()
-		if self.lastDate < self.dateDetector.getTime(line):
-			logSys.debug("Date " + `self.lastDate` + " is " + "smaller than " +
-							`self.dateDetector.getTime(line)`)
-			logSys.debug("Log rotation detected for " + self.logPath)
-			self.lastPos = 0
-		
-		logSys.debug("Setting file position to " + `self.lastPos` + " for " +
-						self.logPath)
-		self.fileHandler.seek(self.lastPos)
+		line = self.crtHandler.readline()
+		lastDate = self.lastDate[self.crtFilename]
+		lineDate = self.dateDetector.getUnixTime(line)
+		if lastDate < lineDate:
+			logSys.debug("Date " + `lastDate` + " is smaller than " + `lineDate`)
+			logSys.debug("Log rotation detected for " + self.crtFilename)
+			self.lastPos[self.crtFilename] = 0
+		lastPos = self.lastPos[self.crtFilename]
+		logSys.debug("Setting file position to " + `lastPos` + " for " +
+					 self.crtFilename)
+		self.crtHandler.seek(lastPos)
 
 	##
 	# Get the file position.
 	
 	def getFilePos(self):
-		return self.fileHandler.tell()
+		return self.crtHandler.tell()
 
 	##
 	# Gets all the failure in the log file.
@@ -338,18 +370,20 @@ class Filter(JailThread):
 	# time.time()-self.findTime. When a failure is detected, a FailTicket
 	# is created and is added to the FailManager.
 	
-	def getFailures(self):
+	def getFailures(self, filename):
 		ipList = dict()
-		logSys.debug(self.logPath)
-		self.openLogFile()
+		logSys.debug(filename)
+		self.openLogFile(filename)
 		self.setFilePos()
 		lastLine = None
-		for line in self.fileHandler:
+		for line in self.crtHandler:
 			try:
 				# Try to convert UTF-8 string to Latin-1
 				line = line.decode('utf-8').encode('latin-1')
 			except UnicodeDecodeError:
 				pass
+			except UnicodeEncodeError:
+				logSys.warn("Mmhh... Are you sure you watch the correct file?")
 			if not self.dateDetector.matchTime(line):
 				# There is no valid time in this line
 				continue
@@ -364,9 +398,9 @@ class Filter(JailThread):
 					continue
 				logSys.debug("Found "+ip)
 				self.failManager.addFailure(FailTicket(ip, unixTime))
-		self.lastPos = self.getFilePos()
+		self.lastPos[filename] = self.getFilePos()
 		if lastLine:
-			self.lastDate = self.dateDetector.getTime(lastLine)
+			self.lastDate[filename] = self.dateDetector.getTime(lastLine)
 		self.closeLogFile()
 
 	##

server/server.py

@@ -123,9 +123,13 @@ class Server:
 			raise ServerUnknownJail(name)
 	
 	# Filter
-	def setLogPath(self, name, file):
+	def addLogPath(self, name, file):
 		if self.jails.has_key(name):
-			self.jails[name].getFilter().setLogPath(file)
+			self.jails[name].getFilter().addLogPath(file)
+	
+	def delLogPath(self, name, file):
+		if self.jails.has_key(name):
+			self.jails[name].getFilter().delLogPath(file)
 	
 	def getLogPath(self, name):
 		return self.getFilter(name).getLogPath()

server/transmitter.py

@@ -129,9 +129,14 @@ class Transmitter:
 				self.server.setIdleJail(name, False)
 			return self.server.getIdleJail(name)
 		# Filter
-		elif action[1] == "logpath":
+		elif action[1] == "addlogpath":
+			value = action[2:]
+			for path in value:
+				self.server.addLogPath(name, path)
+			return self.server.getLogPath(name)
+		elif action[1] == "dellogpath":
 			value = action[2]
-			self.server.setLogPath(name, value)
+			self.server.delLogPath(name, value)
 			return self.server.getLogPath(name)
 		elif action[1] == "timeregex":
 			value = action[2]

testcases/filtertestcase.py

@@ -52,20 +52,21 @@ class IgnoreIP(unittest.TestCase):
 
 class LogFile(unittest.TestCase):
 
+	filename = "testcases/files/testcase01.log"
+
 	def setUp(self):
 		"""Call before every test case."""
 		self.filter = Filter(None)
-		self.filter.setLogPath("testcases/files/testcase01.log")
+		self.filter.addLogPath(LogFile.filename)
 
 	def tearDown(self):
 		"""Call after every test case."""
 		
 	def testOpen(self):
-		self.filter.openLogFile()
+		self.filter.openLogFile(LogFile.filename)
 	
 	def testIsModified(self):
-		self.filter.openLogFile()
-		self.assertTrue(self.filter.isModified())
+		self.assertTrue(self.filter.isModified(LogFile.filename))
 
 
 class GetFailures(unittest.TestCase):
@@ -73,7 +74,7 @@ class GetFailures(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		self.filter = Filter(None)
-		self.filter.setLogPath("testcases/files/testcase01.log")
+		self.filter.addLogPath("testcases/files/testcase01.log")
 		self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 		self.filter.setTimePattern("%b %d %H:%M:%S")
 		self.filter.setFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) (?:::f{4,6}:)?(?P<host>\S*)")