Merge pull request #519 from grooverdan/db-migration

addLog to single SQL statement

MANIFEST

@@ -77,6 +77,7 @@ fail2ban/tests/config/apache-auth/digest_anon/.htaccess
 fail2ban/tests/config/apache-auth/digest_anon/.htpasswd
 fail2ban/tests/config/apache-auth/README
 fail2ban/tests/config/apache-auth/noentry/.htaccess
+fail2ban/tests/files/database_v1.db
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log

fail2ban/server/database.py

@@ -23,6 +23,7 @@ __license__ = "GPL"
 
 import logging
 import sys
+import shutil, time
 import sqlite3
 import json
 import locale
@@ -55,7 +56,40 @@ def commitandrollback(f):
 	return wrapper
 
 class Fail2BanDb(object):
-	__version__ = 1
+	__version__ = 2
+	# Note all _TABLE_* strings must end in ';' for py26 compatibility
+	_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
+	_TABLE_jails = "CREATE TABLE jails(" \
+			"name TEXT NOT NULL UNIQUE, " \
+			"enabled INTEGER NOT NULL DEFAULT 1" \
+			");" \
+			"CREATE INDEX jails_name ON jails(name);"
+	_TABLE_logs = "CREATE TABLE logs(" \
+			"jail TEXT NOT NULL, " \
+			"path TEXT, " \
+			"firstlinemd5 TEXT, " \
+			"lastfilepos INTEGER DEFAULT 0, " \
+			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, " \
+			"UNIQUE(jail, path)," \
+			"UNIQUE(jail, path, firstlinemd5)" \
+			");" \
+			"CREATE INDEX logs_path ON logs(path);" \
+			"CREATE INDEX logs_jail_path ON logs(jail, path);"
+			#TODO: systemd journal features \
+			#"journalmatch TEXT, " \
+			#"journlcursor TEXT, " \
+			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available \
+	_TABLE_bans = "CREATE TABLE bans(" \
+			"jail TEXT NOT NULL, " \
+			"ip TEXT, " \
+			"timeofban INTEGER NOT NULL, " \
+			"data JSON, " \
+			"FOREIGN KEY(jail) REFERENCES jails(name) " \
+			");" \
+			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban);" \
+			"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
+			"CREATE INDEX bans_ip ON bans(ip);" \
+
 	def __init__(self, filename, purgeAge=24*60*60):
 		try:
 			self._db = sqlite3.connect(
@@ -85,8 +119,15 @@ class Fail2BanDb(object):
 		else:
 			version = cur.fetchone()[0]
 			if version < Fail2BanDb.__version__:
+				newversion = self.updateDb(version)
+				if newversion == Fail2BanDb.__version__:
 					logSys.warning( "Database updated from '%i' to '%i'",
-					version, self.updateDb(version))
+						version, newversion)
+				else:
+					logSys.error( "Database update failed to acheive version '%i'"
+						": updated from '%i' to '%i'",
+						Fail2BanDb.__version__, version, newversion)
+					raise Exception('Failed to fully update')
 		finally:
 			cur.close()
 
@@ -102,53 +143,37 @@ class Fail2BanDb(object):
 	@commitandrollback
 	def createDb(self, cur):
 		# Version info
-		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
+		cur.executescript(Fail2BanDb._TABLE_fail2banDb)
 		cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
 			(Fail2BanDb.__version__, ))
-
 		# Jails
-		cur.execute("CREATE TABLE jails("
+		cur.executescript(Fail2BanDb._TABLE_jails)
-			"name TEXT NOT NULL UNIQUE, "
-			"enabled INTEGER NOT NULL DEFAULT 1"
-			")")
-		cur.execute("CREATE INDEX jails_name ON jails(name)")
-
 		# Logs
-		cur.execute("CREATE TABLE logs("
+		cur.executescript(Fail2BanDb._TABLE_logs)
-			"jail TEXT NOT NULL, "
-			"path TEXT, "
-			"firstlinemd5 TEXT, "
-			#TODO: systemd journal features
-			#"journalmatch TEXT, "
-			#"journlcursor TEXT, "
-			"lastfilepos INTEGER DEFAULT 0, "
-			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
-			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, "
-			"UNIQUE(jail, path)"
-			")")
-		cur.execute("CREATE INDEX logs_path ON logs(path)")
-		cur.execute("CREATE INDEX logs_jail_path ON logs(jail, path)")
-
 		# Bans
-		cur.execute("CREATE TABLE bans("
+		cur.executescript(Fail2BanDb._TABLE_bans)
-			"jail TEXT NOT NULL, "
-			"ip TEXT, "
-			"timeofban INTEGER NOT NULL, "
-			"data JSON, "
-			"FOREIGN KEY(jail) REFERENCES jails(name) "
-			")")
-		cur.execute(
-			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban)")
-		cur.execute("CREATE INDEX bans_jail_ip ON bans(jail, ip)")
-		cur.execute("CREATE INDEX bans_ip ON bans(ip)")
 
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
 	@commitandrollback
 	def updateDb(self, cur, version):
+		self.dbBackupFilename = self._dbFilename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+		shutil.copyfile(self._dbFilename, self.dbBackupFilename)
+		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
-			"Only single version of database exists...how did you get here??")
+						"Attempt to travel to future version of database ...how did you get here??")
+
+		if version < 2:
+			cur.executescript("BEGIN TRANSACTION;"
+						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
+						"DROP TABLE logs;"
+						"%s;"
+						"INSERT INTO logs SELECT * from logs_temp;"
+						"DROP TABLE logs_temp;"
+						"UPDATE fail2banDb SET version = 2;"
+						"COMMIT;" % Fail2BanDb._TABLE_logs)
+
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
@@ -187,14 +212,14 @@ class Fail2BanDb(object):
 		try:
 			firstLineMD5, lastLinePos = cur.fetchone()
 		except TypeError:
+			firstLineMD5 = False
+
 		cur.execute(
-				"INSERT INTO logs(jail, path, firstlinemd5, lastfilepos) "
+				"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
 					"VALUES(?, ?, ?, ?)",
 				(jail.getName(), container.getFileName(),
 					container.getHash(), container.getPos()))
-		else:
 		if container.getHash() != firstLineMD5:
-				self._updateLog(cur, jail, container)
 			lastLinePos = None
 		return lastLinePos
 

fail2ban/server/ticket.py

@@ -46,9 +46,20 @@ class Ticket:
 		self.__matches = matches or []
 
 	def __str__(self):
-		return "%s: ip=%s time=%s #attempts=%d" % \
+		return "%s: ip=%s time=%s #attempts=%d matches=%r" % \
-			   (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt)
+			   (self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches)
 
+	def __repr__(self):
+		return str(self)
+
+	def __eq__(self, other):
+		try:
+			return self.__ip == other.__ip and \
+				round(self.__time,2) == round(other.__time,2) and \
+				self.__attempt == other.__attempt and \
+				self.__matches == other.__matches
+		except AttributeError:
+			return False
 
 	def setIP(self, value):
 		if isinstance(value, basestring):

fail2ban/tests/databasetestcase.py

@@ -26,6 +26,7 @@ import os
 import unittest
 import tempfile
 import sqlite3
+import shutil
 
 from fail2ban.server.database import Fail2BanDb
 from fail2ban.server.filter import FileContainer
@@ -45,7 +46,7 @@ class DatabaseTest(unittest.TestCase):
 		# Cleanup
 		os.remove(self.dbFilename)
 
-	def getFilename(self):
+	def testGetFilename(self):
 		self.assertEqual(self.dbFilename, self.db.getFilename())
 
 	def testCreateInvalidPath(self):
@@ -64,8 +65,16 @@ class DatabaseTest(unittest.TestCase):
 			"Jail not retained in Db after disconnect reconnect.")
 
 	def testUpdateDb(self):
-		# TODO: Currently only single version exists
+		shutil.copyfile('fail2ban/tests/files/database_v1.db', self.dbFilename)
-		pass
+		self.db = Fail2BanDb(self.dbFilename)
+		self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets']))
+		self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log']))
+		ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"])
+		self.assertEqual(self.db.getBans()[0], ticket)
+
+		self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__)
+		self.assertRaises(NotImplementedError, self.db.updateDb, Fail2BanDb.__version__ + 1)
+		os.remove(self.db.dbBackupFilename)
 
 	def testAddJail(self):
 		self.jail = DummyJail()
@@ -83,6 +92,7 @@ class DatabaseTest(unittest.TestCase):
 		self.db.addLog(self.jail, self.fileContainer)
 
 		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+		os.remove(filename)
 
 	def testUpdateLog(self):
 		self.testAddLog() # Add log file
@@ -121,6 +131,7 @@ class DatabaseTest(unittest.TestCase):
 		# last position in file
 		self.assertEqual(
 			self.db.addLog(self.jail, self.fileContainer), None)
+		os.remove(filename)
 
 	def testAddBan(self):
 		self.testAddJail()
@@ -131,6 +142,13 @@ class DatabaseTest(unittest.TestCase):
 		self.assertTrue(
 			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
 
+	def testGetBansWithTime(self):
+		self.testAddJail()
+		ticket = FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"])
+		self.db.addBan(self.jail, ticket)
+		self.assertEquals(len(self.db.getBans(jail=self.jail,bantime=50)), 1)
+		self.assertEquals(len(self.db.getBans(jail=self.jail,bantime=20)), 0)
+
 	def testGetBansMerged(self):
 		self.testAddJail()
 

fail2ban/tests/failmanagertestcase.py

@@ -93,16 +93,21 @@ class AddFailure(unittest.TestCase):
 		# finish with rudimentary tests of the ticket
 		# verify consistent str
 		ticket_str = str(ticket)
+		ticket_repr = repr(ticket)
 		self.assertEqual(
 			ticket_str,
-			'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5')
+			'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5 matches=[]')
+		self.assertEqual(
+			ticket_repr,
+			'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5 matches=[]')
+		self.assertFalse(ticket == False)
 		# and some get/set-ers otherwise not tested
 		ticket.setTime(1000002000.0)
 		self.assertEqual(ticket.getTime(), 1000002000.0)
 		# and str() adjusted correspondingly
 		self.assertEqual(
 			str(ticket),
-			'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5')
+			'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5 matches=[]')
 	
 	def testbanNOK(self):
 		self.__failManager.setMaxRetry(10)

fail2ban/tests/servertestcase.py

@@ -670,14 +670,14 @@ class TransmitterLogging(TransmitterBase):
 			self.server.setLogLevel(2)
 			self.assertEqual(self.transm.proceed(["set", "logtarget", fn]), (0, fn))
 			l = logging.getLogger('fail2ban.server.server').parent.parent
-			l.warn("Before file moved")
+			l.warning("Before file moved")
 			try:
 				f2, fn2 = tempfile.mkstemp("fail2ban.log")
 				os.close(f2)
 				os.rename(fn, fn2)
-				l.warn("After file moved")
+				l.warning("After file moved")
 				self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "rolled over"))
-				l.warn("After flushlogs")
+				l.warning("After flushlogs")
 				with open(fn2,'r') as f:
 					line1 = f.next()
 					if line1.find('Changed logging target to') >= 0: