ENH: change addLog to use single SQL statement

ENH: separate out the database creation defination to make updates
easier

TST: add test framework for updates

MANIFEST

@@ -77,6 +77,7 @@ fail2ban/tests/config/apache-auth/digest_anon/.htaccess
 fail2ban/tests/config/apache-auth/digest_anon/.htpasswd
 fail2ban/tests/config/apache-auth/README
 fail2ban/tests/config/apache-auth/noentry/.htaccess
+fail2ban/tests/files/database_v1.db
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log

fail2ban/server/database.py

@@ -23,6 +23,7 @@ __license__ = "GPL"
 
 import logging
 import sys
+import shutil, time
 import sqlite3
 import json
 import locale
@@ -55,7 +56,39 @@ def commitandrollback(f):
 	return wrapper
 
 class Fail2BanDb(object):
-	__version__ = 1
+	__version__ = 2
+	_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER)"
+	_TABLE_jails = "CREATE TABLE jails(" \
+			"name TEXT NOT NULL UNIQUE, " \
+			"enabled INTEGER NOT NULL DEFAULT 1" \
+			");" \
+			"CREATE INDEX jails_name ON jails(name);"
+	_TABLE_logs = "CREATE TABLE logs(" \
+			"jail TEXT NOT NULL, " \
+			"path TEXT, " \
+			"firstlinemd5 TEXT, " \
+			"lastfilepos INTEGER DEFAULT 0, " \
+			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, " \
+			"UNIQUE(jail, path)," \
+			"UNIQUE(jail, path, firstlinemd5)" \
+			");" \
+			"CREATE INDEX logs_path ON logs(path);" \
+			"CREATE INDEX logs_jail_path ON logs(jail, path);"
+			#TODO: systemd journal features \
+			#"journalmatch TEXT, " \
+			#"journlcursor TEXT, " \
+			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available \
+	_TABLE_bans = "CREATE TABLE bans(" \
+			"jail TEXT NOT NULL, " \
+			"ip TEXT, " \
+			"timeofban INTEGER NOT NULL, " \
+			"data JSON, " \
+			"FOREIGN KEY(jail) REFERENCES jails(name) " \
+			");" \
+			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban);" \
+			"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
+			"CREATE INDEX bans_ip ON bans(ip);" \
+
 	def __init__(self, filename, purgeAge=24*60*60):
 		try:
 			self._db = sqlite3.connect(
@@ -85,8 +118,15 @@ class Fail2BanDb(object):
 		else:
 			version = cur.fetchone()[0]
 			if version < Fail2BanDb.__version__:
+				newversion = self.updateDb(version)
+				if newversion == Fail2BanDb.__version__:
 					logSys.warning( "Database updated from '%i' to '%i'",
-					version, self.updateDb(version))
+						version, newversion)
+				else:
+					logSys.error( "Database update failed to acheive version '%i'"
+						": updated from '%i' to '%i'",
+						Fail2BanDb.__version__, version, newversion)
+					raise Exception('Failed to fully update')
 		finally:
 			cur.close()
 
@@ -102,53 +142,37 @@ class Fail2BanDb(object):
 	@commitandrollback
 	def createDb(self, cur):
 		# Version info
-		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
+		cur.executescript(Fail2BanDb._TABLE_fail2banDb)
 		cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
 			(Fail2BanDb.__version__, ))
-
 		# Jails
-		cur.execute("CREATE TABLE jails("
+		cur.executescript(Fail2BanDb._TABLE_jails)
-			"name TEXT NOT NULL UNIQUE, "
-			"enabled INTEGER NOT NULL DEFAULT 1"
-			")")
-		cur.execute("CREATE INDEX jails_name ON jails(name)")
-
 		# Logs
-		cur.execute("CREATE TABLE logs("
+		cur.executescript(Fail2BanDb._TABLE_logs)
-			"jail TEXT NOT NULL, "
-			"path TEXT, "
-			"firstlinemd5 TEXT, "
-			#TODO: systemd journal features
-			#"journalmatch TEXT, "
-			#"journlcursor TEXT, "
-			"lastfilepos INTEGER DEFAULT 0, "
-			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
-			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, "
-			"UNIQUE(jail, path)"
-			")")
-		cur.execute("CREATE INDEX logs_path ON logs(path)")
-		cur.execute("CREATE INDEX logs_jail_path ON logs(jail, path)")
-
 		# Bans
-		cur.execute("CREATE TABLE bans("
+		cur.executescript(Fail2BanDb._TABLE_bans)
-			"jail TEXT NOT NULL, "
-			"ip TEXT, "
-			"timeofban INTEGER NOT NULL, "
-			"data JSON, "
-			"FOREIGN KEY(jail) REFERENCES jails(name) "
-			")")
-		cur.execute(
-			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban)")
-		cur.execute("CREATE INDEX bans_jail_ip ON bans(jail, ip)")
-		cur.execute("CREATE INDEX bans_ip ON bans(ip)")
 
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
 	@commitandrollback
 	def updateDb(self, cur, version):
+		self.dbBackupFilename = self._dbFilename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
+		shutil.copyfile(self._dbFilename, self.dbBackupFilename)
+		if version > Fail2BanDb.__version__:
 			raise NotImplementedError(
-			"Only single version of database exists...how did you get here??")
+						"Attempt to travel to future version of database ...how did you get here??")
+
+		if version < 2:
+			cur.executescript("BEGIN TRANSACTION;"
+						"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
+						"DROP TABLE logs;"
+						"%s;"
+						"INSERT INTO logs SELECT * from logs_temp;"
+						"DROP TABLE logs_temp;"
+						"UPDATE fail2banDb SET version = 2;"
+						"COMMIT;" % Fail2BanDb._TABLE_logs)
+
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
@@ -187,14 +211,14 @@ class Fail2BanDb(object):
 		try:
 			firstLineMD5, lastLinePos = cur.fetchone()
 		except TypeError:
+			firstLineMD5 = False
+
 		cur.execute(
-				"INSERT INTO logs(jail, path, firstlinemd5, lastfilepos) "
+				"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
 					"VALUES(?, ?, ?, ?)",
 				(jail.getName(), container.getFileName(),
 					container.getHash(), container.getPos()))
-		else:
 		if container.getHash() != firstLineMD5:
-				self._updateLog(cur, jail, container)
 			lastLinePos = None
 		return lastLinePos
 

fail2ban/tests/databasetestcase.py

@@ -26,6 +26,7 @@ import os
 import unittest
 import tempfile
 import sqlite3
+import shutil
 
 from fail2ban.server.database import Fail2BanDb
 from fail2ban.server.filter import FileContainer
@@ -64,8 +65,16 @@ class DatabaseTest(unittest.TestCase):
 			"Jail not retained in Db after disconnect reconnect.")
 
 	def testUpdateDb(self):
-		# TODO: Currently only single version exists
+		shutil.copyfile('fail2ban/tests/files/database_v1.db', self.dbFilename)
-		pass
+		self.db = Fail2BanDb(self.dbFilename)
+		self.assertEqual(self.db.getJailNames(), {'DummyJail #29162448 with 0 tickets'})
+		self.assertEqual(self.db.getLogPaths(), {'/tmp/Fail2BanDb_pUlZJh.log'})
+		ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"])
+		self.assertEqual(self.db.getBans()[0], ticket)
+
+		self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__)
+		self.assertRaises(NotImplementedError, self.db.updateDb, Fail2BanDb.__version__ + 1)
+		os.remove(self.db.dbBackupFilename)
 
 	def testAddJail(self):
 		self.jail = DummyJail()
@@ -83,6 +92,7 @@ class DatabaseTest(unittest.TestCase):
 		self.db.addLog(self.jail, self.fileContainer)
 
 		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+		os.remove(filename)
 
 	def testUpdateLog(self):
 		self.testAddLog() # Add log file
@@ -121,6 +131,7 @@ class DatabaseTest(unittest.TestCase):
 		# last position in file
 		self.assertEqual(
 			self.db.addLog(self.jail, self.fileContainer), None)
+		os.remove(filename)
 
 	def testAddBan(self):
 		self.testAddJail()