From bbadef847b19af886547b01d58e09492b6cf7175 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 7 Dec 2013 23:23:28 +0000
Subject: [PATCH 1/7] ENH: Add fail2ban persistent data storage

---
 config/fail2ban.conf               |  11 ++
 fail2ban/client/beautifier.py      |   6 +
 fail2ban/client/fail2banreader.py  |   8 +-
 fail2ban/protocol.py               |   5 +
 fail2ban/server/database.py        | 254 +++++++++++++++++++++++++++++
 fail2ban/server/filter.py          |  18 +-
 fail2ban/server/jail.py            |  10 +-
 fail2ban/server/jails.py           |   4 +-
 fail2ban/server/server.py          |  27 ++-
 fail2ban/server/transmitter.py     |  12 ++
 fail2ban/tests/databasetestcase.py | 156 ++++++++++++++++++
 fail2ban/tests/dummyjail.py        |  14 ++
 fail2ban/tests/filtertestcase.py   |  16 +-
 fail2ban/tests/servertestcase.py   |  12 +-
 fail2ban/tests/utils.py            |   3 +
 setup.py                           |   3 +
 16 files changed, 541 insertions(+), 18 deletions(-)
 create mode 100644 fail2ban/server/database.py
 create mode 100644 fail2ban/tests/databasetestcase.py

diff --git a/config/fail2ban.conf b/config/fail2ban.conf
index 2c487e51..97163fec 100644
--- a/config/fail2ban.conf
+++ b/config/fail2ban.conf
@@ -47,3 +47,14 @@ socket = /var/run/fail2ban/fail2ban.sock
 #
 pidfile = /var/run/fail2ban/fail2ban.pid
 
+# Options: dbfile
+# Notes.: Set the file for the fail2ban persistent data to be stored.
+#         A value of :memory: means database is only stored in memory, and
+#         data is lost once fail2ban is stops.
+# Values: [ FILE ] :memory:  Default: /var/lib/fail2ban/fail2ban.db
+dbfile = /var/lib/fail2ban/fail2ban.db
+
+# Options: dbpurgeage
+# Notes.: Sets age at which bans should be purged from the database
+# Values: [ SECONDS ] Default: 86400 (24hours)
+dbpurgeage = 86400
diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index f2c1fb16..033ce816 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -102,6 +102,12 @@ class Beautifier:
 					msg = msg + "DEBUG"
 				else:
 					msg = msg + `response`
+			elif inC[1] == "dbfile":
+				msg = "Current database file is:\n"
+				msg = msg + "`- " + response
+			elif inC[1] == "dbpurgeage":
+				msg = "Current database purge age is:\n"
+				msg = msg + "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index d4053abe..86fcbb27 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -45,7 +45,9 @@ class Fail2banReader(ConfigReader):
 	
 	def getOptions(self):
 		opts = [["int", "loglevel", 1],
-				["string", "logtarget", "STDERR"]]
+				["string", "logtarget", "STDERR"],
+				["string", "dbfile", "/var/lib/fail2ban/fail2ban.db"],
+				["int", "dbpurgeage", "/var/lib/fail2ban/fail2ban.db"]]
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
 	
 	def convert(self):
@@ -55,5 +57,9 @@ class Fail2banReader(ConfigReader):
 				stream.append(["set", "loglevel", self.__opts[opt]])
 			elif opt == "logtarget":
 				stream.append(["set", "logtarget", self.__opts[opt]])
+			elif opt == "dbfile":
+				stream.append(["set", "dbfile", self.__opts[opt]])
+			elif opt == "dbpurgeage":
+				stream.append(["set", "dbpurgeage", self.__opts[opt]])
 		return stream
 	
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index 25ad3b5c..dd3cf0c0 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -43,6 +43,11 @@ protocol = [
 ["get loglevel", "gets the logging level"], 
 ["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG or a file"], 
 ["get logtarget", "gets logging target"], 
+['', "DATABASE", ""],
+["set dbfile <FILE>", "set the location of fail2ban persistent datastore"], 
+["get dbfile", "get the location of fail2ban persistent datastore"], 
+["set dbpurgeage <SECONDS>", "sets the max age in <SECONDS> that history of bans will be kept"], 
+["get dbpurgeage", "gets the max age in seconds that history of bans will be kept"], 
 ['', "JAIL CONTROL", ""],
 ["add <JAIL> <BACKEND>", "creates <JAIL> using <BACKEND>"], 
 ["start <JAIL>", "starts the jail <JAIL>"], 
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
new file mode 100644
index 00000000..d211c6bf
--- /dev/null
+++ b/fail2ban/server/database.py
@@ -0,0 +1,254 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Steven Hiscocks"
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import logging
+import sys
+from threading import Lock
+import sqlite3
+import json
+import locale
+
+from fail2ban.server.mytime import MyTime
+from fail2ban.server.ticket import FailTicket
+
+# Gets the instance of the logger.
+logSys = logging.getLogger(__name__)
+
+if sys.version_info >= (3,):
+	sqlite3.register_adapter(
+		dict,
+		lambda x: json.dumps(x, ensure_ascii=False).encode(
+			locale.getpreferredencoding(), 'replace'))
+	sqlite3.register_converter(
+		"JSON",
+		lambda x: json.loads(x.decode(
+			locale.getpreferredencoding(), 'replace')))
+else:
+	sqlite3.register_adapter(dict, json.dumps)
+	sqlite3.register_converter("JSON", json.loads)
+
+def lockandcommit():
+	def wrap(f):
+		def func(self, *args, **kw):
+			with self._lock: # Threading lock
+				with self._db: # Auto commit and rollback on exception
+					return f(self, self._db.cursor(), *args, **kw)
+		return func
+	return wrap
+
+class Fail2BanDb(object):
+	__version__ = 1
+	def __init__(self, filename, purgeAge=24*60*60):
+		self._lock = Lock()
+		try:
+			self._db = sqlite3.connect(
+				filename, check_same_thread=False,
+				detect_types=sqlite3.PARSE_DECLTYPES)
+			self._dbFilename = filename
+			self._purgeAge = purgeAge
+
+			logSys.info(
+				"Connected to fail2ban persistent database '%s'", filename)
+		except sqlite3.OperationalError, e:
+			logSys.error(
+				"Error connecting to fail2ban persistent database '%s': %s",
+				filename, e.args[0])
+			raise
+
+		cur = self._db.cursor()
+		cur.execute("PRAGMA foreign_keys = ON;")
+
+		try:
+			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		except sqlite3.OperationalError:
+			logSys.warning("New database created. Version '%i'",
+				self.createDb())
+		else:
+			version = cur.fetchone()[0]
+			if version < Fail2BanDb.__version__:
+				logSys.warning( "Database updated from '%i' to '%i'",
+					version, self.updateDb(version))
+		finally:
+			cur.close()
+
+	def getFilename(self):
+		return self._dbFilename
+
+	def getPurgeAge(self):
+		return self._purgeAge
+
+	def setPurgeAge(self, value):
+		self._purgeAge = int(value)
+
+	@lockandcommit()
+	def createDb(self, cur):
+		# Version info
+		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
+		cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
+			(Fail2BanDb.__version__, ))
+
+		# Jails
+		cur.execute("CREATE TABLE jails("
+			"name TEXT NOT NULL UNIQUE, "
+			"enabled INTEGER NOT NULL DEFAULT 1"
+			")")
+		cur.execute("CREATE INDEX jails_name ON jails(name)")
+
+		# Logs
+		cur.execute("CREATE TABLE logs("
+			"jail TEXT NOT NULL, "
+			"path TEXT, "
+			"firstlinemd5 TEXT, "
+			#TODO: systemd journal features
+			#"journalmatch TEXT, "
+			#"journlcursor TEXT, "
+			"lastfilepos INTEGER DEFAULT 0, "
+			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
+			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, "
+			"UNIQUE(jail, path)"
+			")")
+		cur.execute("CREATE INDEX logs_path ON logs(path)")
+		cur.execute("CREATE INDEX logs_jail_path ON logs(jail, path)")
+
+		# Bans
+		cur.execute("CREATE TABLE bans("
+			"jail TEXT NOT NULL, "
+			"ip TEXT, "
+			"timeofban INTEGER NOT NULL, "
+			"data JSON, "
+			"FOREIGN KEY(jail) REFERENCES jails(name) "
+			")")
+		cur.execute(
+			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban)")
+		cur.execute("CREATE INDEX bans_jail_ip ON bans(jail, ip)")
+		cur.execute("CREATE INDEX bans_ip ON bans(ip)")
+
+		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		return cur.fetchone()[0]
+
+	@lockandcommit()
+	def updateDb(self, cur, version):
+		raise NotImplementedError(
+			"Only single version of database exists...how did you get here??")
+		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		return cur.fetchone()[0]
+
+	@lockandcommit()
+	def addJail(self, cur, jail):
+		cur.execute(
+			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
+			(jail.getName(),))
+
+	def delJail(self, jail):
+		return self.delJailName(jail.getName())
+
+	@lockandcommit()
+	def delJailName(self, cur, name):
+		# Will be deleted by purge as appropriate
+		cur.execute(
+			"UPDATE jails SET enabled=0 WHERE name=?", (name, ))
+
+	@lockandcommit()
+	def getJailNames(self, cur):
+		cur.execute("SELECT name FROM jails")
+		return set(row[0] for row in cur.fetchmany())
+
+	@lockandcommit()
+	def addLog(self, cur, jail, container):
+		lastLinePos = None
+		cur.execute(
+			"SELECT firstlinemd5, lastfilepos FROM logs "
+				"WHERE jail=? AND path=?",
+			(jail.getName(), container.getFileName()))
+		try:
+			firstLineMD5, lastLinePos = cur.fetchone()
+		except TypeError:
+			cur.execute(
+				"INSERT INTO logs(jail, path, firstlinemd5, lastfilepos) "
+					"VALUES(?, ?, ?, ?)",
+				(jail.getName(), container.getFileName(),
+					container.getHash(), container.getPos()))
+		else:
+			if container.getHash() != firstLineMD5:
+				self._updateLog(cur, jail, container)
+				lastLinePos = None
+		return lastLinePos
+
+	@lockandcommit()
+	def getLogPaths(self, cur, jail=None):
+		query = "SELECT path FROM logs"
+		queryArgs = []
+		if jail is not None:
+			query += " WHERE jail=?"
+			queryArgs.append(jail.getName())
+		cur.execute(query, queryArgs)
+		return set(row[0] for row in cur.fetchmany())
+
+	@lockandcommit()
+	def updateLog(self, cur, *args, **kwargs):
+		self._updateLog(cur, *args, **kwargs)
+
+	def _updateLog(self, cur, jail, container):
+		cur.execute(
+			"UPDATE logs SET firstlinemd5=?, lastfilepos=? "
+				"WHERE jail=? AND path=?",
+			(container.getHash(), container.getPos(),
+				jail.getName(), container.getFileName()))
+
+	@lockandcommit()
+	def addBan(self, cur, jail, ticket):
+		#TODO: Implement data parts once arbitrary match keys completed
+		cur.execute(
+			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
+			(jail.getName(), ticket.getIP(), ticket.getTime(),
+				{"matches": ticket.getMatches(),
+					"failures": ticket.getAttempt()}))
+
+	@lockandcommit()
+	def getBans(self, cur, jail=None, bantime=None):
+		query = "SELECT ip, timeofban, data FROM bans"
+		queryArgs = []
+
+		if jail is not None:
+			query += " WHERE jail=?"
+			queryArgs.append(jail.getName())
+		if bantime is not None:
+			query += " AND timeofban > ?"
+			queryArgs.append(MyTime.time() - bantime)
+
+		tickets = []
+		for ip, timeofban, data in cur.execute(query, queryArgs):
+			#TODO: Implement data parts once arbitrary match keys completed
+			tickets.append(FailTicket(ip, timeofban, data['matches']))
+			tickets[-1].setAttempt(data['failures'])
+		return tickets
+
+	@lockandcommit()
+	def purge(self, cur):
+		cur.execute(
+			"DELETE FROM bans WHERE timeofban < ?",
+			(MyTime.time() - self._purgeAge, ))
+		cur.execute(
+			"DELETE FROM jails WHERE enabled = 0 "
+				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name)")
+
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index e2da5e77..c8371dc6 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -527,6 +527,9 @@ class FileFilter(Filter):
 			logSys.error(path + " already exists")
 		else:
 			container = FileContainer(path, self.getLogEncoding(), tail)
+			lastpos = self.jail.getDatabase().addLog(self.jail, container)
+			if lastpos and not tail:
+				container.setPos(lastpos)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@@ -546,6 +549,7 @@ class FileFilter(Filter):
 		for log in self.__logPath:
 			if log.getFileName() == path:
 				self.__logPath.remove(log)
+				self.jail.getDatabase().updateLog(self.jail, log)
 				logSys.info("Removed logfile = %s" % path)
 				self._delLogPath(path)
 				return
@@ -644,6 +648,7 @@ class FileFilter(Filter):
 				break
 			self.processLineAndAdd(line)
 		container.close()
+		self.jail.getDatabase().updateLog(self.jail, container)
 		return True
 
 	def status(self):
@@ -682,7 +687,7 @@ class FileContainer:
 		try:
 			firstLine = handler.readline()
 			# Computes the MD5 of the first line.
-			self.__hash = md5sum(firstLine).digest()
+			self.__hash = md5sum(firstLine).hexdigest()
 			# Start at the beginning of file if tail mode is off.
 			if tail:
 				handler.seek(0, 2)
@@ -702,6 +707,15 @@ class FileContainer:
 	def getEncoding(self):
 		return self.__encoding
 
+	def getHash(self):
+		return self.__hash
+
+	def getPos(self):
+		return self.__pos
+
+	def setPos(self, value):
+		self.__pos = value
+
 	def open(self):
 		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
@@ -717,7 +731,7 @@ class FileContainer:
 			return False
 		firstLine = self.__handler.readline()
 		# Computes the MD5 of the first line.
-		myHash = md5sum(firstLine).digest()
+		myHash = md5sum(firstLine).hexdigest()
 		## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % (
 		## 	self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos,
 		## 	self.__hash != myHash or self.__ino != stats.st_ino)
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index d25fda7e..3895dfd6 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -37,7 +37,8 @@ class Jail:
 	#      list had .index until 2.6
 	_BACKENDS = ['pyinotify', 'gamin', 'polling', 'systemd']
 
-	def __init__(self, name, backend = "auto"):
+	def __init__(self, db, name, backend = "auto"):
+		self.__db = db
 		self.setName(name)
 		self.__queue = Queue.Queue()
 		self.__filter = None
@@ -118,6 +119,9 @@ class Jail:
 	def getName(self):
 		return self.__name
 	
+	def getDatabase(self):
+		return self.__db
+	
 	def getFilter(self):
 		return self.__filter
 	
@@ -126,6 +130,7 @@ class Jail:
 	
 	def putFailTicket(self, ticket):
 		self.__queue.put(ticket)
+		self.__db.addBan(self, ticket)
 	
 	def getFailTicket(self):
 		try:
@@ -136,6 +141,9 @@ class Jail:
 	def start(self):
 		self.__filter.start()
 		self.__action.start()
+		# Restore any previous valid bans from the database
+		for ticket in self.__db.getBans(self, self.__action.getBanTime()):
+			self.__queue.put(ticket)
 		logSys.info("Jail '%s' started" % self.__name)
 	
 	def stop(self):
diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py
index 7ea1dde0..5fe9baef 100644
--- a/fail2ban/server/jails.py
+++ b/fail2ban/server/jails.py
@@ -50,13 +50,13 @@ class Jails:
 	# @param name The name of the jail
 	# @param backend The backend to use
 	
-	def add(self, name, backend):
+	def add(self, db, name, backend):
 		try:
 			self.__lock.acquire()
 			if self.__jails.has_key(name):
 				raise DuplicateJailException(name)
 			else:
-				self.__jails[name] = Jail(name, backend)
+				self.__jails[name] = Jail(db, name, backend)
 		finally:
 			self.__lock.release()
 	
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 69d888f8..408f553f 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -30,6 +30,7 @@ from filter import FileFilter, JournalFilter
 from transmitter import Transmitter
 from asyncserver import AsyncServer
 from asyncserver import AsyncServerException
+from database import Fail2BanDb
 from fail2ban import version
 import logging, logging.handlers, sys, os, signal
 
@@ -50,6 +51,9 @@ class Server:
 		# Set logging level
 		self.setLogLevel(3)
 		self.setLogTarget("STDOUT")
+
+		# Create database, initially in memory
+		self.setDatabase(":memory:")
 	
 	def __sigTERMhandler(self, signum, frame):
 		logSys.debug("Caught signal %d. Exiting" % signum)
@@ -117,10 +121,13 @@ class Server:
 
 	
 	def addJail(self, name, backend):
-		self.__jails.add(name, backend)
+		self.__jails.add(self.__db, name, backend)
+		self.__db.addJail(self.__jails.get(name))
 		
-	def delJail(self, name):
+	def delJail(self, name, dbDel=True):
 		self.__jails.remove(name)
+		if dbDel:
+			self.__db.delJailName(name)
 	
 	def startJail(self, name):
 		try:
@@ -130,13 +137,13 @@ class Server:
 		finally:
 			self.__lock.release()
 	
-	def stopJail(self, name):
+	def stopJail(self, name, dbDel=True):
 		logSys.debug("Stopping jail %s" % name)
 		try:
 			self.__lock.acquire()
 			if self.isAlive(name):
 				self.__jails.get(name).stop()
-				self.delJail(name)
+				self.delJail(name, dbDel=dbDel)
 		finally:
 			self.__lock.release()
 	
@@ -145,7 +152,7 @@ class Server:
 		try:
 			self.__lock.acquire()
 			for jail in self.__jails.getAll():
-				self.stopJail(jail)
+				self.stopJail(jail, dbDel=False)
 		finally:
 			self.__lock.release()
 	
@@ -460,6 +467,16 @@ class Server:
 		finally:
 			self.__loggingLock.release()
 	
+	def setDatabase(self, filename):
+		if self.__jails.size() == 0:
+			self.__db = Fail2BanDb(filename)
+		else:
+			raise RuntimeError(
+				"Cannot change database when there are jails present")
+	
+	def getDatabase(self):
+		return self.__db
+	
 	def __createDaemon(self): # pragma: no cover
 		""" Detach a process from the controlling terminal and run it in the
 			background as a daemon.
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index 14c5cd60..b3712796 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -113,6 +113,13 @@ class Transmitter:
 				return self.__server.getLogTarget()
 			else:
 				raise Exception("Failed to change log target")
+		#Database
+		elif name == "dbfile":
+			self.__server.setDatabase(command[1])
+			return self.__server.getDatabase().getFilename()
+		elif name == "dbpurgeage":
+			self.__server.getDatabase().setPurgeAge(command[1])
+			return self.__server.getDatabase().getPurgeAge()
 		# Jail
 		elif command[1] == "idle":
 			if command[2] == "on":
@@ -257,6 +264,11 @@ class Transmitter:
 			return self.__server.getLogLevel()
 		elif name == "logtarget":
 			return self.__server.getLogTarget()
+		#Database
+		elif name == "dbfile":
+			return self.__server.getDatabase().getFilename()
+		elif name == "dbpurgeage":
+			return self.__server.getDatabase().getPurgeAge()
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
new file mode 100644
index 00000000..e001b865
--- /dev/null
+++ b/fail2ban/tests/databasetestcase.py
@@ -0,0 +1,156 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Fail2Ban developers
+
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import os
+import unittest
+import tempfile
+import sqlite3
+
+from fail2ban.server.database import Fail2BanDb
+from fail2ban.server.filter import FileContainer
+from fail2ban.server.mytime import MyTime
+from fail2ban.server.ticket import FailTicket
+from fail2ban.tests.dummyjail import DummyJail
+
+class DatabaseTest(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
+		self.db = Fail2BanDb(self.dbFilename)
+
+	def tearDown(self):
+		"""Call after every test case."""
+		# Cleanup
+		os.remove(self.dbFilename)
+
+	def getFilename(self):
+		self.assertEqual(self.dbFilename, self.db.getFilename())
+
+	def testCreateInvalidPath(self):
+		self.assertRaises(
+			sqlite3.OperationalError,
+			Fail2BanDb,
+			"/this/path/should/not/exist")
+
+	def testCreateAndReconnect(self):
+		self.testAddJail()
+		# Reconnect...
+		self.db = Fail2BanDb(self.dbFilename)
+		# and check jail of same name still present
+		self.assertTrue(
+			self.jail.getName() in self.db.getJailNames(),
+			"Jail not retained in Db after disconnect reconnect.")
+
+	def testUpdateDb(self):
+		# TODO: Currently only single version exists
+		pass
+
+	def testAddJail(self):
+		self.jail = DummyJail()
+		self.db.addJail(self.jail)
+		self.assertTrue(
+			self.jail.getName() in self.db.getJailNames(),
+			"Jail not added to database")
+
+	def testAddLog(self):
+		self.testAddJail() # Jail required
+
+		_, filename = tempfile.mkstemp(".log", "Fail2BanDb_")
+		self.fileContainer = FileContainer(filename, "utf-8")
+
+		self.db.addLog(self.jail, self.fileContainer)
+
+		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+
+	def testUpdateLog(self):
+		self.testAddLog() # Add log file
+
+		# Write some text
+		filename = self.fileContainer.getFileName()
+		file_ = open(filename, "w")
+		file_.write("Some text to write which will change md5sum\n")
+		file_.close()
+		self.fileContainer.open()
+		self.fileContainer.readline()
+		self.fileContainer.close()
+
+		# Capture position which should be after line just written
+		lastPos = self.fileContainer.getPos()
+		self.assertTrue(lastPos > 0)
+		self.db.updateLog(self.jail, self.fileContainer)
+
+		# New FileContainer for file
+		self.fileContainer = FileContainer(filename, "utf-8")
+		self.assertEqual(self.fileContainer.getPos(), 0)
+
+		# Database should return previous position in file
+		self.assertEqual(
+			self.db.addLog(self.jail, self.fileContainer), lastPos)
+
+		# Change md5sum
+		file_ = open(filename, "w") # Truncate
+		file_.write("Some different text to change md5sum\n")
+		file_.close()
+
+		self.fileContainer = FileContainer(filename, "utf-8")
+		self.assertEqual(self.fileContainer.getPos(), 0)
+
+		# Database should be aware of md5sum change, such doesn't return
+		# last position in file
+		self.assertEqual(
+			self.db.addLog(self.jail, self.fileContainer), None)
+
+	def testAddBan(self):
+		self.testAddJail()
+		ticket = FailTicket("127.0.0.1", 0, [])
+		self.db.addBan(self.jail, ticket)
+
+		self.assertEquals(len(self.db.getBans(self.jail)), 1)
+		self.assertTrue(
+			isinstance(self.db.getBans(self.jail)[0], FailTicket))
+
+	def testPurge(self):
+		self.testAddJail() # Add jail
+
+		self.db.purge() # Jail enabled by default so shouldn't be purged
+		self.assertEqual(len(self.db.getJailNames()), 1)
+
+		self.db.delJail(self.jail)
+		self.db.purge() # Should remove jail
+		self.assertEqual(len(self.db.getJailNames()), 0)
+
+		self.testAddBan()
+		self.db.delJail(self.jail)
+		self.db.purge() # Purge should remove all bans
+		self.assertEqual(len(self.db.getJailNames()), 0)
+		self.assertEqual(len(self.db.getBans(self.jail)), 0)
+
+		# Should leave jail
+		self.testAddJail()
+		self.db.addBan(self.jail, FailTicket("127.0.0.1", MyTime.time(), []))
+		self.db.delJail(self.jail)
+		self.db.purge() # Should leave jail as ban present
+		self.assertEqual(len(self.db.getJailNames()), 1)
+		self.assertEqual(len(self.db.getBans(self.jail)), 1)
diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index bd11bd5b..ef291d71 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -23,12 +23,17 @@ __copyright__ = "Copyright (c) 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
 from threading import Lock
+
+from fail2ban.server.database import Fail2BanDb
+
 class DummyJail(object):
 	"""A simple 'jail' to suck in all the tickets generated by Filter's
 	"""
 	def __init__(self):
 		self.lock = Lock()
 		self.queue = []
+		self.__db = Fail2BanDb(":memory:")
+		self.__db.addJail(self)
 
 	def __len__(self):
 		try:
@@ -54,6 +59,15 @@ class DummyJail(object):
 		finally:
 			self.lock.release()
 
+	def setIdle(self, value):
+		pass
+
+	def getIdle(self):
+		pass
+
 	def getName(self):
 		return "DummyJail #%s with %d tickets" % (id(self), len(self))
 
+	def getDatabase(self):
+		return self.__db
+
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 85d0ec16..c41d7ee4 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -40,6 +40,7 @@ from fail2ban.server.filter import FileFilter, DNSUtils
 from fail2ban.server.failmanager import FailManager
 from fail2ban.server.failmanager import FailManagerEmpty
 from fail2ban.server.mytime import MyTime
+from fail2ban.server.database import Fail2BanDb
 from fail2ban.tests.utils import setUpMyTime, tearDownMyTime
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
@@ -228,7 +229,7 @@ class LogFile(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
-		self.filter = FilterPoll(None)
+		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(LogFile.FILENAME)
 
 	def tearDown(self):
@@ -251,7 +252,7 @@ class LogFileMonitor(unittest.TestCase):
 		self.filter = self.name = 'NA'
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
 		self.file = open(self.name, 'a')
-		self.filter = FilterPoll(None)
+		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(self.name)
 		self.filter.setActive(True)
 		self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
@@ -564,7 +565,7 @@ def get_monitor_failures_testcase(Filter_):
 			# tail written before, so let's not copy anything yet
 			#_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100)
 			# we should detect the failures
-			self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above
+			self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above
 
 			# now copy and get even more
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
@@ -715,7 +716,8 @@ class GetFailures(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		setUpMyTime()
-		self.filter = FileFilter(None)
+		self.jail = DummyJail()
+		self.filter = FileFilter(self.jail)
 		self.filter.setActive(True)
 		# TODO Test this
 		#self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
@@ -802,7 +804,8 @@ class GetFailures(unittest.TestCase):
 		for useDns, output in (('yes',  output_yes),
 							   ('no',   output_no),
 							   ('warn', output_yes)):
-			filter_ = FileFilter(None, useDns=useDns)
+			jail = DummyJail()
+			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.setActive(True)
 			filter_.failManager.setMaxRetry(1)	# we might have just few failures
 
@@ -916,5 +919,6 @@ class JailTests(unittest.TestCase):
 
 	def testSetBackend_gh83(self):
 		# smoke test
-		jail = Jail('test', backend='polling') # Must not fail to initiate
+		# Must not fail to initiate
+		jail = Jail(Fail2BanDb(":memory:"), 'test', backend='polling')
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index ba8f222f..bd8a27d7 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -28,6 +28,7 @@ import unittest, socket, time, tempfile, os, locale, sys
 
 from fail2ban.server.server import Server
 from fail2ban.server.jail import Jail
+from fail2ban.server.database import Fail2BanDb
 from fail2ban.exceptions import UnknownJailException
 try:
 	from fail2ban.server import filtersystemd
@@ -168,6 +169,15 @@ class Transmitter(TransmitterBase):
 		# Approx 1 second delay
 		self.assertAlmostEqual(t1 - t0, 1, places=2)
 
+	def testDatabase(self):
+		_, tmpFilename = tempfile.mkstemp(".db", "Fail2Ban_")
+		# Jails present, cant change database
+		self.setGetTestNOK("dbfile", tmpFilename)
+		self.server.delJail(self.jailName)
+		self.setGetTest("dbfile", tmpFilename)
+		self.setGetTest("dbpurgeage", 600)
+		self.setGetTestNOK("dbpurgeage", "LIZARD")
+
 	def testAddJail(self):
 		jail2 = "TestJail2"
 		jail3 = "TestJail3"
@@ -641,5 +651,5 @@ class JailTests(unittest.TestCase):
 	def testLongName(self):
 		# Just a smoke test for now
 		longname = "veryveryverylongname"
-		jail = Jail(longname)
+		jail = Jail(Fail2BanDb(":memory:"), longname)
 		self.assertEqual(jail.getName(), longname)
diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py
index e719750a..be2bb744 100644
--- a/fail2ban/tests/utils.py
+++ b/fail2ban/tests/utils.py
@@ -146,6 +146,7 @@ def gatherTests(regexps=None, no_network=False):
 	from fail2ban.tests import actiontestcase
 	from fail2ban.tests import sockettestcase
 	from fail2ban.tests import misctestcase
+	from fail2ban.tests import databasetestcase
 	if json:
 		from fail2ban.tests import samplestestcase
 
@@ -184,6 +185,8 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(misctestcase.HelpersTest))
 	tests.addTest(unittest.makeSuite(misctestcase.SetupTest))
 	tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest))
+	# Database
+	tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest))
 
 	# Filter
 	if not no_network:
diff --git a/setup.py b/setup.py
index 38a2269a..407e8232 100755
--- a/setup.py
+++ b/setup.py
@@ -131,6 +131,9 @@ setup(
 						('/var/run/fail2ban',
 							''
 						),
+						('/var/lib/fail2ban',
+							''
+						),
 						('/usr/share/doc/fail2ban',
 							['README.md', 'DEVELOP', 'doc/run-rootless.txt']
 						)

From d6fe80ba5073c40f0ff98042924429f507257c9d Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sat, 7 Dec 2013 23:37:14 +0000
Subject: [PATCH 2/7] TST: Fix test for fail2ban.conf with new database options

---
 fail2ban/tests/clientreadertestcase.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 4743ccba..de7429a7 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -360,7 +360,9 @@ class JailsReaderTest(unittest.TestCase):
 		# and there is logging information left to be passed into the
 		# server
 		self.assertEqual(sorted(commands),
-						 [['set', 'loglevel', 3],
+						 [['set', 'dbfile', '/var/lib/fail2ban/fail2ban.db'],
+						  ['set', 'dbpurgeage', 86400],
+						  ['set', 'loglevel', 3],
 						  ['set', 'logtarget', '/var/log/fail2ban.log']])
 
 		# and if we force change configurator's fail2ban's baseDir

From d8c7bca9b04a7ed53afa0c20b1a846821807e69f Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 8 Dec 2013 11:35:12 +0000
Subject: [PATCH 3/7] BF: Fix dbpurgeage default value, and change default
 dbfile extension

---
 config/fail2ban.conf                   | 4 ++--
 fail2ban/client/fail2banreader.py      | 4 ++--
 fail2ban/tests/clientreadertestcase.py | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/config/fail2ban.conf b/config/fail2ban.conf
index 97163fec..e01de77e 100644
--- a/config/fail2ban.conf
+++ b/config/fail2ban.conf
@@ -51,8 +51,8 @@ pidfile = /var/run/fail2ban/fail2ban.pid
 # Notes.: Set the file for the fail2ban persistent data to be stored.
 #         A value of :memory: means database is only stored in memory, and
 #         data is lost once fail2ban is stops.
-# Values: [ FILE ] :memory:  Default: /var/lib/fail2ban/fail2ban.db
-dbfile = /var/lib/fail2ban/fail2ban.db
+# Values: [ FILE ] :memory:  Default: /var/lib/fail2ban/fail2ban.sqlite3
+dbfile = /var/lib/fail2ban/fail2ban.sqlite3
 
 # Options: dbpurgeage
 # Notes.: Sets age at which bans should be purged from the database
diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py
index 86fcbb27..d5d7285a 100644
--- a/fail2ban/client/fail2banreader.py
+++ b/fail2ban/client/fail2banreader.py
@@ -46,8 +46,8 @@ class Fail2banReader(ConfigReader):
 	def getOptions(self):
 		opts = [["int", "loglevel", 1],
 				["string", "logtarget", "STDERR"],
-				["string", "dbfile", "/var/lib/fail2ban/fail2ban.db"],
-				["int", "dbpurgeage", "/var/lib/fail2ban/fail2ban.db"]]
+				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
+				["int", "dbpurgeage", 86400]]
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
 	
 	def convert(self):
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index de7429a7..d45efb46 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -360,7 +360,8 @@ class JailsReaderTest(unittest.TestCase):
 		# and there is logging information left to be passed into the
 		# server
 		self.assertEqual(sorted(commands),
-						 [['set', 'dbfile', '/var/lib/fail2ban/fail2ban.db'],
+						 [['set', 'dbfile',
+							'/var/lib/fail2ban/fail2ban.sqlite3'],
 						  ['set', 'dbpurgeage', 86400],
 						  ['set', 'loglevel', 3],
 						  ['set', 'logtarget', '/var/log/fail2ban.log']])

From 7f063b46f93aa8a94111867d0330d472154746fa Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 8 Dec 2013 11:40:40 +0000
Subject: [PATCH 4/7] BF: Improve handling of clearing old jails in database

---
 fail2ban/server/database.py |  5 +++++
 fail2ban/server/server.py   | 12 ++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index d211c6bf..e7ac8255 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -168,6 +168,11 @@ class Fail2BanDb(object):
 		cur.execute(
 			"UPDATE jails SET enabled=0 WHERE name=?", (name, ))
 
+	@lockandcommit()
+	def delAllJails(self, cur):
+		# Will be deleted by purge as appropriate
+		cur.execute("UPDATE jails SET enabled=0")
+
 	@lockandcommit()
 	def getJailNames(self, cur):
 		cur.execute("SELECT name FROM jails")
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 408f553f..446a0989 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -124,10 +124,9 @@ class Server:
 		self.__jails.add(self.__db, name, backend)
 		self.__db.addJail(self.__jails.get(name))
 		
-	def delJail(self, name, dbDel=True):
+	def delJail(self, name):
 		self.__jails.remove(name)
-		if dbDel:
-			self.__db.delJailName(name)
+		self.__db.delJailName(name)
 	
 	def startJail(self, name):
 		try:
@@ -137,13 +136,13 @@ class Server:
 		finally:
 			self.__lock.release()
 	
-	def stopJail(self, name, dbDel=True):
+	def stopJail(self, name):
 		logSys.debug("Stopping jail %s" % name)
 		try:
 			self.__lock.acquire()
 			if self.isAlive(name):
 				self.__jails.get(name).stop()
-				self.delJail(name, dbDel=dbDel)
+				self.delJail(name)
 		finally:
 			self.__lock.release()
 	
@@ -152,7 +151,7 @@ class Server:
 		try:
 			self.__lock.acquire()
 			for jail in self.__jails.getAll():
-				self.stopJail(jail, dbDel=False)
+				self.stopJail(jail)
 		finally:
 			self.__lock.release()
 	
@@ -470,6 +469,7 @@ class Server:
 	def setDatabase(self, filename):
 		if self.__jails.size() == 0:
 			self.__db = Fail2BanDb(filename)
+			self.__db.delAllJails()
 		else:
 			raise RuntimeError(
 				"Cannot change database when there are jails present")

From 174f9a243a14f15fd1d6434f6ab8e4355fcd7497 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Sun, 8 Dec 2013 22:03:57 +0000
Subject: [PATCH 5/7] ENH: Remove thread locks from Fail2BanDb

---
 fail2ban/server/database.py | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index e7ac8255..073b426c 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -23,7 +23,6 @@ __license__ = "GPL"
 
 import logging
 import sys
-from threading import Lock
 import sqlite3
 import json
 import locale
@@ -47,19 +46,17 @@ else:
 	sqlite3.register_adapter(dict, json.dumps)
 	sqlite3.register_converter("JSON", json.loads)
 
-def lockandcommit():
+def commitandrollback():
 	def wrap(f):
 		def func(self, *args, **kw):
-			with self._lock: # Threading lock
-				with self._db: # Auto commit and rollback on exception
-					return f(self, self._db.cursor(), *args, **kw)
+			with self._db: # Auto commit and rollback on exception
+				return f(self, self._db.cursor(), *args, **kw)
 		return func
 	return wrap
 
 class Fail2BanDb(object):
 	__version__ = 1
 	def __init__(self, filename, purgeAge=24*60*60):
-		self._lock = Lock()
 		try:
 			self._db = sqlite3.connect(
 				filename, check_same_thread=False,
@@ -100,7 +97,7 @@ class Fail2BanDb(object):
 	def setPurgeAge(self, value):
 		self._purgeAge = int(value)
 
-	@lockandcommit()
+	@commitandrollback()
 	def createDb(self, cur):
 		# Version info
 		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
@@ -146,14 +143,14 @@ class Fail2BanDb(object):
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
-	@lockandcommit()
+	@commitandrollback()
 	def updateDb(self, cur, version):
 		raise NotImplementedError(
 			"Only single version of database exists...how did you get here??")
 		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
 		return cur.fetchone()[0]
 
-	@lockandcommit()
+	@commitandrollback()
 	def addJail(self, cur, jail):
 		cur.execute(
 			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
@@ -162,23 +159,23 @@ class Fail2BanDb(object):
 	def delJail(self, jail):
 		return self.delJailName(jail.getName())
 
-	@lockandcommit()
+	@commitandrollback()
 	def delJailName(self, cur, name):
 		# Will be deleted by purge as appropriate
 		cur.execute(
 			"UPDATE jails SET enabled=0 WHERE name=?", (name, ))
 
-	@lockandcommit()
+	@commitandrollback()
 	def delAllJails(self, cur):
 		# Will be deleted by purge as appropriate
 		cur.execute("UPDATE jails SET enabled=0")
 
-	@lockandcommit()
+	@commitandrollback()
 	def getJailNames(self, cur):
 		cur.execute("SELECT name FROM jails")
 		return set(row[0] for row in cur.fetchmany())
 
-	@lockandcommit()
+	@commitandrollback()
 	def addLog(self, cur, jail, container):
 		lastLinePos = None
 		cur.execute(
@@ -199,7 +196,7 @@ class Fail2BanDb(object):
 				lastLinePos = None
 		return lastLinePos
 
-	@lockandcommit()
+	@commitandrollback()
 	def getLogPaths(self, cur, jail=None):
 		query = "SELECT path FROM logs"
 		queryArgs = []
@@ -209,7 +206,7 @@ class Fail2BanDb(object):
 		cur.execute(query, queryArgs)
 		return set(row[0] for row in cur.fetchmany())
 
-	@lockandcommit()
+	@commitandrollback()
 	def updateLog(self, cur, *args, **kwargs):
 		self._updateLog(cur, *args, **kwargs)
 
@@ -220,7 +217,7 @@ class Fail2BanDb(object):
 			(container.getHash(), container.getPos(),
 				jail.getName(), container.getFileName()))
 
-	@lockandcommit()
+	@commitandrollback()
 	def addBan(self, cur, jail, ticket):
 		#TODO: Implement data parts once arbitrary match keys completed
 		cur.execute(
@@ -229,7 +226,7 @@ class Fail2BanDb(object):
 				{"matches": ticket.getMatches(),
 					"failures": ticket.getAttempt()}))
 
-	@lockandcommit()
+	@commitandrollback()
 	def getBans(self, cur, jail=None, bantime=None):
 		query = "SELECT ip, timeofban, data FROM bans"
 		queryArgs = []
@@ -248,7 +245,7 @@ class Fail2BanDb(object):
 			tickets[-1].setAttempt(data['failures'])
 		return tickets
 
-	@lockandcommit()
+	@commitandrollback()
 	def purge(self, cur):
 		cur.execute(
 			"DELETE FROM bans WHERE timeofban < ?",

From e18af48e34364a9fc37d15ff8ce81a301c70a3b1 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Tue, 10 Dec 2013 21:16:36 +0000
Subject: [PATCH 6/7] ENH: Database now optional, by setting dbfile to "None"

---
 config/fail2ban.conf             |  7 ++++---
 fail2ban/client/beautifier.py    | 14 ++++++++++----
 fail2ban/protocol.py             |  2 +-
 fail2ban/server/filter.py        | 16 +++++++++++-----
 fail2ban/server/jail.py          | 10 ++++++----
 fail2ban/server/jails.py         |  4 ++--
 fail2ban/server/server.py        | 19 +++++++++++--------
 fail2ban/server/transmitter.py   | 26 +++++++++++++++++++++-----
 fail2ban/tests/dummyjail.py      |  6 +-----
 fail2ban/tests/filtertestcase.py |  5 ++---
 fail2ban/tests/servertestcase.py | 19 ++++++++++++++++---
 11 files changed, 85 insertions(+), 43 deletions(-)

diff --git a/config/fail2ban.conf b/config/fail2ban.conf
index e01de77e..fb79da59 100644
--- a/config/fail2ban.conf
+++ b/config/fail2ban.conf
@@ -49,9 +49,10 @@ pidfile = /var/run/fail2ban/fail2ban.pid
 
 # Options: dbfile
 # Notes.: Set the file for the fail2ban persistent data to be stored.
-#         A value of :memory: means database is only stored in memory, and
-#         data is lost once fail2ban is stops.
-# Values: [ FILE ] :memory:  Default: /var/lib/fail2ban/fail2ban.sqlite3
+#         A value of ":memory:" means database is only stored in memory 
+#         and data is lost once fail2ban is stops.
+#         A value of "None" disables the database.
+# Values: [ None :memory: FILE ] Default: /var/lib/fail2ban/fail2ban.sqlite3
 dbfile = /var/lib/fail2ban/fail2ban.sqlite3
 
 # Options: dbpurgeage
diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py
index 033ce816..815a35d4 100644
--- a/fail2ban/client/beautifier.py
+++ b/fail2ban/client/beautifier.py
@@ -103,11 +103,17 @@ class Beautifier:
 				else:
 					msg = msg + `response`
 			elif inC[1] == "dbfile":
-				msg = "Current database file is:\n"
-				msg = msg + "`- " + response
+				if response is None:
+					msg = "Database currently disabled"
+				else:
+					msg = "Current database file is:\n"
+					msg = msg + "`- " + response
 			elif inC[1] == "dbpurgeage":
-				msg = "Current database purge age is:\n"
-				msg = msg + "`- %iseconds" % response
+				if response is None:
+					msg = "Database currently disabled"
+				else:
+					msg = "Current database purge age is:\n"
+					msg = msg + "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"
diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py
index dd3cf0c0..0435a415 100644
--- a/fail2ban/protocol.py
+++ b/fail2ban/protocol.py
@@ -44,7 +44,7 @@ protocol = [
 ["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG or a file"], 
 ["get logtarget", "gets logging target"], 
 ['', "DATABASE", ""],
-["set dbfile <FILE>", "set the location of fail2ban persistent datastore"], 
+["set dbfile <FILE>", "set the location of fail2ban persistent datastore. Set to \"None\" to disable"], 
 ["get dbfile", "get the location of fail2ban persistent datastore"], 
 ["set dbpurgeage <SECONDS>", "sets the max age in <SECONDS> that history of bans will be kept"], 
 ["get dbpurgeage", "gets the max age in seconds that history of bans will be kept"], 
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index c8371dc6..47294a87 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -527,9 +527,11 @@ class FileFilter(Filter):
 			logSys.error(path + " already exists")
 		else:
 			container = FileContainer(path, self.getLogEncoding(), tail)
-			lastpos = self.jail.getDatabase().addLog(self.jail, container)
-			if lastpos and not tail:
-				container.setPos(lastpos)
+			db = self.jail.getDatabase()
+			if db is not None:
+				lastpos = db.addLog(self.jail, container)
+				if lastpos and not tail:
+					container.setPos(lastpos)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@@ -549,7 +551,9 @@ class FileFilter(Filter):
 		for log in self.__logPath:
 			if log.getFileName() == path:
 				self.__logPath.remove(log)
-				self.jail.getDatabase().updateLog(self.jail, log)
+				db = self.jail.getDatabase()
+				if db is not None:
+					db.updateLog(self.jail, log)
 				logSys.info("Removed logfile = %s" % path)
 				self._delLogPath(path)
 				return
@@ -648,7 +652,9 @@ class FileFilter(Filter):
 				break
 			self.processLineAndAdd(line)
 		container.close()
-		self.jail.getDatabase().updateLog(self.jail, container)
+		db = self.jail.getDatabase()
+		if db is not None:
+			db.updateLog(self.jail, container)
 		return True
 
 	def status(self):
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index 3895dfd6..a24810ed 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -37,7 +37,7 @@ class Jail:
 	#      list had .index until 2.6
 	_BACKENDS = ['pyinotify', 'gamin', 'polling', 'systemd']
 
-	def __init__(self, db, name, backend = "auto"):
+	def __init__(self, name, backend = "auto", db=None):
 		self.__db = db
 		self.setName(name)
 		self.__queue = Queue.Queue()
@@ -130,7 +130,8 @@ class Jail:
 	
 	def putFailTicket(self, ticket):
 		self.__queue.put(ticket)
-		self.__db.addBan(self, ticket)
+		if self.__db is not None:
+			self.__db.addBan(self, ticket)
 	
 	def getFailTicket(self):
 		try:
@@ -142,8 +143,9 @@ class Jail:
 		self.__filter.start()
 		self.__action.start()
 		# Restore any previous valid bans from the database
-		for ticket in self.__db.getBans(self, self.__action.getBanTime()):
-			self.__queue.put(ticket)
+		if self.__db is not None:
+			for ticket in self.__db.getBans(self, self.__action.getBanTime()):
+				self.__queue.put(ticket)
 		logSys.info("Jail '%s' started" % self.__name)
 	
 	def stop(self):
diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py
index 5fe9baef..5811dbee 100644
--- a/fail2ban/server/jails.py
+++ b/fail2ban/server/jails.py
@@ -50,13 +50,13 @@ class Jails:
 	# @param name The name of the jail
 	# @param backend The backend to use
 	
-	def add(self, db, name, backend):
+	def add(self, name, backend, db=None):
 		try:
 			self.__lock.acquire()
 			if self.__jails.has_key(name):
 				raise DuplicateJailException(name)
 			else:
-				self.__jails[name] = Jail(db, name, backend)
+				self.__jails[name] = Jail(name, backend, db=None)
 		finally:
 			self.__lock.release()
 	
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 446a0989..572d40ea 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -43,6 +43,7 @@ class Server:
 		self.__loggingLock = Lock()
 		self.__lock = RLock()
 		self.__jails = Jails()
+		self.__db = None
 		self.__daemon = daemon
 		self.__transm = Transmitter(self)
 		self.__asyncServer = AsyncServer(self.__transm)
@@ -51,9 +52,6 @@ class Server:
 		# Set logging level
 		self.setLogLevel(3)
 		self.setLogTarget("STDOUT")
-
-		# Create database, initially in memory
-		self.setDatabase(":memory:")
 	
 	def __sigTERMhandler(self, signum, frame):
 		logSys.debug("Caught signal %d. Exiting" % signum)
@@ -121,12 +119,14 @@ class Server:
 
 	
 	def addJail(self, name, backend):
-		self.__jails.add(self.__db, name, backend)
-		self.__db.addJail(self.__jails.get(name))
+		self.__jails.add(name, backend, self.__db)
+		if self.__db is not None:
+			self.__db.addJail(self.__jails.get(name))
 		
 	def delJail(self, name):
 		self.__jails.remove(name)
-		self.__db.delJailName(name)
+		if self.__db is not None:
+			self.__db.delJailName(name)
 	
 	def startJail(self, name):
 		try:
@@ -468,8 +468,11 @@ class Server:
 	
 	def setDatabase(self, filename):
 		if self.__jails.size() == 0:
-			self.__db = Fail2BanDb(filename)
-			self.__db.delAllJails()
+			if filename.lower() == "none":
+				self.__db = None
+			else:
+				self.__db = Fail2BanDb(filename)
+				self.__db.delAllJails()
 		else:
 			raise RuntimeError(
 				"Cannot change database when there are jails present")
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index b3712796..3e4be6bf 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -116,10 +116,18 @@ class Transmitter:
 		#Database
 		elif name == "dbfile":
 			self.__server.setDatabase(command[1])
-			return self.__server.getDatabase().getFilename()
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getFilename()
 		elif name == "dbpurgeage":
-			self.__server.getDatabase().setPurgeAge(command[1])
-			return self.__server.getDatabase().getPurgeAge()
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				db.setPurgeAge(command[1])
+				return db.getPurgeAge()
 		# Jail
 		elif command[1] == "idle":
 			if command[2] == "on":
@@ -266,9 +274,17 @@ class Transmitter:
 			return self.__server.getLogTarget()
 		#Database
 		elif name == "dbfile":
-			return self.__server.getDatabase().getFilename()
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getFilename()
 		elif name == "dbpurgeage":
-			return self.__server.getDatabase().getPurgeAge()
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getPurgeAge()
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)
diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py
index ef291d71..52e47898 100644
--- a/fail2ban/tests/dummyjail.py
+++ b/fail2ban/tests/dummyjail.py
@@ -24,16 +24,12 @@ __license__ = "GPL"
 
 from threading import Lock
 
-from fail2ban.server.database import Fail2BanDb
-
 class DummyJail(object):
 	"""A simple 'jail' to suck in all the tickets generated by Filter's
 	"""
 	def __init__(self):
 		self.lock = Lock()
 		self.queue = []
-		self.__db = Fail2BanDb(":memory:")
-		self.__db.addJail(self)
 
 	def __len__(self):
 		try:
@@ -69,5 +65,5 @@ class DummyJail(object):
 		return "DummyJail #%s with %d tickets" % (id(self), len(self))
 
 	def getDatabase(self):
-		return self.__db
+		return None
 
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index c41d7ee4..16df110e 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -40,7 +40,6 @@ from fail2ban.server.filter import FileFilter, DNSUtils
 from fail2ban.server.failmanager import FailManager
 from fail2ban.server.failmanager import FailManagerEmpty
 from fail2ban.server.mytime import MyTime
-from fail2ban.server.database import Fail2BanDb
 from fail2ban.tests.utils import setUpMyTime, tearDownMyTime
 
 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
@@ -565,7 +564,7 @@ def get_monitor_failures_testcase(Filter_):
 			# tail written before, so let's not copy anything yet
 			#_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100)
 			# we should detect the failures
-			self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above
+			self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above
 
 			# now copy and get even more
 			_copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100)
@@ -920,5 +919,5 @@ class JailTests(unittest.TestCase):
 	def testSetBackend_gh83(self):
 		# smoke test
 		# Must not fail to initiate
-		jail = Jail(Fail2BanDb(":memory:"), 'test', backend='polling')
+		jail = Jail('test', backend='polling')
 
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index bd8a27d7..4c73ae3f 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -28,7 +28,6 @@ import unittest, socket, time, tempfile, os, locale, sys
 
 from fail2ban.server.server import Server
 from fail2ban.server.jail import Jail
-from fail2ban.server.database import Fail2BanDb
 from fail2ban.exceptions import UnknownJailException
 try:
 	from fail2ban.server import filtersystemd
@@ -175,9 +174,23 @@ class Transmitter(TransmitterBase):
 		self.setGetTestNOK("dbfile", tmpFilename)
 		self.server.delJail(self.jailName)
 		self.setGetTest("dbfile", tmpFilename)
-		self.setGetTest("dbpurgeage", 600)
+		self.setGetTest("dbpurgeage", "600", 600)
 		self.setGetTestNOK("dbpurgeage", "LIZARD")
 
+		# Disable database
+		self.assertEqual(self.transm.proceed(
+			["set", "dbfile", "None"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["get", "dbfile"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["set", "dbpurgeage", "500"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["get", "dbpurgeage"]),
+			(0, None))
+
 	def testAddJail(self):
 		jail2 = "TestJail2"
 		jail3 = "TestJail3"
@@ -651,5 +664,5 @@ class JailTests(unittest.TestCase):
 	def testLongName(self):
 		# Just a smoke test for now
 		longname = "veryveryverylongname"
-		jail = Jail(Fail2BanDb(":memory:"), longname)
+		jail = Jail(longname)
 		self.assertEqual(jail.getName(), longname)

From 00ecd22851836356e26536c5ebb7525746ce4048 Mon Sep 17 00:00:00 2001
From: Steven Hiscocks <steven@hiscocks.me.uk>
Date: Thu, 12 Dec 2013 22:22:30 +0000
Subject: [PATCH 7/7] ENH: Add getBansMerged method to Fail2BanDb

Creates a single ticket for an IP, made up of all previous bans
---
 fail2ban/server/database.py        | 26 +++++++++++++++---
 fail2ban/server/jail.py            |  3 ++-
 fail2ban/tests/databasetestcase.py | 42 ++++++++++++++++++++++++++----
 3 files changed, 61 insertions(+), 10 deletions(-)

diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 073b426c..6b22334e 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -227,24 +227,42 @@ class Fail2BanDb(object):
 					"failures": ticket.getAttempt()}))
 
 	@commitandrollback()
-	def getBans(self, cur, jail=None, bantime=None):
-		query = "SELECT ip, timeofban, data FROM bans"
+	def _getBans(self, cur, jail=None, bantime=None, ip=None):
+		query = "SELECT ip, timeofban, data FROM bans WHERE 1"
 		queryArgs = []
 
 		if jail is not None:
-			query += " WHERE jail=?"
+			query += " AND jail=?"
 			queryArgs.append(jail.getName())
 		if bantime is not None:
 			query += " AND timeofban > ?"
 			queryArgs.append(MyTime.time() - bantime)
+		if ip is not None:
+			query += " AND ip=?"
+			queryArgs.append(ip)
+		query += " ORDER BY timeofban"
 
+		return cur.execute(query, queryArgs)
+
+	def getBans(self, *args, **kwargs):
 		tickets = []
-		for ip, timeofban, data in cur.execute(query, queryArgs):
+		for ip, timeofban, data in self._getBans(*args, **kwargs):
 			#TODO: Implement data parts once arbitrary match keys completed
 			tickets.append(FailTicket(ip, timeofban, data['matches']))
 			tickets[-1].setAttempt(data['failures'])
 		return tickets
 
+	def getBansMerged(self, ip, *args, **kwargs):
+		matches = []
+		failures = 0
+		for ip, timeofban, data in self._getBans(*args, ip=ip, **kwargs):
+			#TODO: Implement data parts once arbitrary match keys completed
+			matches.extend(data['matches'])
+			failures += data['failures']
+		ticket = FailTicket(ip, timeofban, matches)
+		ticket.setAttempt(failures)
+		return ticket
+
 	@commitandrollback()
 	def purge(self, cur):
 		cur.execute(
diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py
index a24810ed..ff780fe9 100644
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@@ -144,7 +144,8 @@ class Jail:
 		self.__action.start()
 		# Restore any previous valid bans from the database
 		if self.__db is not None:
-			for ticket in self.__db.getBans(self, self.__action.getBanTime()):
+			for ticket in self.__db.getBans(
+				jail=self, bantime=self.__action.getBanTime()):
 				self.__queue.put(ticket)
 		logSys.info("Jail '%s' started" % self.__name)
 	
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index e001b865..2c7422b2 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -124,12 +124,43 @@ class DatabaseTest(unittest.TestCase):
 
 	def testAddBan(self):
 		self.testAddJail()
-		ticket = FailTicket("127.0.0.1", 0, [])
+		ticket = FailTicket("127.0.0.1", 0, ["abc\n"])
 		self.db.addBan(self.jail, ticket)
 
 		self.assertEquals(len(self.db.getBans(self.jail)), 1)
 		self.assertTrue(
-			isinstance(self.db.getBans(self.jail)[0], FailTicket))
+			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
+
+	def testGetBansMerged(self):
+		self.testAddJail()
+
+		jail2 = DummyJail()
+		self.db.addJail(jail2)
+
+		ticket = FailTicket("127.0.0.1", 10, ["abc\n"])
+		ticket.setAttempt(10)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.1", 20, ["123\n"])
+		ticket.setAttempt(20)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.2", 30, ["ABC\n"])
+		ticket.setAttempt(30)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.1", 40, ["ABC\n"])
+		ticket.setAttempt(40)
+		self.db.addBan(jail2, ticket)
+
+		# All for IP 127.0.0.1
+		ticket = self.db.getBansMerged("127.0.0.1")
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), 70)
+		self.assertEqual(ticket.getMatches(), ["abc\n", "123\n", "ABC\n"])
+
+		# All for IP 127.0.0.1 for single jail
+		ticket = self.db.getBansMerged("127.0.0.1", jail=self.jail)
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), 30)
+		self.assertEqual(ticket.getMatches(), ["abc\n", "123\n"])
 
 	def testPurge(self):
 		self.testAddJail() # Add jail
@@ -145,12 +176,13 @@ class DatabaseTest(unittest.TestCase):
 		self.db.delJail(self.jail)
 		self.db.purge() # Purge should remove all bans
 		self.assertEqual(len(self.db.getJailNames()), 0)
-		self.assertEqual(len(self.db.getBans(self.jail)), 0)
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 0)
 
 		# Should leave jail
 		self.testAddJail()
-		self.db.addBan(self.jail, FailTicket("127.0.0.1", MyTime.time(), []))
+		self.db.addBan(
+			self.jail, FailTicket("127.0.0.1", MyTime.time(), ["abc\n"]))
 		self.db.delJail(self.jail)
 		self.db.purge() # Should leave jail as ban present
 		self.assertEqual(len(self.db.getJailNames()), 1)
-		self.assertEqual(len(self.db.getBans(self.jail)), 1)
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 1)