MRG: branch 'kwirk/database' into 0.9 - gh-480

Conflicts:
	fail2ban/tests/utils.py
        - Another test suite added in separate commit e09b700

config/fail2ban.conf

@@ -47,3 +47,15 @@ socket = /var/run/fail2ban/fail2ban.sock
 #
 pidfile = /var/run/fail2ban/fail2ban.pid
 
+# Options: dbfile
+# Notes.: Set the file for the fail2ban persistent data to be stored.
+#         A value of ":memory:" means database is only stored in memory 
+#         and data is lost once fail2ban is stops.
+#         A value of "None" disables the database.
+# Values: [ None :memory: FILE ] Default: /var/lib/fail2ban/fail2ban.sqlite3
+dbfile = /var/lib/fail2ban/fail2ban.sqlite3
+
+# Options: dbpurgeage
+# Notes.: Sets age at which bans should be purged from the database
+# Values: [ SECONDS ] Default: 86400 (24hours)
+dbpurgeage = 86400

fail2ban/client/beautifier.py

@@ -102,6 +102,18 @@ class Beautifier:
 					msg = msg + "DEBUG"
 				else:
 					msg = msg + `response`
+			elif inC[1] == "dbfile":
+				if response is None:
+					msg = "Database currently disabled"
+				else:
+					msg = "Current database file is:\n"
+					msg = msg + "`- " + response
+			elif inC[1] == "dbpurgeage":
+				if response is None:
+					msg = "Database currently disabled"
+				else:
+					msg = "Current database purge age is:\n"
+					msg = msg + "`- %iseconds" % response
 			elif inC[2] in ("logpath", "addlogpath", "dellogpath"):
 				if len(response) == 0:
 					msg = "No file is currently monitored"

fail2ban/client/fail2banreader.py

@@ -45,7 +45,9 @@ class Fail2banReader(ConfigReader):
 	
 	def getOptions(self):
 		opts = [["int", "loglevel", 1],
-				["string", "logtarget", "STDERR"]]
+				["string", "logtarget", "STDERR"],
+				["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"],
+				["int", "dbpurgeage", 86400]]
 		self.__opts = ConfigReader.getOptions(self, "Definition", opts)
 	
 	def convert(self):
@@ -55,5 +57,9 @@ class Fail2banReader(ConfigReader):
 				stream.append(["set", "loglevel", self.__opts[opt]])
 			elif opt == "logtarget":
 				stream.append(["set", "logtarget", self.__opts[opt]])
+			elif opt == "dbfile":
+				stream.append(["set", "dbfile", self.__opts[opt]])
+			elif opt == "dbpurgeage":
+				stream.append(["set", "dbpurgeage", self.__opts[opt]])
 		return stream
 	

fail2ban/protocol.py

@@ -43,6 +43,11 @@ protocol = [
 ["get loglevel", "gets the logging level"], 
 ["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG or a file"], 
 ["get logtarget", "gets logging target"], 
+['', "DATABASE", ""],
+["set dbfile <FILE>", "set the location of fail2ban persistent datastore. Set to \"None\" to disable"], 
+["get dbfile", "get the location of fail2ban persistent datastore"], 
+["set dbpurgeage <SECONDS>", "sets the max age in <SECONDS> that history of bans will be kept"], 
+["get dbpurgeage", "gets the max age in seconds that history of bans will be kept"], 
 ['', "JAIL CONTROL", ""],
 ["add <JAIL> <BACKEND>", "creates <JAIL> using <BACKEND>"], 
 ["start <JAIL>", "starts the jail <JAIL>"], 

fail2ban/server/database.py

@@ -0,0 +1,274 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+__author__ = "Steven Hiscocks"
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import logging
+import sys
+import sqlite3
+import json
+import locale
+
+from fail2ban.server.mytime import MyTime
+from fail2ban.server.ticket import FailTicket
+
+# Gets the instance of the logger.
+logSys = logging.getLogger(__name__)
+
+if sys.version_info >= (3,):
+	sqlite3.register_adapter(
+		dict,
+		lambda x: json.dumps(x, ensure_ascii=False).encode(
+			locale.getpreferredencoding(), 'replace'))
+	sqlite3.register_converter(
+		"JSON",
+		lambda x: json.loads(x.decode(
+			locale.getpreferredencoding(), 'replace')))
+else:
+	sqlite3.register_adapter(dict, json.dumps)
+	sqlite3.register_converter("JSON", json.loads)
+
+def commitandrollback():
+	def wrap(f):
+		def func(self, *args, **kw):
+			with self._db: # Auto commit and rollback on exception
+				return f(self, self._db.cursor(), *args, **kw)
+		return func
+	return wrap
+
+class Fail2BanDb(object):
+	__version__ = 1
+	def __init__(self, filename, purgeAge=24*60*60):
+		try:
+			self._db = sqlite3.connect(
+				filename, check_same_thread=False,
+				detect_types=sqlite3.PARSE_DECLTYPES)
+			self._dbFilename = filename
+			self._purgeAge = purgeAge
+
+			logSys.info(
+				"Connected to fail2ban persistent database '%s'", filename)
+		except sqlite3.OperationalError, e:
+			logSys.error(
+				"Error connecting to fail2ban persistent database '%s': %s",
+				filename, e.args[0])
+			raise
+
+		cur = self._db.cursor()
+		cur.execute("PRAGMA foreign_keys = ON;")
+
+		try:
+			cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		except sqlite3.OperationalError:
+			logSys.warning("New database created. Version '%i'",
+				self.createDb())
+		else:
+			version = cur.fetchone()[0]
+			if version < Fail2BanDb.__version__:
+				logSys.warning( "Database updated from '%i' to '%i'",
+					version, self.updateDb(version))
+		finally:
+			cur.close()
+
+	def getFilename(self):
+		return self._dbFilename
+
+	def getPurgeAge(self):
+		return self._purgeAge
+
+	def setPurgeAge(self, value):
+		self._purgeAge = int(value)
+
+	@commitandrollback()
+	def createDb(self, cur):
+		# Version info
+		cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
+		cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
+			(Fail2BanDb.__version__, ))
+
+		# Jails
+		cur.execute("CREATE TABLE jails("
+			"name TEXT NOT NULL UNIQUE, "
+			"enabled INTEGER NOT NULL DEFAULT 1"
+			")")
+		cur.execute("CREATE INDEX jails_name ON jails(name)")
+
+		# Logs
+		cur.execute("CREATE TABLE logs("
+			"jail TEXT NOT NULL, "
+			"path TEXT, "
+			"firstlinemd5 TEXT, "
+			#TODO: systemd journal features
+			#"journalmatch TEXT, "
+			#"journlcursor TEXT, "
+			"lastfilepos INTEGER DEFAULT 0, "
+			#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
+			"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, "
+			"UNIQUE(jail, path)"
+			")")
+		cur.execute("CREATE INDEX logs_path ON logs(path)")
+		cur.execute("CREATE INDEX logs_jail_path ON logs(jail, path)")
+
+		# Bans
+		cur.execute("CREATE TABLE bans("
+			"jail TEXT NOT NULL, "
+			"ip TEXT, "
+			"timeofban INTEGER NOT NULL, "
+			"data JSON, "
+			"FOREIGN KEY(jail) REFERENCES jails(name) "
+			")")
+		cur.execute(
+			"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban)")
+		cur.execute("CREATE INDEX bans_jail_ip ON bans(jail, ip)")
+		cur.execute("CREATE INDEX bans_ip ON bans(ip)")
+
+		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		return cur.fetchone()[0]
+
+	@commitandrollback()
+	def updateDb(self, cur, version):
+		raise NotImplementedError(
+			"Only single version of database exists...how did you get here??")
+		cur.execute("SELECT version FROM fail2banDb LIMIT 1")
+		return cur.fetchone()[0]
+
+	@commitandrollback()
+	def addJail(self, cur, jail):
+		cur.execute(
+			"INSERT OR REPLACE INTO jails(name, enabled) VALUES(?, 1)",
+			(jail.getName(),))
+
+	def delJail(self, jail):
+		return self.delJailName(jail.getName())
+
+	@commitandrollback()
+	def delJailName(self, cur, name):
+		# Will be deleted by purge as appropriate
+		cur.execute(
+			"UPDATE jails SET enabled=0 WHERE name=?", (name, ))
+
+	@commitandrollback()
+	def delAllJails(self, cur):
+		# Will be deleted by purge as appropriate
+		cur.execute("UPDATE jails SET enabled=0")
+
+	@commitandrollback()
+	def getJailNames(self, cur):
+		cur.execute("SELECT name FROM jails")
+		return set(row[0] for row in cur.fetchmany())
+
+	@commitandrollback()
+	def addLog(self, cur, jail, container):
+		lastLinePos = None
+		cur.execute(
+			"SELECT firstlinemd5, lastfilepos FROM logs "
+				"WHERE jail=? AND path=?",
+			(jail.getName(), container.getFileName()))
+		try:
+			firstLineMD5, lastLinePos = cur.fetchone()
+		except TypeError:
+			cur.execute(
+				"INSERT INTO logs(jail, path, firstlinemd5, lastfilepos) "
+					"VALUES(?, ?, ?, ?)",
+				(jail.getName(), container.getFileName(),
+					container.getHash(), container.getPos()))
+		else:
+			if container.getHash() != firstLineMD5:
+				self._updateLog(cur, jail, container)
+				lastLinePos = None
+		return lastLinePos
+
+	@commitandrollback()
+	def getLogPaths(self, cur, jail=None):
+		query = "SELECT path FROM logs"
+		queryArgs = []
+		if jail is not None:
+			query += " WHERE jail=?"
+			queryArgs.append(jail.getName())
+		cur.execute(query, queryArgs)
+		return set(row[0] for row in cur.fetchmany())
+
+	@commitandrollback()
+	def updateLog(self, cur, *args, **kwargs):
+		self._updateLog(cur, *args, **kwargs)
+
+	def _updateLog(self, cur, jail, container):
+		cur.execute(
+			"UPDATE logs SET firstlinemd5=?, lastfilepos=? "
+				"WHERE jail=? AND path=?",
+			(container.getHash(), container.getPos(),
+				jail.getName(), container.getFileName()))
+
+	@commitandrollback()
+	def addBan(self, cur, jail, ticket):
+		#TODO: Implement data parts once arbitrary match keys completed
+		cur.execute(
+			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
+			(jail.getName(), ticket.getIP(), ticket.getTime(),
+				{"matches": ticket.getMatches(),
+					"failures": ticket.getAttempt()}))
+
+	@commitandrollback()
+	def _getBans(self, cur, jail=None, bantime=None, ip=None):
+		query = "SELECT ip, timeofban, data FROM bans WHERE 1"
+		queryArgs = []
+
+		if jail is not None:
+			query += " AND jail=?"
+			queryArgs.append(jail.getName())
+		if bantime is not None:
+			query += " AND timeofban > ?"
+			queryArgs.append(MyTime.time() - bantime)
+		if ip is not None:
+			query += " AND ip=?"
+			queryArgs.append(ip)
+		query += " ORDER BY timeofban"
+
+		return cur.execute(query, queryArgs)
+
+	def getBans(self, *args, **kwargs):
+		tickets = []
+		for ip, timeofban, data in self._getBans(*args, **kwargs):
+			#TODO: Implement data parts once arbitrary match keys completed
+			tickets.append(FailTicket(ip, timeofban, data['matches']))
+			tickets[-1].setAttempt(data['failures'])
+		return tickets
+
+	def getBansMerged(self, ip, *args, **kwargs):
+		matches = []
+		failures = 0
+		for ip, timeofban, data in self._getBans(*args, ip=ip, **kwargs):
+			#TODO: Implement data parts once arbitrary match keys completed
+			matches.extend(data['matches'])
+			failures += data['failures']
+		ticket = FailTicket(ip, timeofban, matches)
+		ticket.setAttempt(failures)
+		return ticket
+
+	@commitandrollback()
+	def purge(self, cur):
+		cur.execute(
+			"DELETE FROM bans WHERE timeofban < ?",
+			(MyTime.time() - self._purgeAge, ))
+		cur.execute(
+			"DELETE FROM jails WHERE enabled = 0 "
+				"AND NOT EXISTS(SELECT * FROM bans WHERE jail = jails.name)")
+

fail2ban/server/filter.py

@@ -530,6 +530,11 @@ class FileFilter(Filter):
 			logSys.error(path + " already exists")
 		else:
 			container = FileContainer(path, self.getLogEncoding(), tail)
+			db = self.jail.getDatabase()
+			if db is not None:
+				lastpos = db.addLog(self.jail, container)
+				if lastpos and not tail:
+					container.setPos(lastpos)
 			self.__logPath.append(container)
 			logSys.info("Added logfile = %s" % path)
 			self._addLogPath(path)			# backend specific
@@ -549,6 +554,9 @@ class FileFilter(Filter):
 		for log in self.__logPath:
 			if log.getFileName() == path:
 				self.__logPath.remove(log)
+				db = self.jail.getDatabase()
+				if db is not None:
+					db.updateLog(self.jail, log)
 				logSys.info("Removed logfile = %s" % path)
 				self._delLogPath(path)
 				return
@@ -647,6 +655,9 @@ class FileFilter(Filter):
 				break
 			self.processLineAndAdd(line)
 		container.close()
+		db = self.jail.getDatabase()
+		if db is not None:
+			db.updateLog(self.jail, container)
 		return True
 
 	def status(self):
@@ -685,7 +696,7 @@ class FileContainer:
 		try:
 			firstLine = handler.readline()
 			# Computes the MD5 of the first line.
-			self.__hash = md5sum(firstLine).digest()
+			self.__hash = md5sum(firstLine).hexdigest()
 			# Start at the beginning of file if tail mode is off.
 			if tail:
 				handler.seek(0, 2)
@@ -705,6 +716,15 @@ class FileContainer:
 	def getEncoding(self):
 		return self.__encoding
 
+	def getHash(self):
+		return self.__hash
+
+	def getPos(self):
+		return self.__pos
+
+	def setPos(self, value):
+		self.__pos = value
+
 	def open(self):
 		self.__handler = open(self.__filename, 'rb')
 		# Set the file descriptor to be FD_CLOEXEC
@@ -720,7 +740,7 @@ class FileContainer:
 			return False
 		firstLine = self.__handler.readline()
 		# Computes the MD5 of the first line.
-		myHash = md5sum(firstLine).digest()
+		myHash = md5sum(firstLine).hexdigest()
 		## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % (
 		## 	self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos,
 		## 	self.__hash != myHash or self.__ino != stats.st_ino)

fail2ban/server/jail.py

@@ -37,7 +37,8 @@ class Jail:
 	#      list had .index until 2.6
 	_BACKENDS = ['pyinotify', 'gamin', 'polling', 'systemd']
 
-	def __init__(self, name, backend = "auto"):
+	def __init__(self, name, backend = "auto", db=None):
+		self.__db = db
 		self.setName(name)
 		self.__queue = Queue.Queue()
 		self.__filter = None
@@ -118,6 +119,9 @@ class Jail:
 	def getName(self):
 		return self.__name
 	
+	def getDatabase(self):
+		return self.__db
+	
 	def getFilter(self):
 		return self.__filter
 	
@@ -126,6 +130,8 @@ class Jail:
 	
 	def putFailTicket(self, ticket):
 		self.__queue.put(ticket)
+		if self.__db is not None:
+			self.__db.addBan(self, ticket)
 	
 	def getFailTicket(self):
 		try:
@@ -136,6 +142,11 @@ class Jail:
 	def start(self):
 		self.__filter.start()
 		self.__action.start()
+		# Restore any previous valid bans from the database
+		if self.__db is not None:
+			for ticket in self.__db.getBans(
+				jail=self, bantime=self.__action.getBanTime()):
+				self.__queue.put(ticket)
 		logSys.info("Jail '%s' started" % self.__name)
 	
 	def stop(self):

fail2ban/server/jails.py

@@ -50,13 +50,13 @@ class Jails:
 	# @param name The name of the jail
 	# @param backend The backend to use
 	
-	def add(self, name, backend):
+	def add(self, name, backend, db=None):
 		try:
 			self.__lock.acquire()
 			if self.__jails.has_key(name):
 				raise DuplicateJailException(name)
 			else:
-				self.__jails[name] = Jail(name, backend)
+				self.__jails[name] = Jail(name, backend, db=None)
 		finally:
 			self.__lock.release()
 	

fail2ban/server/server.py

@@ -30,6 +30,7 @@ from filter import FileFilter, JournalFilter
 from transmitter import Transmitter
 from asyncserver import AsyncServer
 from asyncserver import AsyncServerException
+from database import Fail2BanDb
 from fail2ban import version
 import logging, logging.handlers, sys, os, signal
 
@@ -42,6 +43,7 @@ class Server:
 		self.__loggingLock = Lock()
 		self.__lock = RLock()
 		self.__jails = Jails()
+		self.__db = None
 		self.__daemon = daemon
 		self.__transm = Transmitter(self)
 		self.__asyncServer = AsyncServer(self.__transm)
@@ -117,10 +119,14 @@ class Server:
 
 	
 	def addJail(self, name, backend):
-		self.__jails.add(name, backend)
+		self.__jails.add(name, backend, self.__db)
+		if self.__db is not None:
+			self.__db.addJail(self.__jails.get(name))
 		
 	def delJail(self, name):
 		self.__jails.remove(name)
+		if self.__db is not None:
+			self.__db.delJailName(name)
 	
 	def startJail(self, name):
 		try:
@@ -460,6 +466,20 @@ class Server:
 		finally:
 			self.__loggingLock.release()
 	
+	def setDatabase(self, filename):
+		if self.__jails.size() == 0:
+			if filename.lower() == "none":
+				self.__db = None
+			else:
+				self.__db = Fail2BanDb(filename)
+				self.__db.delAllJails()
+		else:
+			raise RuntimeError(
+				"Cannot change database when there are jails present")
+	
+	def getDatabase(self):
+		return self.__db
+	
 	def __createDaemon(self): # pragma: no cover
 		""" Detach a process from the controlling terminal and run it in the
 			background as a daemon.

fail2ban/server/transmitter.py

@@ -113,6 +113,21 @@ class Transmitter:
 				return self.__server.getLogTarget()
 			else:
 				raise Exception("Failed to change log target")
+		#Database
+		elif name == "dbfile":
+			self.__server.setDatabase(command[1])
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getFilename()
+		elif name == "dbpurgeage":
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				db.setPurgeAge(command[1])
+				return db.getPurgeAge()
 		# Jail
 		elif command[1] == "idle":
 			if command[2] == "on":
@@ -257,6 +272,19 @@ class Transmitter:
 			return self.__server.getLogLevel()
 		elif name == "logtarget":
 			return self.__server.getLogTarget()
+		#Database
+		elif name == "dbfile":
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getFilename()
+		elif name == "dbpurgeage":
+			db = self.__server.getDatabase()
+			if db is None:
+				return None
+			else:
+				return db.getPurgeAge()
 		# Filter
 		elif command[1] == "logpath":
 			return self.__server.getLogPath(name)

fail2ban/tests/clientreadertestcase.py

@@ -360,7 +360,10 @@ class JailsReaderTest(unittest.TestCase):
 		# and there is logging information left to be passed into the
 		# server
 		self.assertEqual(sorted(commands),
-						 [['set', 'loglevel', 3],
+						 [['set', 'dbfile',
+							'/var/lib/fail2ban/fail2ban.sqlite3'],
+						  ['set', 'dbpurgeage', 86400],
+						  ['set', 'loglevel', 3],
 						  ['set', 'logtarget', '/var/log/fail2ban.log']])
 
 		# and if we force change configurator's fail2ban's baseDir

fail2ban/tests/databasetestcase.py

@@ -0,0 +1,188 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Fail2Ban developers
+
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import os
+import unittest
+import tempfile
+import sqlite3
+
+from fail2ban.server.database import Fail2BanDb
+from fail2ban.server.filter import FileContainer
+from fail2ban.server.mytime import MyTime
+from fail2ban.server.ticket import FailTicket
+from fail2ban.tests.dummyjail import DummyJail
+
+class DatabaseTest(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
+		self.db = Fail2BanDb(self.dbFilename)
+
+	def tearDown(self):
+		"""Call after every test case."""
+		# Cleanup
+		os.remove(self.dbFilename)
+
+	def getFilename(self):
+		self.assertEqual(self.dbFilename, self.db.getFilename())
+
+	def testCreateInvalidPath(self):
+		self.assertRaises(
+			sqlite3.OperationalError,
+			Fail2BanDb,
+			"/this/path/should/not/exist")
+
+	def testCreateAndReconnect(self):
+		self.testAddJail()
+		# Reconnect...
+		self.db = Fail2BanDb(self.dbFilename)
+		# and check jail of same name still present
+		self.assertTrue(
+			self.jail.getName() in self.db.getJailNames(),
+			"Jail not retained in Db after disconnect reconnect.")
+
+	def testUpdateDb(self):
+		# TODO: Currently only single version exists
+		pass
+
+	def testAddJail(self):
+		self.jail = DummyJail()
+		self.db.addJail(self.jail)
+		self.assertTrue(
+			self.jail.getName() in self.db.getJailNames(),
+			"Jail not added to database")
+
+	def testAddLog(self):
+		self.testAddJail() # Jail required
+
+		_, filename = tempfile.mkstemp(".log", "Fail2BanDb_")
+		self.fileContainer = FileContainer(filename, "utf-8")
+
+		self.db.addLog(self.jail, self.fileContainer)
+
+		self.assertTrue(filename in self.db.getLogPaths(self.jail))
+
+	def testUpdateLog(self):
+		self.testAddLog() # Add log file
+
+		# Write some text
+		filename = self.fileContainer.getFileName()
+		file_ = open(filename, "w")
+		file_.write("Some text to write which will change md5sum\n")
+		file_.close()
+		self.fileContainer.open()
+		self.fileContainer.readline()
+		self.fileContainer.close()
+
+		# Capture position which should be after line just written
+		lastPos = self.fileContainer.getPos()
+		self.assertTrue(lastPos > 0)
+		self.db.updateLog(self.jail, self.fileContainer)
+
+		# New FileContainer for file
+		self.fileContainer = FileContainer(filename, "utf-8")
+		self.assertEqual(self.fileContainer.getPos(), 0)
+
+		# Database should return previous position in file
+		self.assertEqual(
+			self.db.addLog(self.jail, self.fileContainer), lastPos)
+
+		# Change md5sum
+		file_ = open(filename, "w") # Truncate
+		file_.write("Some different text to change md5sum\n")
+		file_.close()
+
+		self.fileContainer = FileContainer(filename, "utf-8")
+		self.assertEqual(self.fileContainer.getPos(), 0)
+
+		# Database should be aware of md5sum change, such doesn't return
+		# last position in file
+		self.assertEqual(
+			self.db.addLog(self.jail, self.fileContainer), None)
+
+	def testAddBan(self):
+		self.testAddJail()
+		ticket = FailTicket("127.0.0.1", 0, ["abc\n"])
+		self.db.addBan(self.jail, ticket)
+
+		self.assertEquals(len(self.db.getBans(self.jail)), 1)
+		self.assertTrue(
+			isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
+
+	def testGetBansMerged(self):
+		self.testAddJail()
+
+		jail2 = DummyJail()
+		self.db.addJail(jail2)
+
+		ticket = FailTicket("127.0.0.1", 10, ["abc\n"])
+		ticket.setAttempt(10)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.1", 20, ["123\n"])
+		ticket.setAttempt(20)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.2", 30, ["ABC\n"])
+		ticket.setAttempt(30)
+		self.db.addBan(self.jail, ticket)
+		ticket = FailTicket("127.0.0.1", 40, ["ABC\n"])
+		ticket.setAttempt(40)
+		self.db.addBan(jail2, ticket)
+
+		# All for IP 127.0.0.1
+		ticket = self.db.getBansMerged("127.0.0.1")
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), 70)
+		self.assertEqual(ticket.getMatches(), ["abc\n", "123\n", "ABC\n"])
+
+		# All for IP 127.0.0.1 for single jail
+		ticket = self.db.getBansMerged("127.0.0.1", jail=self.jail)
+		self.assertEqual(ticket.getIP(), "127.0.0.1")
+		self.assertEqual(ticket.getAttempt(), 30)
+		self.assertEqual(ticket.getMatches(), ["abc\n", "123\n"])
+
+	def testPurge(self):
+		self.testAddJail() # Add jail
+
+		self.db.purge() # Jail enabled by default so shouldn't be purged
+		self.assertEqual(len(self.db.getJailNames()), 1)
+
+		self.db.delJail(self.jail)
+		self.db.purge() # Should remove jail
+		self.assertEqual(len(self.db.getJailNames()), 0)
+
+		self.testAddBan()
+		self.db.delJail(self.jail)
+		self.db.purge() # Purge should remove all bans
+		self.assertEqual(len(self.db.getJailNames()), 0)
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 0)
+
+		# Should leave jail
+		self.testAddJail()
+		self.db.addBan(
+			self.jail, FailTicket("127.0.0.1", MyTime.time(), ["abc\n"]))
+		self.db.delJail(self.jail)
+		self.db.purge() # Should leave jail as ban present
+		self.assertEqual(len(self.db.getJailNames()), 1)
+		self.assertEqual(len(self.db.getBans(jail=self.jail)), 1)

fail2ban/tests/dummyjail.py

@@ -23,6 +23,7 @@ __copyright__ = "Copyright (c) 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
 from threading import Lock
+
 class DummyJail(object):
 	"""A simple 'jail' to suck in all the tickets generated by Filter's
 	"""
@@ -54,6 +55,15 @@ class DummyJail(object):
 		finally:
 			self.lock.release()
 
+	def setIdle(self, value):
+		pass
+
+	def getIdle(self):
+		pass
+
 	def getName(self):
 		return "DummyJail #%s with %d tickets" % (id(self), len(self))
 
+	def getDatabase(self):
+		return None
+

fail2ban/tests/filtertestcase.py

@@ -228,7 +228,7 @@ class LogFile(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
-		self.filter = FilterPoll(None)
+		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(LogFile.FILENAME)
 
 	def tearDown(self):
@@ -251,7 +251,7 @@ class LogFileMonitor(unittest.TestCase):
 		self.filter = self.name = 'NA'
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
 		self.file = open(self.name, 'a')
-		self.filter = FilterPoll(None)
+		self.filter = FilterPoll(DummyJail())
 		self.filter.addLogPath(self.name)
 		self.filter.setActive(True)
 		self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
@@ -715,7 +715,8 @@ class GetFailures(unittest.TestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		setUpMyTime()
-		self.filter = FileFilter(None)
+		self.jail = DummyJail()
+		self.filter = FileFilter(self.jail)
 		self.filter.setActive(True)
 		# TODO Test this
 		#self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
@@ -799,7 +800,8 @@ class GetFailures(unittest.TestCase):
 		for useDns, output in (('yes',  output_yes),
 							   ('no',   output_no),
 							   ('warn', output_yes)):
-			filter_ = FileFilter(None, useDns=useDns)
+			jail = DummyJail()
+			filter_ = FileFilter(jail, useDns=useDns)
 			filter_.setActive(True)
 			filter_.failManager.setMaxRetry(1)	# we might have just few failures
 
@@ -913,5 +915,6 @@ class JailTests(unittest.TestCase):
 
 	def testSetBackend_gh83(self):
 		# smoke test
-		jail = Jail('test', backend='polling') # Must not fail to initiate
+		# Must not fail to initiate
+		jail = Jail('test', backend='polling')
 

fail2ban/tests/servertestcase.py

@@ -168,6 +168,29 @@ class Transmitter(TransmitterBase):
 		# Approx 1 second delay
 		self.assertAlmostEqual(t1 - t0, 1, places=2)
 
+	def testDatabase(self):
+		_, tmpFilename = tempfile.mkstemp(".db", "Fail2Ban_")
+		# Jails present, cant change database
+		self.setGetTestNOK("dbfile", tmpFilename)
+		self.server.delJail(self.jailName)
+		self.setGetTest("dbfile", tmpFilename)
+		self.setGetTest("dbpurgeage", "600", 600)
+		self.setGetTestNOK("dbpurgeage", "LIZARD")
+
+		# Disable database
+		self.assertEqual(self.transm.proceed(
+			["set", "dbfile", "None"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["get", "dbfile"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["set", "dbpurgeage", "500"]),
+			(0, None))
+		self.assertEqual(self.transm.proceed(
+			["get", "dbpurgeage"]),
+			(0, None))
+
 	def testAddJail(self):
 		jail2 = "TestJail2"
 		jail3 = "TestJail3"

fail2ban/tests/utils.py

@@ -146,6 +146,7 @@ def gatherTests(regexps=None, no_network=False):
 	from fail2ban.tests import actiontestcase
 	from fail2ban.tests import sockettestcase
 	from fail2ban.tests import misctestcase
+	from fail2ban.tests import databasetestcase
 	if json:
 		from fail2ban.tests import samplestestcase
 
@@ -185,6 +186,8 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(misctestcase.SetupTest))
 	tests.addTest(unittest.makeSuite(misctestcase.TestsUtilsTest))
 	tests.addTest(unittest.makeSuite(misctestcase.CustomDateFormatsTest))
+	# Database
+	tests.addTest(unittest.makeSuite(databasetestcase.DatabaseTest))
 
 	# Filter
 	if not no_network:

setup.py

@@ -131,6 +131,9 @@ setup(
 						('/var/run/fail2ban',
 							''
 						),
+						('/var/lib/fail2ban',
+							''
+						),
 						('/usr/share/doc/fail2ban',
 							['README.md', 'DEVELOP', 'doc/run-rootless.txt']
 						)