github
/
fail2ban
mirror of https://github.com/fail2ban/fail2ban
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #519 from grooverdan/db-migration 

addLog to single SQL statement
pull/525/head
Steven Hiscocks 2013-12-27 13:45:52 -08:00
parent 652260e4eb 41bd0470bd
commit d129321e7b
7 changed files with 116 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST
View File

@ -77,6 +77,7 @@ fail2ban/tests/config/apache-auth/digest_anon/.htaccess
fail2ban/tests/config/apache-auth/digest_anon/.htpasswd
fail2ban/tests/config/apache-auth/README
fail2ban/tests/config/apache-auth/noentry/.htaccess
fail2ban/tests/files/database_v1.db
fail2ban/tests/files/testcase01.log
fail2ban/tests/files/testcase02.log
fail2ban/tests/files/testcase03.log

115
fail2ban/server/database.py
View File

@ -23,6 +23,7 @@ __license__ = "GPL"
import logging
import sys
import shutil, time
import sqlite3
import json
import locale
@ -55,7 +56,40 @@ def commitandrollback(f):
return wrapper
class Fail2BanDb(object):
__version__ = 1
__version__ = 2
# Note all _TABLE_* strings must end in ';' for py26 compatibility
_TABLE_fail2banDb = "CREATE TABLE fail2banDb(version INTEGER);"
_TABLE_jails = "CREATE TABLE jails(" \
"name TEXT NOT NULL UNIQUE, " \
"enabled INTEGER NOT NULL DEFAULT 1" \
");" \
"CREATE INDEX jails_name ON jails(name);"
_TABLE_logs = "CREATE TABLE logs(" \
"jail TEXT NOT NULL, " \
"path TEXT, " \
"firstlinemd5 TEXT, " \
"lastfilepos INTEGER DEFAULT 0, " \
"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, " \
"UNIQUE(jail, path)," \
"UNIQUE(jail, path, firstlinemd5)" \
");" \
"CREATE INDEX logs_path ON logs(path);" \
"CREATE INDEX logs_jail_path ON logs(jail, path);"
#TODO: systemd journal features \
#"journalmatch TEXT, " \
#"journlcursor TEXT, " \
#"lastfiletime INTEGER DEFAULT 0, " # is this easily available \
_TABLE_bans = "CREATE TABLE bans(" \
"jail TEXT NOT NULL, " \
"ip TEXT, " \
"timeofban INTEGER NOT NULL, " \
"data JSON, " \
"FOREIGN KEY(jail) REFERENCES jails(name) " \
");" \
"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban);" \
"CREATE INDEX bans_jail_ip ON bans(jail, ip);" \
"CREATE INDEX bans_ip ON bans(ip);" \
def __init__(self, filename, purgeAge=24*60*60):
try:
self._db = sqlite3.connect(
@ -85,8 +119,15 @@ class Fail2BanDb(object):
else:
version = cur.fetchone()[0]
if version < Fail2BanDb.__version__:
logSys.warning( "Database updated from '%i' to '%i'",
version, self.updateDb(version))
newversion = self.updateDb(version)
if newversion == Fail2BanDb.__version__:
logSys.warning( "Database updated from '%i' to '%i'",
version, newversion)
else:
logSys.error( "Database update failed to acheive version '%i'"
": updated from '%i' to '%i'",
Fail2BanDb.__version__, version, newversion)
raise Exception('Failed to fully update')
finally:
cur.close()
@ -102,53 +143,37 @@ class Fail2BanDb(object):
@commitandrollback
def createDb(self, cur):
# Version info
cur.execute("CREATE TABLE fail2banDb(version INTEGER)")
cur.executescript(Fail2BanDb._TABLE_fail2banDb)
cur.execute("INSERT INTO fail2banDb(version) VALUES(?)",
(Fail2BanDb.__version__, ))
# Jails
cur.execute("CREATE TABLE jails("
"name TEXT NOT NULL UNIQUE, "
"enabled INTEGER NOT NULL DEFAULT 1"
")")
cur.execute("CREATE INDEX jails_name ON jails(name)")
cur.executescript(Fail2BanDb._TABLE_jails)
# Logs
cur.execute("CREATE TABLE logs("
"jail TEXT NOT NULL, "
"path TEXT, "
"firstlinemd5 TEXT, "
#TODO: systemd journal features
#"journalmatch TEXT, "
#"journlcursor TEXT, "
"lastfilepos INTEGER DEFAULT 0, "
#"lastfiletime INTEGER DEFAULT 0, " # is this easily available
"FOREIGN KEY(jail) REFERENCES jails(name) ON DELETE CASCADE, "
"UNIQUE(jail, path)"
")")
cur.execute("CREATE INDEX logs_path ON logs(path)")
cur.execute("CREATE INDEX logs_jail_path ON logs(jail, path)")
cur.executescript(Fail2BanDb._TABLE_logs)
# Bans
cur.execute("CREATE TABLE bans("
"jail TEXT NOT NULL, "
"ip TEXT, "
"timeofban INTEGER NOT NULL, "
"data JSON, "
"FOREIGN KEY(jail) REFERENCES jails(name) "
")")
cur.execute(
"CREATE INDEX bans_jail_timeofban_ip ON bans(jail, timeofban)")
cur.execute("CREATE INDEX bans_jail_ip ON bans(jail, ip)")
cur.execute("CREATE INDEX bans_ip ON bans(ip)")
cur.executescript(Fail2BanDb._TABLE_bans)
cur.execute("SELECT version FROM fail2banDb LIMIT 1")
return cur.fetchone()[0]
@commitandrollback
def updateDb(self, cur, version):
raise NotImplementedError(
"Only single version of database exists...how did you get here??")
self.dbBackupFilename = self._dbFilename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
shutil.copyfile(self._dbFilename, self.dbBackupFilename)
if version > Fail2BanDb.__version__:
raise NotImplementedError(
"Attempt to travel to future version of database ...how did you get here??")
if version < 2:
cur.executescript("BEGIN TRANSACTION;"
"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
"DROP TABLE logs;"
"%s;"
"INSERT INTO logs SELECT * from logs_temp;"
"DROP TABLE logs_temp;"
"UPDATE fail2banDb SET version = 2;"
"COMMIT;" % Fail2BanDb._TABLE_logs)
cur.execute("SELECT version FROM fail2banDb LIMIT 1")
return cur.fetchone()[0]
@ -187,15 +212,15 @@ class Fail2BanDb(object):
try:
firstLineMD5, lastLinePos = cur.fetchone()
except TypeError:
cur.execute(
"INSERT INTO logs(jail, path, firstlinemd5, lastfilepos) "
firstLineMD5 = False
cur.execute(
"INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) "
"VALUES(?, ?, ?, ?)",
(jail.getName(), container.getFileName(),
container.getHash(), container.getPos()))
else:
if container.getHash() != firstLineMD5:
self._updateLog(cur, jail, container)
lastLinePos = None
if container.getHash() != firstLineMD5:
lastLinePos = None
return lastLinePos
@commitandrollback

17
fail2ban/server/ticket.py
View File

@ -46,9 +46,20 @@ class Ticket:
self.__matches = matches or []
def __str__(self):
return "%s: ip=%s time=%s #attempts=%d" % \
(self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt)
return "%s: ip=%s time=%s #attempts=%d matches=%r" % \
(self.__class__.__name__.split('.')[-1], self.__ip, self.__time, self.__attempt, self.__matches)
def __repr__(self):
return str(self)
def __eq__(self, other):
try:
return self.__ip == other.__ip and \
round(self.__time,2) == round(other.__time,2) and \
self.__attempt == other.__attempt and \
self.__matches == other.__matches
except AttributeError:
return False
def setIP(self, value):
if isinstance(value, basestring):

24
fail2ban/tests/databasetestcase.py
View File

@ -26,6 +26,7 @@ import os
import unittest
import tempfile
import sqlite3
import shutil
from fail2ban.server.database import Fail2BanDb
from fail2ban.server.filter import FileContainer
@ -45,7 +46,7 @@ class DatabaseTest(unittest.TestCase):
# Cleanup
os.remove(self.dbFilename)
def getFilename(self):
def testGetFilename(self):
self.assertEqual(self.dbFilename, self.db.getFilename())
def testCreateInvalidPath(self):
@ -64,8 +65,16 @@ class DatabaseTest(unittest.TestCase):
"Jail not retained in Db after disconnect reconnect.")
def testUpdateDb(self):
# TODO: Currently only single version exists
pass
shutil.copyfile('fail2ban/tests/files/database_v1.db', self.dbFilename)
self.db = Fail2BanDb(self.dbFilename)
self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets']))
self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log']))
ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"])
self.assertEqual(self.db.getBans()[0], ticket)
self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__)
self.assertRaises(NotImplementedError, self.db.updateDb, Fail2BanDb.__version__ + 1)
os.remove(self.db.dbBackupFilename)
def testAddJail(self):
self.jail = DummyJail()
@ -83,6 +92,7 @@ class DatabaseTest(unittest.TestCase):
self.db.addLog(self.jail, self.fileContainer)
self.assertTrue(filename in self.db.getLogPaths(self.jail))
os.remove(filename)
def testUpdateLog(self):
self.testAddLog() # Add log file
@ -121,6 +131,7 @@ class DatabaseTest(unittest.TestCase):
# last position in file
self.assertEqual(
self.db.addLog(self.jail, self.fileContainer), None)
os.remove(filename)
def testAddBan(self):
self.testAddJail()
@ -131,6 +142,13 @@ class DatabaseTest(unittest.TestCase):
self.assertTrue(
isinstance(self.db.getBans(jail=self.jail)[0], FailTicket))
def testGetBansWithTime(self):
self.testAddJail()
ticket = FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"])
self.db.addBan(self.jail, ticket)
self.assertEquals(len(self.db.getBans(jail=self.jail,bantime=50)), 1)
self.assertEquals(len(self.db.getBans(jail=self.jail,bantime=20)), 0)
def testGetBansMerged(self):
self.testAddJail()

9
fail2ban/tests/failmanagertestcase.py
View File

@ -93,16 +93,21 @@ class AddFailure(unittest.TestCase):
# finish with rudimentary tests of the ticket
# verify consistent str
ticket_str = str(ticket)
ticket_repr = repr(ticket)
self.assertEqual(
ticket_str,
'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5')
'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5 matches=[]')
self.assertEqual(
ticket_repr,
'FailTicket: ip=193.168.0.128 time=1167605999.0 #attempts=5 matches=[]')
self.assertFalse(ticket == False)
# and some get/set-ers otherwise not tested
ticket.setTime(1000002000.0)
self.assertEqual(ticket.getTime(), 1000002000.0)
# and str() adjusted correspondingly
self.assertEqual(
str(ticket),
'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5')
'FailTicket: ip=193.168.0.128 time=1000002000.0 #attempts=5 matches=[]')
def testbanNOK(self):
self.__failManager.setMaxRetry(10)

BIN
fail2ban/tests/files/database_v1.db Normal file
View File

Binary file not shown.

6
fail2ban/tests/servertestcase.py
View File

@ -670,14 +670,14 @@ class TransmitterLogging(TransmitterBase):
self.server.setLogLevel(2)
self.assertEqual(self.transm.proceed(["set", "logtarget", fn]), (0, fn))
l = logging.getLogger('fail2ban.server.server').parent.parent
l.warn("Before file moved")
l.warning("Before file moved")
try:
f2, fn2 = tempfile.mkstemp("fail2ban.log")
os.close(f2)
os.rename(fn, fn2)
l.warn("After file moved")
l.warning("After file moved")
self.assertEqual(self.transm.proceed(["flushlogs"]), (0, "rolled over"))
l.warn("After flushlogs")
l.warning("After flushlogs")
with open(fn2,'r') as f:
line1 = f.next()
if line1.find('Changed logging target to') >= 0: