Merge pull request #1436 from sebres/speedup-db

extremely speedup of all database operations
pull/1439/head
Serg G. Brester 2016-05-21 13:39:48 +02:00
commit b56f4c533e
2 changed files with 28 additions and 4 deletions

View File

@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
* New Actions: * New Actions:
- action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367) - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
- Enhancements: - Enhancements:
* Extreme speedup of all sqlite database operations (gh-1436),
by using of following sqlite options:
- (synchronous = OFF) write data through OS without syncing
- (journal_mode = MEMORY) use memory for the transaction logging
- (temp_store = MEMORY) temporary tables and indices are kept in memory
* journald journalmatch for pure-ftpd (gh-1362) * journald journalmatch for pure-ftpd (gh-1362)
* Add additional regex filter for dovecot ldap authentication failures (gh-1370) * Add additional regex filter for dovecot ldap authentication failures (gh-1370)
* added additional regex filters for exim (gh-1371) * added additional regex filters for exim (gh-1371)

View File

@ -181,8 +181,23 @@ class Fail2BanDb(object):
filename, e.args[0]) filename, e.args[0])
raise raise
# differentiate pypy: switch journal mode later (save it during the upgrade),
# to prevent errors like "database table is locked":
try:
import __pypy__
pypy = True
except ImportError:
pypy = False
cur = self._db.cursor() cur = self._db.cursor()
cur.execute("PRAGMA foreign_keys = ON;") cur.execute("PRAGMA foreign_keys = ON")
# speedup: write data through OS without syncing (no wait):
cur.execute("PRAGMA synchronous = OFF")
# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
if not pypy:
cur.execute("PRAGMA journal_mode = MEMORY")
# speedup: temporary tables and indices are kept in memory:
cur.execute("PRAGMA temp_store = MEMORY")
try: try:
cur.execute("SELECT version FROM fail2banDb LIMIT 1") cur.execute("SELECT version FROM fail2banDb LIMIT 1")
@ -202,6 +217,9 @@ class Fail2BanDb(object):
Fail2BanDb.__version__, version, newversion) Fail2BanDb.__version__, version, newversion)
raise RuntimeError('Failed to fully update') raise RuntimeError('Failed to fully update')
finally: finally:
# pypy: set journal mode after possible upgrade db:
if pypy:
cur.execute("PRAGMA journal_mode = MEMORY")
cur.close() cur.close()
@property @property
@ -244,13 +262,14 @@ class Fail2BanDb(object):
A timestamped backup is also created prior to attempting the update. A timestamped backup is also created prior to attempting the update.
""" """
self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
shutil.copyfile(self.filename, self._dbBackupFilename)
logSys.info("Database backup created: %s", self._dbBackupFilename)
if version > Fail2BanDb.__version__: if version > Fail2BanDb.__version__:
raise NotImplementedError( raise NotImplementedError(
"Attempt to travel to future version of database ...how did you get here??") "Attempt to travel to future version of database ...how did you get here??")
self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
shutil.copyfile(self.filename, self._dbBackupFilename)
logSys.info("Database backup created: %s", self._dbBackupFilename)
if version < 2: if version < 2:
cur.executescript("BEGIN TRANSACTION;" cur.executescript("BEGIN TRANSACTION;"
"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;" "CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"