mirror of https://github.com/fail2ban/fail2ban
Merge pull request #1436 from sebres/speedup-db
extremely speedup of all database operationspull/1439/head
commit
b56f4c533e
|
@ -30,6 +30,11 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released
|
||||||
* New Actions:
|
* New Actions:
|
||||||
- action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
|
- action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367)
|
||||||
- Enhancements:
|
- Enhancements:
|
||||||
|
* Extreme speedup of all sqlite database operations (gh-1436),
|
||||||
|
by using of following sqlite options:
|
||||||
|
- (synchronous = OFF) write data through OS without syncing
|
||||||
|
- (journal_mode = MEMORY) use memory for the transaction logging
|
||||||
|
- (temp_store = MEMORY) temporary tables and indices are kept in memory
|
||||||
* journald journalmatch for pure-ftpd (gh-1362)
|
* journald journalmatch for pure-ftpd (gh-1362)
|
||||||
* Add additional regex filter for dovecot ldap authentication failures (gh-1370)
|
* Add additional regex filter for dovecot ldap authentication failures (gh-1370)
|
||||||
* added additional regex filters for exim (gh-1371)
|
* added additional regex filters for exim (gh-1371)
|
||||||
|
|
|
@ -181,8 +181,23 @@ class Fail2BanDb(object):
|
||||||
filename, e.args[0])
|
filename, e.args[0])
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
# differentiate pypy: switch journal mode later (save it during the upgrade),
|
||||||
|
# to prevent errors like "database table is locked":
|
||||||
|
try:
|
||||||
|
import __pypy__
|
||||||
|
pypy = True
|
||||||
|
except ImportError:
|
||||||
|
pypy = False
|
||||||
|
|
||||||
cur = self._db.cursor()
|
cur = self._db.cursor()
|
||||||
cur.execute("PRAGMA foreign_keys = ON;")
|
cur.execute("PRAGMA foreign_keys = ON")
|
||||||
|
# speedup: write data through OS without syncing (no wait):
|
||||||
|
cur.execute("PRAGMA synchronous = OFF")
|
||||||
|
# speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible):
|
||||||
|
if not pypy:
|
||||||
|
cur.execute("PRAGMA journal_mode = MEMORY")
|
||||||
|
# speedup: temporary tables and indices are kept in memory:
|
||||||
|
cur.execute("PRAGMA temp_store = MEMORY")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cur.execute("SELECT version FROM fail2banDb LIMIT 1")
|
cur.execute("SELECT version FROM fail2banDb LIMIT 1")
|
||||||
|
@ -202,6 +217,9 @@ class Fail2BanDb(object):
|
||||||
Fail2BanDb.__version__, version, newversion)
|
Fail2BanDb.__version__, version, newversion)
|
||||||
raise RuntimeError('Failed to fully update')
|
raise RuntimeError('Failed to fully update')
|
||||||
finally:
|
finally:
|
||||||
|
# pypy: set journal mode after possible upgrade db:
|
||||||
|
if pypy:
|
||||||
|
cur.execute("PRAGMA journal_mode = MEMORY")
|
||||||
cur.close()
|
cur.close()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
@ -244,13 +262,14 @@ class Fail2BanDb(object):
|
||||||
|
|
||||||
A timestamped backup is also created prior to attempting the update.
|
A timestamped backup is also created prior to attempting the update.
|
||||||
"""
|
"""
|
||||||
self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
|
|
||||||
shutil.copyfile(self.filename, self._dbBackupFilename)
|
|
||||||
logSys.info("Database backup created: %s", self._dbBackupFilename)
|
|
||||||
if version > Fail2BanDb.__version__:
|
if version > Fail2BanDb.__version__:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Attempt to travel to future version of database ...how did you get here??")
|
"Attempt to travel to future version of database ...how did you get here??")
|
||||||
|
|
||||||
|
self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime())
|
||||||
|
shutil.copyfile(self.filename, self._dbBackupFilename)
|
||||||
|
logSys.info("Database backup created: %s", self._dbBackupFilename)
|
||||||
|
|
||||||
if version < 2:
|
if version < 2:
|
||||||
cur.executescript("BEGIN TRANSACTION;"
|
cur.executescript("BEGIN TRANSACTION;"
|
||||||
"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
|
"CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;"
|
||||||
|
|
Loading…
Reference in New Issue