From 1c65b946171c3bbc626ddcd9320ea2515018677b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 19 Mar 2014 18:55:54 +0000 Subject: [PATCH 1/9] BF: Handle case when no sqlite library is available for the database --- fail2ban/server/server.py | 16 ++++++++++--- fail2ban/tests/databasetestcase.py | 36 +++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index f1816993..1bf8dcbb 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -31,12 +31,17 @@ from .jails import Jails from .filter import FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException -from .database import Fail2BanDb from .. import version # Gets the instance of the logger. logSys = logging.getLogger(__name__) +try: + from .database import Fail2BanDb +except ImportError: + # Dont print error here, as database may not even be used + Fail2BanDb = None + class Server: def __init__(self, daemon = False): @@ -439,8 +444,13 @@ class Server: if filename.lower() == "none": self.__db = None else: - self.__db = Fail2BanDb(filename) - self.__db.delAllJails() + if Fail2BanDb is not None: + self.__db = Fail2BanDb(filename) + self.__db.delAllJails() + else: + logSys.error( + "Unable to import fail2ban database module as sqlite " + "is not available.") else: raise RuntimeError( "Cannot change database when there are jails present") diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index dfc26772..1cfd78b9 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -23,16 +23,20 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import os +import sys import unittest import tempfile import sqlite3 import shutil -from ..server.database import Fail2BanDb from ..server.filter import FileContainer from ..server.mytime import MyTime from ..server.ticket import FailTicket from .dummyjail import DummyJail +try: + from ..server.database import Fail2BanDb +except ImportError: + Fail2BanDb = None TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -40,24 +44,38 @@ class DatabaseTest(unittest.TestCase): def setUp(self): """Call before every test case.""" + if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover + raise unittest.SkipTest( + "Unable to import fail2ban database module as sqlite is not " + "available.") + elif Fail2BanDb is None: + return _, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_") self.db = Fail2BanDb(self.dbFilename) def tearDown(self): """Call after every test case.""" + if Fail2BanDb is None: # pragma: no cover + return # Cleanup os.remove(self.dbFilename) def testGetFilename(self): + if Fail2BanDb is None: # pragma: no cover + return self.assertEqual(self.dbFilename, self.db.filename) def testCreateInvalidPath(self): + if Fail2BanDb is None: # pragma: no cover + return self.assertRaises( sqlite3.OperationalError, Fail2BanDb, "/this/path/should/not/exist") def testCreateAndReconnect(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() # Reconnect... self.db = Fail2BanDb(self.dbFilename) @@ -67,6 +85,8 @@ class DatabaseTest(unittest.TestCase): "Jail not retained in Db after disconnect reconnect.") def testUpdateDb(self): + if Fail2BanDb is None: # pragma: no cover + return shutil.copyfile( os.path.join(TEST_FILES_DIR, 'database_v1.db'), self.dbFilename) self.db = Fail2BanDb(self.dbFilename) @@ -80,6 +100,8 @@ class DatabaseTest(unittest.TestCase): os.remove(self.db._dbBackupFilename) def testAddJail(self): + if Fail2BanDb is None: # pragma: no cover + return self.jail = DummyJail() self.db.addJail(self.jail) self.assertTrue( @@ -87,6 +109,8 @@ class DatabaseTest(unittest.TestCase): "Jail not added to database") def testAddLog(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() # Jail required _, filename = tempfile.mkstemp(".log", "Fail2BanDb_") @@ -98,6 +122,8 @@ class DatabaseTest(unittest.TestCase): os.remove(filename) def testUpdateLog(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddLog() # Add log file # Write some text @@ -137,6 +163,8 @@ class DatabaseTest(unittest.TestCase): os.remove(filename) def testAddBan(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() ticket = FailTicket("127.0.0.1", 0, ["abc\n"]) self.db.addBan(self.jail, ticket) @@ -146,6 +174,8 @@ class DatabaseTest(unittest.TestCase): isinstance(self.db.getBans(jail=self.jail)[0], FailTicket)) def testGetBansWithTime(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() ticket = FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"]) self.db.addBan(self.jail, ticket) @@ -153,6 +183,8 @@ class DatabaseTest(unittest.TestCase): self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=20)), 0) def testGetBansMerged(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() jail2 = DummyJail() @@ -197,6 +229,8 @@ class DatabaseTest(unittest.TestCase): id(self.db.getBansMerged("127.0.0.1", jail=self.jail))) def testPurge(self): + if Fail2BanDb is None: # pragma: no cover + return self.testAddJail() # Add jail self.db.purge() # Jail enabled by default so shouldn't be purged From 8c129cc28359019c147a075ffd227f270925a215 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 19 Mar 2014 18:59:00 +0000 Subject: [PATCH 2/9] DOC: Update ChangeLog fixes --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 6f5ebe78..bfb078f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * badips.py action error when logging HTTP error raised with badips request * fail2ban-regex failed to work in python3 due to space/tab mix * journalmatch for recidive incorrect PRIORITY + * Handle case when no sqlite library is available for persistent database - New features: From 1470e3c01d49841335e11ed7ca7898516d1b8be8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 19 Mar 2014 19:09:07 +0000 Subject: [PATCH 3/9] BF: fail2ban.conf reader expected "int" type for `loglevel` Closes #657 --- ChangeLog | 1 + fail2ban/client/fail2banreader.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6f5ebe78..2016a9fd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,6 +15,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * badips.py action error when logging HTTP error raised with badips request * fail2ban-regex failed to work in python3 due to space/tab mix * journalmatch for recidive incorrect PRIORITY + * loglevel couldn't be changed in fail2ban.conf - New features: diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index f17ff92b..251c6985 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -45,7 +45,7 @@ class Fail2banReader(ConfigReader): return ConfigReader.getOptions(self, "Definition", opts) def getOptions(self): - opts = [["int", "loglevel", "INFO" ], + opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbpurgeage", 86400]] From 75325da09091f3ae800a2efbcde1a016617e5f1a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 19 Mar 2014 19:21:23 +0000 Subject: [PATCH 4/9] TST: Skip SYSLOG log target test if '/dev/log' not present --- fail2ban/tests/servertestcase.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 231aecd2..c4163db9 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -678,6 +678,12 @@ class TransmitterLogging(TransmitterBase): self.setGetTest("logtarget", "STDOUT") self.setGetTest("logtarget", "STDERR") + + def testLogTargetSYSLOG(self): + if not os.path.exists("/dev/log") and sys.version_info >= (2, 7): + raise unittest.SkipTest("'/dev/log' not present") + elif not os.path.exists("/dev/log"): + return self.setGetTest("logtarget", "SYSLOG") def testLogLevel(self): From 175c5934620adb600fe4435732a3887855320669 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 19 Mar 2014 19:30:48 +0000 Subject: [PATCH 5/9] TST: Skip badips.py test is no network option set --- fail2ban/tests/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 456a829d..85c1d929 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -209,6 +209,9 @@ def gatherTests(regexps=None, no_network=False): for file_ in os.listdir( os.path.abspath(os.path.dirname(action_d.__file__))): if file_.startswith("test_") and file_.endswith(".py"): + if no_network and file_ in ['test_badips.py']: #pragma: no cover + # Test required network + continue tests.addTest(testloader.loadTestsFromName( "%s.%s" % (action_d.__name__, os.path.splitext(file_)[0]))) From e437bd5e6faf10c6bf9c3c275b610d5ca036d656 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 21 Mar 2014 18:35:23 +0000 Subject: [PATCH 6/9] DOC: Add database settings for fail2ban.conf to jail.conf(5) man page --- man/jail.conf.5 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index ef243a2d..de7fd44b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -121,6 +121,14 @@ This is used for communication with the fail2ban server daemon. Do not remove th .B pidfile PID filename. Default: /var/run/fail2ban/fail2ban.pid. This is used to store the process ID of the fail2ban server. +.TP +.B dbfile +Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 +This defines where the persistent data for fail2ban is stored. This persistent data allows bans to be reinstated and continue reading log files from the last read position when fail2ban is restarted. A value of \fINone\fR disables this feature. +.TP +.B dbpurgeage +Database purge age in seconds. Default: 86400 (24hours) +This sets the age at which bans should be purged from the database. .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. From c7c203964cac5f6efbf2df4499a123f0aa1d4ba3 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 22 Mar 2014 17:28:49 +0000 Subject: [PATCH 7/9] ENH: Add iptables and firewalld to "After" for systemd service file. Also remove no longer supported syslog.target. Closes #658 --- files/fail2ban.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/fail2ban.service b/files/fail2ban.service index 9c44042b..7694bcf0 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -1,6 +1,6 @@ [Unit] -Description=Fail2ban Service -After=syslog.target network.target +Description=Fail2Ban Service +After=network.target iptables.service firewalld.service [Service] Type=forking From b73ed9b59eb2d81fce14f93d7d48fb74a36967e7 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 24 Mar 2014 00:30:46 +0000 Subject: [PATCH 8/9] BF: Ignored IPs no longer being banned from database on restart --- fail2ban/server/jail.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 9c17fcae..dec04c73 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -211,7 +211,8 @@ class Jail: if self.database is not None: for ticket in self.database.getBans( jail=self, bantime=self.actions.getBanTime()): - self.__queue.put(ticket) + if not self.filter.inIgnoreIPList(ticket.getIP()): + self.__queue.put(ticket) logSys.info("Jail '%s' started" % self.name) def stop(self): From 1695d5c076ed6dfb675f976bcd9e96dae7ef86ab Mon Sep 17 00:00:00 2001 From: Ruben Kerkhof Date: Mon, 24 Mar 2014 13:16:52 +0000 Subject: [PATCH 9/9] Fix a few typos Found with https://github.com/lucasdemarchi/codespell Signed-off-by: Ruben Kerkhof --- ChangeLog | 6 +++--- config/action.d/blocklist_de.conf | 6 +++--- config/action.d/xarf-login-attack.conf | 2 +- config/filter.d/dovecot.conf | 2 +- config/filter.d/solid-pop3d.conf | 2 +- config/jail.conf | 2 +- fail2ban/client/jailreader.py | 2 +- fail2ban/server/database.py | 4 ++-- fail2ban/server/datetemplate.py | 2 +- fail2ban/server/filter.py | 2 +- fail2ban/tests/servertestcase.py | 4 ++-- fail2ban/tests/sockettestcase.py | 2 +- man/fail2ban.1 | 2 +- man/jail.conf.5 | 2 +- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ChangeLog b/ChangeLog index c9967067..22db55d9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -269,15 +269,15 @@ and bug requests. Daniel Black & Sebastian Arcus * filter.d/asterisk -- more regexes Daniel Black - * action.d/hostsdeny -- NOTE: new dependancy 'ed'. Switched to use 'ed' across + * action.d/hostsdeny -- NOTE: new dependency 'ed'. Switched to use 'ed' across all platforms to ensure permissions are the same before and after a ban. Closes gh-266. hostsdeny supports daemon_list now too. - * action.d/bsd-ipfw - action option unsed. Change blocktype to port unreach + * action.d/bsd-ipfw - action option unused. Change blocktype to port unreach instead of deny for consistancy. * filter.d/dovecot - added to support different dovecot failure "..disallowed plaintext auth". Closes Debian bug #709324 * filter.d/roundcube-auth - timezone offset can be positive or negative - * action.d/bsd-ipfw - action option unsed. Fixed to blocktype for + * action.d/bsd-ipfw - action option unused. Fixed to blocktype for consistency. default to port unreach instead of deny * filter.d/dropbear - fix regexs to match standard dropbear and the patched http://www.unchartedbackwaters.co.uk/files/dropbear/dropbear-0.52.patch diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index d4170cab..6d520694 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -7,13 +7,13 @@ # Action to report IP address to blocklist.de # Blocklist.de must be signed up to at www.blocklist.de # Once registered, one or more servers can be added. -# This action requires the server 'email address' and the assoicate apikey. +# This action requires the server 'email address' and the associated apikey. # # From blocklist.de: # www.blocklist.de is a free and voluntary service provided by a # Fraud/Abuse-specialist, whose servers are often attacked on SSH-, # Mail-Login-, FTP-, Webserver- and other services. -# The mission is to report all attacks to the abuse deparments of the +# The mission is to report all attacks to the abuse departments of the # infected PCs/servers to ensure that the responsible provider can inform # the customer about the infection and disable them # @@ -25,7 +25,7 @@ # * The recidive where the IP has been banned multiple times # * Where maxretry has been set quite high, beyond the normal user typing # password incorrectly. -# * For filters that have a low likelyhood of receiving human errors +# * For filters that have a low likelihood of receiving human errors # [Definition] diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 32c611a1..1282d4c8 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -8,7 +8,7 @@ # * The recidive where the IP has been banned multiple times # * Where maxretry has been set quite high, beyond the normal user typing # password incorrectly. -# * For filters that have a low likelyhood of receiving human errors +# * For filters that have a low likelihood of receiving human errors # # DEPENDANCIES: # diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index 732eb3a3..0b154ffb 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -23,7 +23,7 @@ journalmatch = _SYSTEMD_UNIT=dovecot.service # * the first regex is essentially a copy of pam-generic.conf # * Probably doesn't do dovecot sql/ldap backends properly # * Removed the 'no auth attempts' log lines from the matches because produces -# lots of false positives on misconfigured MTAs making regexp unuseable +# lots of false positives on misconfigured MTAs making regexp unusable # # Author: Martin Waschbuesch # Daniel Black (rewrote with begin and end anchors) diff --git a/config/filter.d/solid-pop3d.conf b/config/filter.d/solid-pop3d.conf index d97cc134..ba19d664 100644 --- a/config/filter.d/solid-pop3d.conf +++ b/config/filter.d/solid-pop3d.conf @@ -1,4 +1,4 @@ -# Fail2Ban filter for unsuccesful solid-pop3 authentication attempts +# Fail2Ban filter for unsuccessful solid-pop3 authentication attempts # # Doesn't currently provide PAM support as PAM log messages don't include rhost as # remote IP. diff --git a/config/jail.conf b/config/jail.conf index 8b29bc11..96b3096f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -595,7 +595,7 @@ logpath = /var/log/nsd.log # -# Miscelaneous +# Miscellaneous # [asterisk] diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 7bdb11d2..42c04ffb 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -222,7 +222,7 @@ class JailReader(ConfigReader): def extractOptions(option): match = JailReader.optionCRE.match(option) if not match: - # TODO propper error handling + # TODO proper error handling return None, None option_name, optstr = match.groups() option_opts = dict() diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 2f6667cd..0fad396b 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -156,7 +156,7 @@ class Fail2BanDb(object): logSys.warning( "Database updated from '%i' to '%i'", version, newversion) else: - logSys.error( "Database update failed to acheive version '%i'" + logSys.error( "Database update failed to achieve version '%i'" ": updated from '%i' to '%i'", Fail2BanDb.__version__, version, newversion) raise RuntimeError('Failed to fully update') @@ -357,7 +357,7 @@ class Fail2BanDb(object): Parameters ---------- jail : Jail - Jail in which the ban has occured. + Jail in which the ban has occurred. ticket : BanTicket Ticket of the ban to be added. """ diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 9989e22b..19bdd0ef 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -73,7 +73,7 @@ class DateTemplate(object): The regex the template will use for searching for a date. wordBegin : bool Defines whether the regex should be modified to search at - begining of a word, by adding "\\b" to start of regex. + beginning of a word, by adding "\\b" to start of regex. Default True. Raises diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cc171fea..c82395ea 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -717,7 +717,7 @@ class FileContainer: self.setEncoding(encoding) self.__tail = tail self.__handler = None - # Try to open the file. Raises an exception if an error occured. + # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') stats = os.fstat(handler.fileno()) self.__ino = stats.st_ino diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index c4163db9..9aba7a62 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -155,7 +155,7 @@ class Transmitter(TransmitterBase): def testDatabase(self): _, tmpFilename = tempfile.mkstemp(".db", "Fail2Ban_") - # Jails present, cant change database + # Jails present, can't change database self.setGetTestNOK("dbfile", tmpFilename) self.server.delJail(self.jailName) self.setGetTest("dbfile", tmpFilename) @@ -581,7 +581,7 @@ class Transmitter(TransmitterBase): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): raise unittest.SkipTest( - "systemd python interface not avilable") + "systemd python interface not available") return jailName = "TestJail2" self.server.addJail(jailName, "systemd") diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index e668be39..8a548998 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -68,7 +68,7 @@ class Socket(unittest.TestCase): self.assertRaises( AsyncServerException, self.server.start, self.sock_name, False) - # Try agin with force set + # Try again with force set serverThread = threading.Thread( target=self.server.start, args=(self.sock_name, True)) serverThread.daemon = True diff --git a/man/fail2ban.1 b/man/fail2ban.1 index 660168f1..23c74b88 100644 --- a/man/fail2ban.1 +++ b/man/fail2ban.1 @@ -34,7 +34,7 @@ mechanisms if you really want to protect services. A local user is able to inject messages into syslog and using a Fail2Ban jail that reads from syslog, they can effectively trigger a DoS attack against -any IP. Know this risk and configure Fail2Ban/grant shell access acordingly. +any IP. Know this risk and configure Fail2Ban/grant shell access accordingly. .SH FILES \fI/etc/fail2ban/*\fR diff --git a/man/jail.conf.5 b/man/jail.conf.5 index de7fd44b..c4215fdd 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -137,7 +137,7 @@ The following options are applicable to any jail. They appear in a section speci name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. Only one filter can be specified. .TP .B logpath -filename(s) of the log files to be monitored, seperate by new lines. Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. +filename(s) of the log files to be monitored, separated by new lines. Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. Optional space separated option 'tail' can be added to the end of the path to cause the log file to be read from the end, else default 'head' option reads file from the beginning