Merge master branch '_0.9/systemd-journal-path-gh-1408' into 0.10

# Conflicts: # fail2ban/tests/filtertestcase.py # fail2ban/tests/utils.py
2016-09-01 16:24:59 +02:00 · 2016-09-01 16:24:59 +02:00 · 387aa6ba47
parent c0e0cfb39d 5f35b52b9a
commit 387aa6ba47
8 changed files with 155 additions and 73 deletions
--- a/5
+++ b/5
@ -112,6 +112,11 @@ releases.
 * New forward compatibility method assertRaisesRegexp (normally python >= 2.7).
  Methods assertIn, assertNotIn, assertRaisesRegexp, assertLogged, assertNotLogged
  are test covered now
+* Jail configuration extended with new syntax to pass options to the backend (see gh-1408),
+  examples:
+    - `backend = systemd[journalpath=/run/log/journal/machine-1]`
+    - `backend = systemd[journalfiles="/run/log/journal/machine-1/system.journal, /run/log/journal/machine-1/user.journal"]`
+    - `backend = systemd[journalflags=2]`


 ver. 0.9.5 (2016/07/15) - old-not-obsolete
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@ -197,7 +197,7 @@ class JailReader(ConfigReader):
 		stream = []
 		for opt, value in self.__opts.iteritems():
 			if opt == "logpath" and	\
-					self.__opts.get('backend', None) != "systemd":
+					not self.__opts.get('backend', None).startswith("systemd"):
 				found_files = 0
 				for path in value.split("\n"):
 					path = path.rsplit(" ", 1)
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@ -33,7 +33,7 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204':
 from .failmanager import FailManagerEmpty
 from .filter import JournalFilter
 from .mytime import MyTime
-from ..helpers import getLogger
+from ..helpers import getLogger, logging, splitwords

 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@ -54,14 +54,45 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	# @param jail the jail object

 	def __init__(self, jail, **kwargs):
+		jrnlargs = FilterSystemd._getJournalArgs(kwargs)
 		JournalFilter.__init__(self, jail, **kwargs)
-		self.__modified = False
+		self.__modified = 0
 		# Initialise systemd-journal connection
-		self.__journal = journal.Reader(converters={'__CURSOR': lambda x: x})
+		self.__journal = journal.Reader(**jrnlargs)
 		self.__matches = []
 		self.setDatePattern(None)
+		self.ticks = 0
 		logSys.debug("Created FilterSystemd")

+	@staticmethod
+	def _getJournalArgs(kwargs):
+		args = {'converters':{'__CURSOR': lambda x: x}}
+		try:
+			args['path'] = kwargs.pop('journalpath')
+		except KeyError:
+			pass
+
+		try:
+			args['files'] = kwargs.pop('journalfiles')
+		except KeyError:
+			pass
+		else:
+			import glob
+			p = args['files']
+			if not isinstance(p, (list, set, tuple)):
+				p = splitwords(p)
+			files = []
+			for p in p:
+				files.extend(glob.glob(p))
+			args['files'] = list(set(files))
+
+		try:
+			args['flags'] = kwargs.pop('journalflags')
+		except KeyError:
+			pass
+
+		return args
+
 	##
 	# Add a journal match filters from list structure
 	#
@ -207,6 +238,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		return (('', date.isoformat(), logline),
 			time.mktime(date.timetuple()) + date.microsecond/1.0E6)

+	def seekToTime(self, date):
+		if not isinstance(date, datetime.datetime):
+			date = datetime.datetime.fromtimestamp(date)
+		self.__journal.seek_realtime(date)
+
 	##
 	# Main loop.
 	#
@ -224,7 +260,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		# Seek to now - findtime in journal
 		start_time = datetime.datetime.now() - \
 				datetime.timedelta(seconds=int(self.getFindTime()))
-		self.__journal.seek_realtime(start_time)
+		self.seekToTime(start_time)
 		# Move back one entry to ensure do not end up in dead space
 		# if start time beyond end of journal
 		try:
@ -233,29 +269,38 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			pass # Reading failure, so safe to ignore

 		while self.active:
-			if not self.idle:
-				while self.active:
-					try:
-						logentry = self.__journal.get_next()
-					except OSError:
-						logSys.warning(
-							"Error reading line from systemd journal")
-						continue
-					if logentry:
-						self.processLineAndAdd(
-							*self.formatJournalEntry(logentry))
-						self.__modified = True
-					else:
-						break
-				if self.__modified:
-					try:
-						while True:
-							ticket = self.failManager.toBan()
-							self.jail.putFailTicket(ticket)
-					except FailManagerEmpty:
-						self.failManager.cleanup(MyTime.time())
-					self.__modified = False
+			# wait for records (or for timeout in sleeptime seconds):
 			self.__journal.wait(self.sleeptime)
+			if self.idle:
+				# because journal.wait will returns immediatelly if we have records in journal,
+				# just wait a little bit here for not idle, to prevent hi-load:
+				time.sleep(self.sleeptime)
+				continue
+			self.__modified = 0
+			while self.active:
+				logentry = None
+				try:
+					logentry = self.__journal.get_next()
+				except OSError as e:
+					logSys.error("Error reading line from systemd journal: %s",
+						e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG)
+				self.ticks += 1
+				if logentry:
+					self.processLineAndAdd(
+						*self.formatJournalEntry(logentry))
+					self.__modified += 1
+					if self.__modified >= 100: # todo: should be configurable
+						break
+				else:
+					break
+			if self.__modified:
+				try:
+					while True:
+						ticket = self.failManager.toBan()
+						self.jail.putFailTicket(ticket)
+				except FailManagerEmpty:
+					self.failManager.cleanup(MyTime.time())
+
 		logSys.debug((self.jail is not None and self.jail.name
                      or "jailless") +" filter terminated")
 		return True
--- a/fail2ban/server/jail.py
+++ b/fail2ban/server/jail.py
@ -27,6 +27,7 @@ import logging
 import Queue

 from .actions import Actions
+from ..client.jailreader import JailReader
 from ..helpers import getLogger

 # Gets the instance of the logger.
@ -83,6 +84,7 @@ class Jail(object):
 		return "%s(%r)" % (self.__class__.__name__, self.name)

 	def _setBackend(self, backend):
+		backend, beArgs = JailReader.extractOptions(backend)
 		backend = backend.lower()		# to assure consistent matching

 		backends = self._BACKENDS
@ -99,7 +101,7 @@ class Jail(object):
 		for b in backends:
 			initmethod = getattr(self, '_init%s' % b.capitalize())
 			try:
-				initmethod()
+				initmethod(**beArgs)
 				if backend != 'auto' and b != backend:
 					logSys.warning("Could only initiated %r backend whenever "
 								   "%r was requested" % (b, backend))
@ -119,28 +121,28 @@ class Jail(object):
 		raise RuntimeError(
 			"Failed to initialize any backend for Jail %r" % self.name)

-	def _initPolling(self):
+	def _initPolling(self, **kwargs):
 		from filterpoll import FilterPoll
-		logSys.info("Jail '%s' uses poller" % self.name)
-		self.__filter = FilterPoll(self)
+		logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs))
+		self.__filter = FilterPoll(self, **kwargs)

-	def _initGamin(self):
+	def _initGamin(self, **kwargs):
 		# Try to import gamin
 		from filtergamin import FilterGamin
-		logSys.info("Jail '%s' uses Gamin" % self.name)
-		self.__filter = FilterGamin(self)
+		logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs))
+		self.__filter = FilterGamin(self, **kwargs)

-	def _initPyinotify(self):
+	def _initPyinotify(self, **kwargs):
 		# Try to import pyinotify
 		from filterpyinotify import FilterPyinotify
-		logSys.info("Jail '%s' uses pyinotify" % self.name)
-		self.__filter = FilterPyinotify(self)
+		logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs))
+		self.__filter = FilterPyinotify(self, **kwargs)

-	def _initSystemd(self): # pragma: systemd no cover
+	def _initSystemd(self, **kwargs): # pragma: systemd no cover
 		# Try to import systemd
 		from filtersystemd import FilterSystemd
-		logSys.info("Jail '%s' uses systemd" % self.name)
-		self.__filter = FilterSystemd(self)
+		logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs))
+		self.__filter = FilterSystemd(self, **kwargs)

 	@property
 	def name(self):
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@ -37,7 +37,7 @@ from ..client.actionreader import ActionReader
 from ..client.configurator import Configurator
 from ..server.mytime import MyTime
 from ..version import version
-from .utils import LogCaptureTestCase
+from .utils import LogCaptureTestCase, with_tmpdir

 TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
 TEST_FILES_DIR_SHARE_CFG = {}
@ -97,9 +97,8 @@ option = %s
 		if not os.access(f, os.R_OK):
 			self.assertFalse(self.c.read('d'))	# should not be readable BUT present
 		else:
-			# SkipTest introduced only in 2.7 thus can't yet use generally
-			# raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform)
-			pass
+			import platform
+			raise unittest.SkipTest("Skipping on %s -- access rights are not enforced" % platform.platform())

 	def testOptionalDotDDir(self):
 		self.assertFalse(self.c.read('c'))	# nothing is there yet
@ -298,8 +297,8 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(eval(act[2][5]).get('agent', '<wrong>'), useragent)
 		self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent])

-	def testGlob(self):
-		d = tempfile.mkdtemp(prefix="f2b-temp")
+	@with_tmpdir
+	def testGlob(self, d):
 		# Generate few files
 		# regular file
 		f1 = os.path.join(d, 'f1')
@ -314,9 +313,6 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(JailReader._glob(f2), [])
 		self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2)
 		self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), [])
-		os.remove(f1)
-		os.remove(f2)
-		os.rmdir(d)

 		
 class FilterReaderTest(unittest.TestCase):
@ -448,11 +444,11 @@ class JailsReaderTestCache(LogCaptureTestCase):
 				cnt += 1
 		return cnt

-	def testTestJailConfCache(self):
+	@with_tmpdir
+	def testTestJailConfCache(self, basedir):
 		unittest.F2B.SkipIfFast()
 		saved_ll = configparserinc.logLevel
 		configparserinc.logLevel = logging.DEBUG
-		basedir = tempfile.mkdtemp("fail2ban_conf")
 		try:
 			shutil.rmtree(basedir)
 			shutil.copytree(CONFIG_DIR, basedir)
@ -484,7 +480,6 @@ class JailsReaderTestCache(LogCaptureTestCase):
 			cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf')
 			self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt)
 		finally:
-			shutil.rmtree(basedir)
 			configparserinc.logLevel = saved_ll


@ -739,8 +734,8 @@ class JailsReaderTest(LogCaptureTestCase):
 			self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
 			self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)

-	def testMultipleSameAction(self):
-		basedir = tempfile.mkdtemp("fail2ban_conf")
+	@with_tmpdir
+	def testMultipleSameAction(self, basedir):
 		os.mkdir(os.path.join(basedir, "filter.d"))
 		os.mkdir(os.path.join(basedir, "action.d"))
 		open(os.path.join(basedir, "action.d", "testaction1.conf"), 'w').close()
@ -769,4 +764,33 @@ filter = testfilter1
 		# Python actions should not be passed `actname`
 		self.assertEqual(add_actions[-1][-1], "{}")

-		shutil.rmtree(basedir)
+	def testLogPathFileFilterBackend(self):
+		self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", 
+			self._testLogPath, backend='polling')
+
+	def testLogPathSystemdBackend(self):
+		try: # pragma: systemd no cover
+			from ..server.filtersystemd import FilterSystemd
+		except Exception, e: # pragma: no cover
+			raise unittest.SkipTest("systemd python interface not available")
+		self._testLogPath(backend='systemd')
+		self._testLogPath(backend='systemd[journalflags=2]')
+	
+	@with_tmpdir
+	def _testLogPath(self, basedir, backend):
+		jailfd = open(os.path.join(basedir, "jail.conf"), 'w')
+		jailfd.write("""
+[testjail1]
+enabled = true
+backend = %s
+logpath = %s/not/exist.log
+          /this/path/should/not/exist.log
+action = 
+filter = 
+failregex = test <HOST>
+""" % (backend, basedir))
+		jailfd.close()
+		jails = JailsReader(basedir=basedir)
+		self.assertTrue(jails.read())
+		self.assertTrue(jails.getOptions())
+		jails.convert()
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@ -55,12 +55,10 @@ class DatabaseTest(LogCaptureTestCase):
 	def setUp(self):
 		"""Call before every test case."""
 		super(DatabaseTest, self).setUp()
-		if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover
+		if Fail2BanDb is None: # pragma: no cover
 			raise unittest.SkipTest(
 				"Unable to import fail2ban database module as sqlite is not "
 				"available.")
-		elif Fail2BanDb is None:
-			return
 		self.dbFilename = None
 		if not unittest.F2B.memory_db:
 			_, self.dbFilename = tempfile.mkstemp(".db", "fail2ban_")
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@ -932,11 +932,16 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			super(MonitorJournalFailures, self).setUp()
 			self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log")
 			self.jail = DummyJail()
-			self.filter = Filter_(self.jail)
+			self.filter = None
 			# UUID used to ensure that only meeages generated
 			# as part of this test are picked up by the filter
 			self.test_uuid = str(uuid.uuid4())
 			self.name = "%s-%s" % (testclass_name, self.test_uuid)
+			self.journal_fields = {
+				'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid}
+
+		def _initFilter(self, **kwargs):
+			self.filter = Filter_(self.jail, **kwargs)
 			self.filter.addJournalMatch([
 				"SYSLOG_IDENTIFIER=fail2ban-testcases",
 				"TEST_FIELD=1",
@ -945,16 +950,20 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				"SYSLOG_IDENTIFIER=fail2ban-testcases",
 				"TEST_FIELD=2",
 				"TEST_UUID=%s" % self.test_uuid])
-			self.journal_fields = {
-				'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid}
-			self.filter.active = True
 			self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
-			self.filter.start()

 		def tearDown(self):
-			self.filter.stop()
-			self.filter.join()		  # wait for the thread to terminate
-			pass
+			if self.filter and self.filter.active:
+				self.filter.stop()
+				self.filter.join()		  # wait for the thread to terminate
+				pass
+
+		def testJournalFlagsArg(self):
+			self._initFilter(journalflags=2) # journal.RUNTIME_ONLY
+
+		def __str__(self):
+			return "MonitorJournalFailures%s(%s)" \
+			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')

 		def assert_correct_ban(self, test_ip, test_attempts):
 			self.assertTrue(self.waitFailTotal(test_attempts, 10)) # give Filter a chance to react
@ -969,6 +978,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.assertEqual(attempts, test_attempts)

 		def test_grow_file(self):
+			self._initFilter()
+			self.filter.start()
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)

 			# Now let's feed it with entries from the file
@ -998,6 +1009,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 			self.assert_correct_ban("193.168.0.128", 3)

 		def test_delJournalMatch(self):
+			self._initFilter()
+			self.filter.start()
 			# Smoke test for removing of match

 			# basic full test
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@ -717,10 +717,7 @@ class Transmitter(TransmitterBase):

 	def testJournalMatch(self):
 		if not filtersystemd: # pragma: no cover
-			if sys.version_info >= (2, 7):
-				raise unittest.SkipTest(
-					"systemd python interface not available")
-			return
+			raise unittest.SkipTest("systemd python interface not available")
 		jailName = "TestJail2"
 		self.server.addJail(jailName, "systemd")
 		values = [
@ -820,10 +817,8 @@ class TransmitterLogging(TransmitterBase):
 		self.setGetTest("logtarget", "STDERR")

 	def testLogTargetSYSLOG(self):
-		if not os.path.exists("/dev/log") and sys.version_info >= (2, 7):
+		if not os.path.exists("/dev/log"):
 			raise unittest.SkipTest("'/dev/log' not present")
-		elif not os.path.exists("/dev/log"):
-			return
 		self.assertTrue(self.server.getSyslogSocket(), "auto")
 		self.setGetTest("logtarget", "SYSLOG")
 		self.assertTrue(self.server.getSyslogSocket(), "/dev/log")