Merge branch '0.10' into 0.11

ChangeLog

@@ -71,10 +71,14 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition
   - database: improve adapter/converter handlers working on invalid characters in sense of json and/or sqlite-database;
     additionally both are exception-safe now, so avoid possible locking of database (closes gh-2137);
   - logging in fail2ban is process-wide exception-safe now.
+* repaired start-time of initial seek to time (as well as other log-parsing related data), 
+  if parameter `logpath` specified before `findtime`, `backend`, `datepattern`, etc (gh-2173)
+* systemd: fixed type error on option `journalflags`: an integer is required (gh-2125);
 
 ### New Features
 
 ### Enhancements
+* `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168)
 * since v.0.10.4, fail2ban-client, fail2ban-server and fail2ban-regex will return version without logo info,
   additionally option `-V` can be used to get version in normalized machine-readable short format.
 

config/filter.d/dovecot.conf

@@ -12,8 +12,8 @@ _daemon = (?:dovecot(?:-auth)?|auth)
 
 prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
+            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
             ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
             ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$
             <mdre-<mode>>

fail2ban/client/jailreader.py

@@ -90,9 +90,6 @@ class JailReader(ConfigReader):
 		opts1st = [["bool", "enabled", False],
 				["string", "filter", ""]]
 		opts = [["bool", "enabled", False],
-				["string", "logpath", None],
-				["string", "logtimezone", None],
-				["string", "logencoding", None],
 				["string", "backend", "auto"],
 				["int",    "maxretry", None],
 				["string", "findtime", None],
@@ -112,6 +109,9 @@ class JailReader(ConfigReader):
 				["string", "ignoreip", None],
 				["string", "filter", ""],
 				["string", "datepattern", None],
+				["string", "logtimezone", None],
+				["string", "logencoding", None],
+				["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc)
 				["string", "action", ""]]
 
 		# Before interpolation (substitution) add static options always available as default:

fail2ban/server/banmanager.py

@@ -156,7 +156,7 @@ class BanManager:
 		# get cymru info:
 		try:
 			for ip in banIPs:
-				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
+				# Reference: https://www.team-cymru.com/IP-ASN-mapping.html#dns
 				question = ip.getPTR(
 					"origin.asn.cymru.com" if ip.isIPv4
 					else "origin6.asn.cymru.com"

fail2ban/server/filter.py

@@ -891,9 +891,6 @@ class FileFilter(Filter):
 			self.__logs[path] = log
 			logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
 			if autoSeek:
-				# if default, seek to "current time" - "find time":
-				if isinstance(autoSeek, bool):
-					autoSeek = MyTime.time() - self.getFindTime()
 				self.__autoSeek[path] = autoSeek
 			self._addLogPath(path)			# backend specific
 
@@ -1003,18 +1000,21 @@ class FileFilter(Filter):
 				return False
 
 			# seek to find time for first usage only (prevent performance decline with polling of big files)
-			if self.__autoSeek.get(filename):
-				startTime = self.__autoSeek[filename]
-				del self.__autoSeek[filename]
-				# prevent completely read of big files first time (after start of service), 
-				# initial seek to start time using half-interval search algorithm:
-				try:
-					self.seekToTime(log, startTime)
-				except Exception as e: # pragma: no cover
-					logSys.error("Error during seek to start time in \"%s\"", filename)
-					raise
-					logSys.exception(e)
-					return False
+			if self.__autoSeek:
+				startTime = self.__autoSeek.pop(filename, None)
+				if startTime:
+					# if default, seek to "current time" - "find time":
+					if isinstance(startTime, bool):
+						startTime = MyTime.time() - self.getFindTime()
+					# prevent completely read of big files first time (after start of service), 
+					# initial seek to start time using half-interval search algorithm:
+					try:
+						self.seekToTime(log, startTime)
+					except Exception as e: # pragma: no cover
+						logSys.error("Error during seek to start time in \"%s\"", filename)
+						raise
+						logSys.exception(e)
+						return False
 
 			if has_content:
 				while not self.idle:

fail2ban/server/filtersystemd.py

@@ -87,7 +87,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			args['files'] = list(set(files))
 
 		try:
-			args['flags'] = kwargs.pop('journalflags')
+			args['flags'] = int(kwargs.pop('journalflags'))
 		except KeyError:
 			pass
 

fail2ban/tests/servertestcase.py

@@ -733,7 +733,7 @@ class Transmitter(TransmitterBase):
 		self.assertEqual(
 			self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1)
 
-	def testJournalMatch(self):
+	def testJournalMatch(self): # pragma: systemd no cover
 		if not filtersystemd: # pragma: no cover
 			raise unittest.SkipTest("systemd python interface not available")
 		jailName = "TestJail2"
@@ -803,6 +803,28 @@ class Transmitter(TransmitterBase):
 		result = self.transm.proceed(
 			["set", jailName, "deljournalmatch", value])
 		self.assertTrue(isinstance(result[1], ValueError))
+	
+	def testJournalFlagsMatch(self): # pragma: systemd no cover
+		if not filtersystemd: # pragma: no cover
+			raise unittest.SkipTest("systemd python interface not available")
+		self.assertTrue(True)
+		jailName = "TestJail3"
+		self.server.addJail(jailName, "systemd[journalflags=2]")
+		values = [
+			"_SYSTEMD_UNIT=sshd.service",
+			"TEST_FIELD1=ABC",
+			"_HOSTNAME=example.com",
+		]
+		for n, value in enumerate(values):
+			self.assertEqual(
+				self.transm.proceed(
+					["set", jailName, "addjournalmatch", value]),
+				(0, [[val] for val in values[:n+1]]))
+		for n, value in enumerate(values):
+			self.assertEqual(
+				self.transm.proceed(
+					["set", jailName, "deljournalmatch", value]),
+				(0, [[val] for val in values[n+1:]]))
 
 
 class TransmitterLogging(TransmitterBase):