Merge branch '0.10' into 0.11

.travis.yml

@@ -26,6 +26,7 @@ matrix:
   - python:  3.6
   - python:  3.7
   - python:  3.8
+  - python:  3.9-dev
   - python:  pypy3.5
 before_install:
   - echo "running under $TRAVIS_PYTHON_VERSION"

ChangeLog

@@ -38,6 +38,9 @@ ver. 0.11.2-dev (20??/??/??) - development edition
     IPv6-capable now.
 
 ### Fixes
+* [stability] prevent race condition - no ban if filter (backend) is continuously busy if
+  too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660)
+* python 3.9 compatibility (and Travis CI support)
 * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed
 * manual ban is written to database, so can be restored by restart (gh-2647)
 * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`

fail2ban/server/filter.py

@@ -112,6 +112,8 @@ class Filter(JailThread):
 		self.onIgnoreRegex = None
 		## if true ignores obsolete failures (failure time < now - findTime):
 		self.checkFindTime = True
+		## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes):
+		self.banASAP = True
 		## Ticks counter
 		self.ticks = 0
 		## Thread name:
@@ -626,7 +628,11 @@ class Filter(JailThread):
 				logSys.info(
 					"[%s] Found %s - %s", self.jailName, ip, MyTime.time2str(unixTime)
 				)
-				self.failManager.addFailure(tick)
+				attempts = self.failManager.addFailure(tick)
+				# avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry,
+				# we can speedup ban, so do it as soon as possible:
+				if self.banASAP and attempts >= self.failManager.getMaxRetry():
+					self.performBan(ip)
 				# report to observer - failure was found, for possibly increasing of it retry counter (asynchronous)
 				if Observers.Main is not None:
 					Observers.Main.add('failureFound', self.failManager, self.jail, tick)

fail2ban/server/filtergamin.py

@@ -79,7 +79,8 @@ class FilterGamin(FileFilter):
 		this is a common logic and must be shared/provided by FileFilter
 		"""
 		self.getFailures(path)
-		self.performBan()
+		if not self.banASAP: # pragma: no cover
+			self.performBan()
 		self.__modified = False
 
 	##

fail2ban/server/filterpoll.py

@@ -117,7 +117,8 @@ class FilterPoll(FileFilter):
 
 				self.ticks += 1
 				if self.__modified:
-					self.performBan()
+					if not self.banASAP: # pragma: no cover
+						self.performBan()
 					self.__modified = False
 			except Exception as e: # pragma: no cover
 				if not self.active: # if not active - error by stop...

fail2ban/server/filterpyinotify.py

@@ -140,7 +140,8 @@ class FilterPyinotify(FileFilter):
 		"""
 		if not self.idle:
 			self.getFailures(path)
-			self.performBan()
+			if not self.banASAP: # pragma: no cover
+				self.performBan()
 			self.__modified = False
 
 	def _addPending(self, path, reason, isDir=False):

fail2ban/server/filtersystemd.py

@@ -318,7 +318,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 					else:
 						break
 				if self.__modified:
-					self.performBan()
+					if not self.banASAP: # pragma: no cover
+						self.performBan()
 					self.__modified = 0
 					# update position in log (time and iso string):
 					if self.jail.database is not None:

fail2ban/server/jailthread.py

@@ -120,3 +120,6 @@ class JailThread(Thread):
 ## python 2.x replace binding of private __bootstrap method:
 if sys.version_info < (3,): # pragma: 3.x no cover
 	JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap
+## python 3.9, restore isAlive method:
+elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover
+	 JailThread.isAlive = JailThread.is_alive

fail2ban/tests/filtertestcase.py

@@ -399,6 +399,7 @@ class IgnoreIP(LogCaptureTestCase):
 			self.filter.addFailRegex('^<HOST>')
 			self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s')
 			self.filter.setFindTime(10); # max 10 seconds back
+			self.filter.setMaxRetry(5); # don't ban here
 			#
 			self.pruneLog('[phase 1] DST time jump')
 			# check local time jump (DST hole):
@@ -757,6 +758,7 @@ class LogFileMonitor(LogCaptureTestCase):
 		_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
 		self.file = open(self.name, 'a')
 		self.filter = FilterPoll(DummyJail())
+		self.filter.banASAP = False # avoid immediate ban in this tests
 		self.filter.addLogPath(self.name, autoSeek=False)
 		self.filter.active = True
 		self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>")
@@ -974,6 +976,7 @@ def get_monitor_failures_testcase(Filter_):
 			self.file = open(self.name, 'a')
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
+			self.filter.banASAP = False # avoid immediate ban in this tests
 			self.filter.addLogPath(self.name, autoSeek=False)
 			# speedup search using exact date pattern:
 			self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
@@ -1272,6 +1275,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 		def _initFilter(self, **kwargs):
 			self._getRuntimeJournal() # check journal available
 			self.filter = Filter_(self.jail, **kwargs)
+			self.filter.banASAP = False # avoid immediate ban in this tests
 			self.filter.addJournalMatch([
 				"SYSLOG_IDENTIFIER=fail2ban-testcases",
 				"TEST_FIELD=1",
@@ -1525,6 +1529,7 @@ class GetFailures(LogCaptureTestCase):
 		setUpMyTime()
 		self.jail = DummyJail()
 		self.filter = FileFilter(self.jail)
+		self.filter.banASAP = False # avoid immediate ban in this tests
 		self.filter.active = True
 		# speedup search using exact date pattern:
 		self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?')
@@ -1714,6 +1719,7 @@ class GetFailures(LogCaptureTestCase):
 			self.pruneLog("[test-phase useDns=%s]" % useDns)
 			jail = DummyJail()
 			filter_ = FileFilter(jail, useDns=useDns)
+			filter_.banASAP = False # avoid immediate ban in this tests
 			filter_.active = True
 			filter_.failManager.setMaxRetry(1)	# we might have just few failures
 

fail2ban/tests/sockettestcase.py

@@ -87,7 +87,7 @@ class Socket(LogCaptureTestCase):
 	def _stopServerThread(self):
 		serverThread = self.serverThread
 		# wait for end of thread :
-		Utils.wait_for(lambda: not serverThread.isAlive() 
+		Utils.wait_for(lambda: not serverThread.is_alive() 
 			or serverThread.join(Utils.DEFAULT_SLEEP_TIME), unittest.F2B.maxWaitTime(10))
 		self.serverThread = None
 
@@ -98,7 +98,7 @@ class Socket(LogCaptureTestCase):
 		self.server.close()
 		# wait for end of thread :
 		self._stopServerThread()
-		self.assertFalse(serverThread.isAlive())
+		self.assertFalse(serverThread.is_alive())
 		# clean :
 		self.server.stop()
 		self.assertFalse(self.server.isActive())
@@ -139,7 +139,7 @@ class Socket(LogCaptureTestCase):
 		self.server.stop()
 		# wait for end of thread :
 		self._stopServerThread()
-		self.assertFalse(serverThread.isAlive())
+		self.assertFalse(serverThread.is_alive())
 		self.assertFalse(self.server.isActive())
 		self.assertFalse(os.path.exists(self.sock_name))
 
@@ -180,7 +180,7 @@ class Socket(LogCaptureTestCase):
 		self.server.stop()
 		# wait for end of thread :
 		self._stopServerThread()
-		self.assertFalse(serverThread.isAlive())
+		self.assertFalse(serverThread.is_alive())
 
 	def testLoopErrors(self):
 		# replace poll handler to produce error in loop-cycle:
@@ -216,7 +216,7 @@ class Socket(LogCaptureTestCase):
 		self.server.stop()
 		# wait for end of thread :
 		self._stopServerThread()
-		self.assertFalse(serverThread.isAlive())
+		self.assertFalse(serverThread.is_alive())
 		self.assertFalse(self.server.isActive())
 		self.assertFalse(os.path.exists(self.sock_name))