pyinotify/polling: test filter reaction by delete of watching file, better detection of pending file (avoid errors in fail2ban.log during log-rotation).

Closes gh-1865 for filterpyinotify ("cannot remove WD=2").

fail2ban/server/filterpoll.py

@@ -164,9 +164,13 @@ class FilterPoll(FileFilter):
 				return False
 			# log error:
 			if self.__file404Cnt[filename] < 2:
-				logSys.error("Unable to get stat on %s because of: %s",
+				if e.errno == 2:
+					logSys.debug("Log absence detected (possibly rotation) for %s, reason: %s",
+							 filename, e)
+				else: # pragma: no cover
+					logSys.error("Unable to get stat on %s because of: %s",
 							 filename, e, 
-							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+							 exc_info=e.errno != 2 and logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
 			self.commonError()
@@ -175,3 +179,6 @@ class FilterPoll(FileFilter):
 				self.__file404Cnt[filename] = 0
 				self.delLogPath(filename)
 			return False
+
+	def getPendingPaths(self):
+		return self.__file404Cnt.keys()

fail2ban/server/filterpyinotify.py

@@ -161,6 +161,9 @@ class FilterPyinotify(FileFilter):
 			del self.__pending[path]
 		except KeyError: pass
 
+	def getPendingPaths(self):
+		return self.__pending.keys()
+
 	def _checkPending(self):
 		if not self.__pending:
 			return
@@ -225,7 +228,11 @@ class FilterPyinotify(FileFilter):
 	def _delFileWatcher(self, path):
 		try:
 			wdInt = self.__watchFiles.pop(path)
-			wd = self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				wd = self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for file %s", wdInt, path)
+				wd = {wdInt: 1}
 			if wd[wdInt]:
 				logSys.debug("Removed file watcher for %s", path)
 				return True
@@ -249,7 +256,10 @@ class FilterPyinotify(FileFilter):
 		# Remove watches for the directory:
 		try:
 			wdInt = self.__watchDirs.pop(path_dir)
-			self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for directory %s", wdInt, path_dir)
 		except KeyError: # pragma: no cover
 			pass
 			# EnvironmentError is parent of IOError, OSError, etc.

fail2ban/tests/filtertestcase.py

@@ -980,6 +980,18 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)
 
+		def test_del_file(self):
+			# test filter reaction by delete watching file:
+			self.file.close()
+			self.waitForTicks(1)
+			# remove file (cause detection of log-rotation)...
+			os.unlink(self.name)
+			# check it was detected (in pending files):
+			self.waitForTicks(2)
+			if hasattr(self.filter, "getPendingPaths"):
+				self.assertTrue(Utils.wait_for(lambda: self.name in self.filter.getPendingPaths(), _maxWaitTime(10)))
+				self.assertEqual(len(self.filter.getPendingPaths()), 1)
+
 		@with_tmpdir
 		def test_move_dir(self, tmp):
 			self.file.close()