From a4459765ef438db83a2898ba832ff7acba033e29 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 Oct 2017 14:16:14 +0200
Subject: [PATCH 1/2] pyinotify/polling: test filter reaction by delete of
 watching file, better detection of pending file (avoid errors in fail2ban.log
 during log-rotation). Closes gh-1865 for filterpyinotify ("cannot remove
 WD=2").

---
 fail2ban/server/filterpoll.py      | 11 +++++++++--
 fail2ban/server/filterpyinotify.py | 14 ++++++++++++--
 fail2ban/tests/filtertestcase.py   | 12 ++++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index e785a0e4..5b736a4e 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -164,9 +164,13 @@ class FilterPoll(FileFilter):
 				return False
 			# log error:
 			if self.__file404Cnt[filename] < 2:
-				logSys.error("Unable to get stat on %s because of: %s",
+				if e.errno == 2:
+					logSys.debug("Log absence detected (possibly rotation) for %s, reason: %s",
+							 filename, e)
+				else: # pragma: no cover
+					logSys.error("Unable to get stat on %s because of: %s",
 							 filename, e, 
-							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+							 exc_info=e.errno != 2 and logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
 			self.commonError()
@@ -175,3 +179,6 @@ class FilterPoll(FileFilter):
 				self.__file404Cnt[filename] = 0
 				self.delLogPath(filename)
 			return False
+
+	def getPendingPaths(self):
+		return self.__file404Cnt.keys()
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 10f51847..e59c203e 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -161,6 +161,9 @@ class FilterPyinotify(FileFilter):
 			del self.__pending[path]
 		except KeyError: pass
 
+	def getPendingPaths(self):
+		return self.__pending.keys()
+
 	def _checkPending(self):
 		if not self.__pending:
 			return
@@ -225,7 +228,11 @@ class FilterPyinotify(FileFilter):
 	def _delFileWatcher(self, path):
 		try:
 			wdInt = self.__watchFiles.pop(path)
-			wd = self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				wd = self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for file %s", wdInt, path)
+				wd = {wdInt: 1}
 			if wd[wdInt]:
 				logSys.debug("Removed file watcher for %s", path)
 				return True
@@ -249,7 +256,10 @@ class FilterPyinotify(FileFilter):
 		# Remove watches for the directory:
 		try:
 			wdInt = self.__watchDirs.pop(path_dir)
-			self.__monitor.rm_watch(wdInt)
+			if self.__monitor.get_path(wdInt) is not None:
+				self.__monitor.rm_watch(wdInt)
+			else: # pragma: no cover
+				logSys.debug("Non-existing file watcher %r for directory %s", wdInt, path_dir)
 		except KeyError: # pragma: no cover
 			pass
 			# EnvironmentError is parent of IOError, OSError, etc.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 52187b3d..f8912fef 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -980,6 +980,18 @@ def get_monitor_failures_testcase(Filter_):
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)
 			self.assertEqual(self.filter.failManager.getFailTotal(), 6)
 
+		def test_del_file(self):
+			# test filter reaction by delete watching file:
+			self.file.close()
+			self.waitForTicks(1)
+			# remove file (cause detection of log-rotation)...
+			os.unlink(self.name)
+			# check it was detected (in pending files):
+			self.waitForTicks(2)
+			if hasattr(self.filter, "getPendingPaths"):
+				self.assertTrue(Utils.wait_for(lambda: self.name in self.filter.getPendingPaths(), _maxWaitTime(10)))
+				self.assertEqual(len(self.filter.getPendingPaths()), 1)
+
 		@with_tmpdir
 		def test_move_dir(self, tmp):
 			self.file.close()

From ceff489a46396b561d4a890ae75213e266837a04 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Wed, 4 Oct 2017 14:24:21 +0200
Subject: [PATCH 2/2] amend to a4459765ef438db83a2898ba832ff7acba033e29:
 irrelevant condition removed

---
 fail2ban/server/filterpoll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py
index 5b736a4e..62cf76f0 100644
--- a/fail2ban/server/filterpoll.py
+++ b/fail2ban/server/filterpoll.py
@@ -170,7 +170,7 @@ class FilterPoll(FileFilter):
 				else: # pragma: no cover
 					logSys.error("Unable to get stat on %s because of: %s",
 							 filename, e, 
-							 exc_info=e.errno != 2 and logSys.getEffectiveLevel()<=logging.DEBUG)
+							 exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			# increase file and common error counters:
 			self.__file404Cnt[filename] += 1
 			self.commonError()