diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 6d0172da..9796e26f 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -188,7 +188,8 @@ class FilterPyinotify(FileFilter):
 		for path, isDir in found.iteritems():
 			self._delPending(path)
 			# refresh monitoring of this:
-			self._refreshWatcher(path, isDir=isDir)
+			if isDir is not None:
+				self._refreshWatcher(path, isDir=isDir)
 			if isDir:
 				# check all files belong to this dir:
 				for logpath in self.__watchFiles:
@@ -276,8 +277,8 @@ class FilterPyinotify(FileFilter):
 			# we can execute it right now:
 			self._process_file(path)
 		else:
-			# retard until filter gets started:
-			self._addPending(path, ('INITIAL', path))
+			# retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor):
+			self._addPending(path, ('INITIAL', path), isDir=None)
 
     ##
 	# Delete a log path
@@ -285,9 +286,9 @@ class FilterPyinotify(FileFilter):
 	# @param path the log file to delete
 
 	def _delLogPath(self, path):
+		self._delPending(path)
 		if not self._delFileWatcher(path): # pragma: no cover
 			logSys.error("Failed to remove watch on path: %s", path)
-		self._delPending(path)
 
 		path_dir = dirname(path)
 		for k in self.__watchFiles:
@@ -297,8 +298,8 @@ class FilterPyinotify(FileFilter):
 		if path_dir:
 			# Remove watches for the directory
 			# since there is no other monitored file under this directory
-			self._delDirWatcher(path_dir)
 			self._delPending(path_dir)
+			self._delDirWatcher(path_dir)
 
 	# pyinotify.ProcessEvent default handler:
 	def __process_default(self, event):