Asynchronous Dynamic Module Loading Support (#1071)

apprise/manager.py

@@ -32,6 +32,7 @@ import sys
 import time
 import hashlib
 import inspect
+import threading
 from .utils import import_module
 from .utils import Singleton
 from .utils import parse_list
@@ -60,6 +61,9 @@ class PluginManager(metaclass=Singleton):
     # The module path to scan
     module_path = join(abspath(dirname(__file__)), _id)
 
+    # thread safe loading
+    _lock = threading.Lock()
+
     def __init__(self, *args, **kwargs):
         """
         Over-ride our class instantiation to provide a singleton
@@ -103,11 +107,15 @@ class PluginManager(metaclass=Singleton):
         # effort/overhead doing it again
         self._paths_previously_scanned = set()
 
+        # Track loaded module paths to prevent from loading them again
+        self._loaded = set()
+
     def unload_modules(self, disable_native=False):
         """
         Reset our object and unload all modules
         """
 
+        with self._lock:
             if self._custom_module_map:
                 # Handle Custom Module Assignments
                 for meta in self._custom_module_map.values():
@@ -129,14 +137,19 @@ class PluginManager(metaclass=Singleton):
             self._disabled.clear()
 
             # Reset our variables
-        self._module_map = None if not disable_native else {}
             self._schema_map = {}
             self._custom_module_map = {}
+            if disable_native:
+                self._module_map = {}
+
+            else:
+                self._module_map = None
+                self._loaded = set()
 
             # Reset our path cache
             self._paths_previously_scanned = set()
 
-    def load_modules(self, path=None, name=None):
+    def load_modules(self, path=None, name=None, force=False):
         """
         Load our modules into memory
         """
@@ -145,6 +158,15 @@ class PluginManager(metaclass=Singleton):
         module_name_prefix = self.module_name_prefix if name is None else name
         module_path = self.module_path if path is None else path
 
+        with self._lock:
+            if not force and module_path in self._loaded:
+                # We're done
+                return
+
+            # Our base reference
+            module_count = len(self._module_map) if self._module_map else 0
+            schema_count = len(self._schema_map) if self._schema_map else 0
+
             if not self:
                 # Initialize our maps
                 self._module_map = {}
@@ -152,9 +174,11 @@ class PluginManager(metaclass=Singleton):
                 self._custom_module_map = {}
 
             # Used for the detection of additional Notify Services objects
-        # The .py extension is optional as we support loading directories too
+            # The .py extension is optional as we support loading directories
+            # too
             module_re = re.compile(
-            r'^(?P<name>' + self.fname_prefix + r'[a-z0-9]+)(\.py)?$', re.I)
+                r'^(?P<name>' + self.fname_prefix + r'[a-z0-9]+)(\.py)?$',
+                re.I)
 
             t_start = time.time()
             for f in os.listdir(module_path):
@@ -227,7 +251,8 @@ class PluginManager(metaclass=Singleton):
                     if schema in self._schema_map:
                         logger.error(
                             "{} schema ({}) mismatch detected - {} to {}"
-                        .format(self.name, schema, self._schema_map, plugin))
+                            .format(self.name, schema, self._schema_map,
+                                    plugin))
                         continue
 
                     # Assign plugin
@@ -236,10 +261,16 @@ class PluginManager(metaclass=Singleton):
                 logger.trace(
                     '{} {} loaded in {:.6f}s'.format(
                         self.name, module_name, (time.time() - tl_start)))
+
+            # Track the directory loaded so we never load it again
+            self._loaded.add(module_path)
+
             logger.debug(
                 '{} {}(s) and {} Schema(s) loaded in {:.4f}s'
                 .format(
-                self.name, len(self._module_map), len(self._schema_map),
+                    self.name,
+                    len(self._module_map) - module_count,
+                    len(self._schema_map) - schema_count,
                     (time.time() - t_start)))
 
     def module_detection(self, paths, cache=True):
@@ -334,6 +365,7 @@ class PluginManager(metaclass=Singleton):
             # end of _import_module()
             return
 
+        with self._lock:
             for _path in paths:
                 path = os.path.abspath(os.path.expanduser(_path))
                 if (cache and path in self._paths_previously_scanned) \
@@ -366,8 +398,8 @@ class PluginManager(metaclass=Singleton):
                                 continue
 
                         if not cache or \
-                            (cache and
-                             new_path not in self._paths_previously_scanned):
+                                (cache and new_path not in
+                                 self._paths_previously_scanned):
                             # Load our module
                             _import_module(new_path)
 
@@ -375,8 +407,9 @@ class PluginManager(metaclass=Singleton):
                             self._paths_previously_scanned.add(new_path)
                 else:
                     if os.path.isdir(path):
-                    # This logic is safe to apply because we already validated
-                    # the directories state above; update our path
+                        # This logic is safe to apply because we already
+                        # validated the directories state above; update our
+                        # path
                         path = os.path.join(path, '__init__.py')
                         if cache and path in self._paths_previously_scanned:
                             continue
@@ -714,4 +747,4 @@ class PluginManager(metaclass=Singleton):
         """
         Determines if object has loaded or not
         """
-        return True if self._module_map is not None else False
+        return True if self._loaded and self._module_map is not None else False

test/test_notification_manager.py

@@ -29,8 +29,10 @@
 import re
 import pytest
 import types
+import threading
 from inspect import cleandoc
 
+from apprise import Apprise
 from apprise.NotificationManager import NotificationManager
 from apprise.plugins.NotifyBase import NotifyBase
 
@@ -248,6 +250,48 @@ def test_notification_manager_module_loading(tmpdir):
     N_MGR.load_modules()
     N_MGR.load_modules()
 
+    #
+    # Thread Testing
+    #
+
+    # This tests against a racing condition when the modules have not been
+    # loaded.  When multiple instances of Apprise are all instantiated,
+    # the loading of the modules will occur for each instance if detected
+    # having not been previously done, this tests that we can dynamically
+    # support the loading of modules once whe multiple instances to apprise
+    # are instantiated.
+    thread_count = 10
+
+    def thread_test(result, no):
+        """
+        Load our apprise object with valid URLs and store our result
+        """
+        apobj = Apprise()
+        result[no] = apobj.add('json://localhost') and \
+            apobj.add('form://localhost') and \
+            apobj.add('xml://localhost')
+
+    # Unload our modules
+    N_MGR.unload_modules()
+
+    # Prepare threads to load
+    results = [None] * thread_count
+    threads = [
+        threading.Thread(target=thread_test, args=(results, no))
+        for no in range(thread_count)
+    ]
+
+    # Verify we can safely load our modules in a thread safe environment
+    for t in threads:
+        t.start()
+
+    for t in threads:
+        t.join()
+
+    # Verify we loaded our urls in all threads successfully
+    for result in results:
+        assert result is True
+
 
 def test_notification_manager_decorators(tmpdir):
     """
@@ -376,6 +420,10 @@ def test_notification_manager_decorators(tmpdir):
     """))
     assert 'mytest' not in N_MGR
     N_MGR.load_modules(path=str(notify_base))
+
+    # It's still not loaded because the path has already been scanned
+    assert 'mytest' not in N_MGR
+    N_MGR.load_modules(path=str(notify_base), force=True)
     assert 'mytest' in N_MGR
 
     # Could not be loaded because the filename did not align with the class
@@ -387,3 +435,7 @@ def test_notification_manager_decorators(tmpdir):
     N_MGR.load_modules(path=str(notify_base))
     # Our item is still loaded as expected
     assert 'mytest' in N_MGR
+
+    # Simple test to make sure we can handle duplicate entries loaded
+    N_MGR.load_modules(path=str(notify_base), force=True)
+    N_MGR.load_modules(path=str(notify_base), force=True)