refactor: enhance cache management in plugin setting config (#6141)

#### What type of PR is this? /kind feature /area plugin /area core /milestone 2.17.x #### What this PR does / why we need it: 增强插件配置的缓存管理 1. 通过 SettingFetcher/ReactiveSettingFetcher 获取插件配置可以不在考虑获取数据的性能问题，当数据变更后会自动更新缓存 2. 现在你可以通过在插件中监听 `PluginConfigUpdatedEvent` 事件来做一些处理，它会在用户更改插件配置后被触发 #### Does this PR introduce a user-facing change? ```release-note 增强插件配置的缓存管理并支持通过监听 `PluginConfigUpdatedEvent` 事件做一些特殊处理 ```
2024-06-26 19:20:51 +08:00 · 2024-06-26 19:20:51 +08:00 · 68d428aa29
parent 59edade8bb
commit 68d428aa29
11 changed files with 327 additions and 75 deletions
--- a/api/src/main/java/run/halo/app/plugin/PluginConfigUpdatedEvent.java
+++ b/api/src/main/java/run/halo/app/plugin/PluginConfigUpdatedEvent.java
@ -0,0 +1,32 @@
+package run.halo.app.plugin;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Map;
+import lombok.Builder;
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+import run.halo.app.core.extension.Plugin;
+import run.halo.app.extension.ConfigMap;
+
+/**
+ * <p>Event that is triggered when the {@link ConfigMap } represented by
+ * {@link Plugin.PluginSpec#getConfigMapName()} in the {@link Plugin} is updated.</p>
+ * <p>has two properties, oldConfig and newConfig, which represent the {@link ConfigMap#getData()}
+ * property value of the {@link ConfigMap}.</p>
+ *
+ * @author guqing
+ * @since 2.17.0
+ */
+@Getter
+public class PluginConfigUpdatedEvent extends ApplicationEvent {
+    private final Map<String, JsonNode> oldConfig;
+    private final Map<String, JsonNode> newConfig;
+
+    @Builder
+    public PluginConfigUpdatedEvent(Object source, Map<String, JsonNode> oldConfig,
+        Map<String, JsonNode> newConfig) {
+        super(source);
+        this.oldConfig = oldConfig;
+        this.newConfig = newConfig;
+    }
+}
--- a/api/src/main/java/run/halo/app/plugin/PluginContext.java
+++ b/api/src/main/java/run/halo/app/plugin/PluginContext.java
@ -1,5 +1,6 @@
 package run.halo.app.plugin;

+import lombok.Builder;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import org.pf4j.RuntimeMode;
@ -17,10 +18,13 @@ import org.pf4j.RuntimeMode;
 * @since 2.10.0
 */
@Getter
+@Builder
@RequiredArgsConstructor
 public class PluginContext {
    private final String name;

+    private final String configMapName;
+
    private final String version;

    private final RuntimeMode runtimeMode;
--- a/application/src/main/java/run/halo/app/plugin/DefaultPluginApplicationContextFactory.java
+++ b/application/src/main/java/run/halo/app/plugin/DefaultPluginApplicationContextFactory.java
@ -1,5 +1,6 @@
 package run.halo.app.plugin;

+import static org.springframework.beans.factory.config.BeanDefinition.SCOPE_SINGLETON;
 import static org.springframework.util.ResourceUtils.CLASSPATH_URL_PREFIX;

 import java.io.IOException;
@ -101,10 +102,9 @@ public class DefaultPluginApplicationContextFactory implements PluginApplication

        rootContext.getBeanProvider(ReactiveExtensionClient.class)
            .ifUnique(client -> {
-                var reactiveSettingFetcher = new DefaultReactiveSettingFetcher(client, pluginId);
-                var settingFetcher = new DefaultSettingFetcher(reactiveSettingFetcher);
-                beanFactory.registerSingleton("reactiveSettingFetcher", reactiveSettingFetcher);
-                beanFactory.registerSingleton("settingFetcher", settingFetcher);
+                context.registerBean("reactiveSettingFetcher",
+                    DefaultReactiveSettingFetcher.class, bhd -> bhd.setScope(SCOPE_SINGLETON));
+                beanFactory.registerSingleton("settingFetcher", DefaultSettingFetcher.class);
            });

        rootContext.getBeanProvider(PluginRequestMappingHandlerMapping.class)
--- a/application/src/main/java/run/halo/app/plugin/DefaultPluginGetter.java
+++ b/application/src/main/java/run/halo/app/plugin/DefaultPluginGetter.java
@ -0,0 +1,29 @@
+package run.halo.app.plugin;
+
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+import run.halo.app.core.extension.Plugin;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.infra.exception.NotFoundException;
+
+/**
+ * Default implementation of {@link PluginGetter}.
+ *
+ * @author guqing
+ * @since 2.17.0
+ */
+@Component
+@RequiredArgsConstructor
+public class DefaultPluginGetter implements PluginGetter {
+    private final ExtensionClient client;
+
+    @Override
+    public Plugin getPlugin(String name) {
+        if (StringUtils.isBlank(name)) {
+            throw new IllegalArgumentException("Plugin name must not be blank");
+        }
+        return client.fetch(Plugin.class, name)
+            .orElseThrow(() -> new NotFoundException("Plugin not found"));
+    }
+}
--- a/application/src/main/java/run/halo/app/plugin/DefaultReactiveSettingFetcher.java
+++ b/application/src/main/java/run/halo/app/plugin/DefaultReactiveSettingFetcher.java
@ -1,16 +1,30 @@
 package run.halo.app.plugin;

+import static run.halo.app.extension.index.query.QueryFactory.equal;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.cache.Cache;
+import org.springframework.cache.CacheManager;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
 import org.springframework.lang.NonNull;
+import org.springframework.lang.Nullable;
 import reactor.core.publisher.Mono;
-import run.halo.app.core.extension.Plugin;
 import run.halo.app.extension.ConfigMap;
+import run.halo.app.extension.DefaultExtensionMatcher;
+import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.ReactiveExtensionClient;
+import run.halo.app.extension.controller.Controller;
+import run.halo.app.extension.controller.ControllerBuilder;
+import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.router.selector.FieldSelector;
 import run.halo.app.infra.utils.JsonParseException;
 import run.halo.app.infra.utils.JsonUtils;

@ -20,15 +34,36 @@ import run.halo.app.infra.utils.JsonUtils;
 * @author guqing
 * @since 2.0.0
 */
-public class DefaultReactiveSettingFetcher implements ReactiveSettingFetcher {
+public class DefaultReactiveSettingFetcher
+    implements ReactiveSettingFetcher, Reconciler<Reconciler.Request>, DisposableBean,
+    ApplicationContextAware {

    private final ReactiveExtensionClient client;

+    private final ExtensionClient blockingClient;
+
+    private final CacheManager cacheManager;
+
+    /**
+     * The application context of the plugin.
+     */
+    private ApplicationContext applicationContext;
+
    private final String pluginName;

-    public DefaultReactiveSettingFetcher(ReactiveExtensionClient client, String pluginName) {
+    private final String configMapName;
+
+    private final String cacheName;
+
+    public DefaultReactiveSettingFetcher(PluginContext pluginContext,
+        ReactiveExtensionClient client, ExtensionClient blockingClient,
+        CacheManager cacheManager) {
        this.client = client;
-        this.pluginName = pluginName;
+        this.pluginName = pluginContext.getName();
+        this.configMapName = pluginContext.getConfigMapName();
+        this.blockingClient = blockingClient;
+        this.cacheManager = cacheManager;
+        this.cacheName = buildCacheKey(pluginName);
    }

    @Override
@ -60,26 +95,31 @@ public class DefaultReactiveSettingFetcher implements ReactiveSettingFetcher {
            .defaultIfEmpty(JsonNodeFactory.instance.missingNode());
    }

-    private Mono<Map<String, JsonNode>> getValuesInternal() {
-        return configMap(pluginName)
-            .mapNotNull(ConfigMap::getData)
-            .map(data -> {
-                Map<String, JsonNode> result = new LinkedHashMap<>();
-                data.forEach((key, value) -> result.put(key, readTree(value)));
-                return result;
-            })
-            .defaultIfEmpty(Map.of());
-    }
-
-    private Mono<ConfigMap> configMap(String pluginName) {
-        return client.fetch(Plugin.class, pluginName)
-            .flatMap(plugin -> {
-                String configMapName = plugin.getSpec().getConfigMapName();
-                if (StringUtils.isBlank(configMapName)) {
-                    return Mono.empty();
-                }
-                return client.fetch(ConfigMap.class, plugin.getSpec().getConfigMapName());
-            });
+    Mono<Map<String, JsonNode>> getValuesInternal() {
+        var cache = getCache();
+        var cachedValue = getCachedConfigData(cache);
+        if (cachedValue != null) {
+            return Mono.justOrEmpty(cachedValue);
+        }
+        return Mono.defer(() -> {
+            // double check
+            var newCachedValue = getCachedConfigData(cache);
+            if (newCachedValue != null) {
+                return Mono.justOrEmpty(newCachedValue);
+            }
+            if (StringUtils.isBlank(configMapName)) {
+                return Mono.empty();
+            }
+            return client.fetch(ConfigMap.class, configMapName)
+                .mapNotNull(ConfigMap::getData)
+                .map(data -> {
+                    Map<String, JsonNode> result = new LinkedHashMap<>();
+                    data.forEach((key, value) -> result.put(key, readTree(value)));
+                    return result;
+                })
+                .defaultIfEmpty(Map.of())
+                .doOnNext(values -> cache.put(pluginName, values));
+        });
    }

    private JsonNode readTree(String json) {
@ -96,4 +136,76 @@ public class DefaultReactiveSettingFetcher implements ReactiveSettingFetcher {
    private <T> T convertValue(JsonNode jsonNode, Class<T> clazz) {
        return JsonUtils.DEFAULT_JSON_MAPPER.convertValue(jsonNode, clazz);
    }
+
+    @NonNull
+    private Cache getCache() {
+        var cache = cacheManager.getCache(cacheName);
+        if (cache == null) {
+            // should never happen
+            throw new IllegalStateException("Cache [" + cacheName + "] not found.");
+        }
+        return cache;
+    }
+
+    static String buildCacheKey(String pluginName) {
+        return "plugin-" + pluginName + "-configmap";
+    }
+
+    @Override
+    public Result reconcile(Request request) {
+        blockingClient.fetch(ConfigMap.class, configMapName)
+            .ifPresent(configMap -> {
+                var cache = getCache();
+                var existData = getCachedConfigData(cache);
+                var configMapData = configMap.getData();
+                Map<String, JsonNode> result = new LinkedHashMap<>();
+                if (configMapData != null) {
+                    configMapData.forEach((key, value) -> result.put(key, readTree(value)));
+                }
+                applicationContext.publishEvent(PluginConfigUpdatedEvent.builder()
+                    .source(this)
+                    .oldConfig(existData)
+                    .newConfig(result)
+                    .build());
+                // update cache
+                cache.put(pluginName, result);
+            });
+        return Result.doNotRetry();
+    }
+
+    @Nullable
+    @SuppressWarnings("unchecked")
+    private Map<String, JsonNode> getCachedConfigData(@NonNull Cache cache) {
+        var existData = cache.get(pluginName);
+        if (existData == null) {
+            return null;
+        }
+        return (Map<String, JsonNode>) existData.get();
+    }
+
+    @Override
+    public Controller setupWith(ControllerBuilder builder) {
+        var configMap = new ConfigMap();
+        var extensionMatcher =
+            DefaultExtensionMatcher.builder(blockingClient, configMap.groupVersionKind())
+                .fieldSelector(FieldSelector.of(equal("metadata.name", configMapName)))
+                .build();
+        return builder
+            .extension(configMap)
+            .syncAllOnStart(false)
+            .onAddMatcher(extensionMatcher)
+            .onUpdateMatcher(extensionMatcher)
+            .build();
+    }
+
+    @Override
+    public void destroy() {
+        getCache().invalidate();
+    }
+
+    @Override
+    public void setApplicationContext(@NonNull ApplicationContext applicationContext)
+        throws BeansException {
+        this.applicationContext = applicationContext;
+    }
 }
--- a/application/src/main/java/run/halo/app/plugin/HaloPluginManager.java
+++ b/application/src/main/java/run/halo/app/plugin/HaloPluginManager.java
@ -79,7 +79,8 @@ public class HaloPluginManager extends DefaultPluginManager implements SpringPlu
    @Override
    protected PluginFactory createPluginFactory() {
        var contextFactory = new DefaultPluginApplicationContextFactory(this);
-        return new SpringPluginFactory(contextFactory);
+        var pluginGetter = rootContext.getBean(PluginGetter.class);
+        return new SpringPluginFactory(contextFactory, pluginGetter);
    }

    @Override
--- a/application/src/main/java/run/halo/app/plugin/PluginGetter.java
+++ b/application/src/main/java/run/halo/app/plugin/PluginGetter.java
@ -0,0 +1,24 @@
+package run.halo.app.plugin;
+
+import run.halo.app.core.extension.Plugin;
+import run.halo.app.infra.exception.NotFoundException;
+
+/**
+ * An interface to get {@link Plugin} by name.
+ *
+ * @author guqing
+ * @since 2.17.0
+ */
+@FunctionalInterface
+public interface PluginGetter {
+
+    /**
+     * Get plugin by name.
+     *
+     * @param name plugin name must not be null
+     * @return plugin
+     * @throws IllegalArgumentException if plugin name is null
+     * @throws NotFoundException    if plugin not found
+     */
+    Plugin getPlugin(String name);
+}
--- a/application/src/main/java/run/halo/app/plugin/SharedApplicationContext.java
+++ b/application/src/main/java/run/halo/app/plugin/SharedApplicationContext.java
@ -1,15 +0,0 @@
-package run.halo.app.plugin;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.GenericApplicationContext;
-
-/**
- * <p>An {@link ApplicationContext} implementation shared by plugins.</p>
- * <p>Beans in the Core that need to be shared with plugins will be injected into this
- * {@link SharedApplicationContext}.</p>
- *
- * @author guqing
- * @since 2.0.0
- */
-public class SharedApplicationContext extends GenericApplicationContext {
-}
--- a/application/src/main/java/run/halo/app/plugin/SharedApplicationContextFactory.java
+++ b/application/src/main/java/run/halo/app/plugin/SharedApplicationContextFactory.java
@ -1,5 +1,6 @@
 package run.halo.app.plugin;

+import org.springframework.cache.CacheManager;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.GenericApplicationContext;
 import org.springframework.security.web.server.context.ServerSecurityContextRepository;
@ -56,6 +57,8 @@ public enum SharedApplicationContextFactory {
            rootContext.getBean(ExternalLinkProcessor.class));
        beanFactory.registerSingleton("postContentService",
            rootContext.getBean(PostContentService.class));
+        beanFactory.registerSingleton("cacheManager",
+            rootContext.getBean(CacheManager.class));
        // TODO add more shared instance here

        sharedContext.refresh();
--- a/application/src/main/java/run/halo/app/plugin/SpringPluginFactory.java
+++ b/application/src/main/java/run/halo/app/plugin/SpringPluginFactory.java
@ -16,24 +16,27 @@ import org.pf4j.PluginWrapper;
@Slf4j
 public class SpringPluginFactory implements PluginFactory {

-
    private final PluginApplicationContextFactory contextFactory;
+    private final PluginGetter pluginGetter;

-    public SpringPluginFactory(PluginApplicationContextFactory contextFactory) {
+    public SpringPluginFactory(PluginApplicationContextFactory contextFactory,
+        PluginGetter pluginGetter) {
        this.contextFactory = contextFactory;
+        this.pluginGetter = pluginGetter;
    }

    @Override
    public Plugin create(PluginWrapper pluginWrapper) {
-        var pluginContext = new PluginContext(
-            pluginWrapper.getPluginId(),
-            pluginWrapper.getDescriptor().getVersion(),
-            pluginWrapper.getRuntimeMode()
-        );
+        var plugin = pluginGetter.getPlugin(pluginWrapper.getPluginId());
+        var pluginContext = PluginContext.builder()
+            .name(pluginWrapper.getPluginId())
+            .configMapName(plugin.getSpec().getConfigMapName())
+            .version(pluginWrapper.getDescriptor().getVersion())
+            .runtimeMode(pluginWrapper.getRuntimeMode())
+            .build();
        return new SpringPlugin(
            contextFactory,
            pluginContext
        );
    }
-
 }
--- a/application/src/test/java/run/halo/app/plugin/DefaultSettingFetcherTest.java
+++ b/application/src/test/java/run/halo/app/plugin/DefaultSettingFetcherTest.java
@ -6,8 +6,10 @@ import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static run.halo.app.plugin.DefaultReactiveSettingFetcher.buildCacheKey;

 import com.fasterxml.jackson.databind.JsonNode;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@ -18,11 +20,18 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.skyscreamer.jsonassert.JSONAssert;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.cache.Cache;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.concurrent.ConcurrentMapCache;
+import org.springframework.context.ApplicationContext;
 import reactor.core.publisher.Mono;
 import run.halo.app.core.extension.Plugin;
 import run.halo.app.extension.ConfigMap;
+import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.Metadata;
 import run.halo.app.extension.ReactiveExtensionClient;
+import run.halo.app.extension.controller.Reconciler;
 import run.halo.app.infra.utils.JsonUtils;

 /**
@ -35,31 +44,50 @@ import run.halo.app.infra.utils.JsonUtils;
 class DefaultSettingFetcherTest {

    @Mock
-    private ReactiveExtensionClient extensionClient;
+    private ReactiveExtensionClient client;

+    @Mock
+    private ExtensionClient blockingClient;
+
+    @Mock
+    private CacheManager cacheManager;
+
+    @MockBean
+    private final PluginContext pluginContext = PluginContext.builder()
+        .name("fake")
+        .configMapName("fake-config")
+        .build();
+
+    @Mock
+    private ApplicationContext applicationContext;
+
+    private DefaultReactiveSettingFetcher reactiveSettingFetcher;
    private DefaultSettingFetcher settingFetcher;

+    private final Cache cache = new ConcurrentMapCache(buildCacheKey(pluginContext.getName()));
+
    @BeforeEach
    void setUp() {
-        DefaultReactiveSettingFetcher reactiveSettingFetcher =
-            new DefaultReactiveSettingFetcher(extensionClient, "fake");
-        settingFetcher = new DefaultSettingFetcher(reactiveSettingFetcher);
-        // do not call extensionClient when the settingFetcher first time created
-        verify(extensionClient, times(0)).fetch(eq(ConfigMap.class), any());
-        verify(extensionClient, times(0)).fetch(eq(Plugin.class), any());
+        cache.invalidate();

-        Plugin plugin = buildPlugin();
-        when(extensionClient.fetch(eq(Plugin.class), any())).thenReturn(Mono.just(plugin));
+        this.reactiveSettingFetcher = new DefaultReactiveSettingFetcher(pluginContext, client,
+            blockingClient, cacheManager);
+        reactiveSettingFetcher.setApplicationContext(applicationContext);
+
+        settingFetcher = new DefaultSettingFetcher(reactiveSettingFetcher);
+
+        when(cacheManager.getCache(eq(cache.getName()))).thenReturn(cache);

        ConfigMap configMap = buildConfigMap();
-        when(extensionClient.fetch(eq(ConfigMap.class), any())).thenReturn(Mono.just(configMap));
+        when(client.fetch(eq(ConfigMap.class), eq(pluginContext.getConfigMapName())))
+            .thenReturn(Mono.just(configMap));
    }

    @Test
    void getValues() throws JSONException {
        Map<String, JsonNode> values = settingFetcher.getValues();

-        verify(extensionClient, times(1)).fetch(eq(ConfigMap.class), any());
+        verify(client, times(1)).fetch(eq(ConfigMap.class), any());

        assertThat(values).hasSize(2);
        JSONAssert.assertEquals(getSns(), JsonUtils.objectToJson(values.get("sns")), true);
@ -67,6 +95,40 @@ class DefaultSettingFetcherTest {
        // The extensionClient will only be called once
        Map<String, JsonNode> callAgain = settingFetcher.getValues();
        assertThat(callAgain).isNotNull();
+        verify(client, times(1)).fetch(eq(ConfigMap.class), any());
+    }
+
+    @Test
+    void getValuesWithUpdateCache() throws JSONException {
+        Map<String, JsonNode> values = settingFetcher.getValues();
+
+        verify(client, times(1)).fetch(eq(ConfigMap.class), any());
+        JSONAssert.assertEquals(getSns(), JsonUtils.objectToJson(values.get("sns")), true);
+
+        ConfigMap configMap = buildConfigMap();
+        configMap.getData().put("sns", """
+            {
+                "email": "abc@example.com",
+                "github": "abc"
+            }
+            """);
+        when(blockingClient.fetch(eq(ConfigMap.class), eq(pluginContext.getConfigMapName())))
+            .thenReturn(Optional.of(configMap));
+        reactiveSettingFetcher.reconcile(new Reconciler.Request(pluginContext.getConfigMapName()));
+        verify(applicationContext).publishEvent(any());
+
+        Map<String, JsonNode> updatedValues = settingFetcher.getValues();
+        verify(client, times(1)).fetch(eq(ConfigMap.class), any());
+        assertThat(updatedValues).hasSize(2);
+        JSONAssert.assertEquals(configMap.getData().get("sns"),
+            JsonUtils.objectToJson(updatedValues.get("sns")), true);
+
+        // cleanup cache
+        reactiveSettingFetcher.destroy();
+
+        updatedValues = settingFetcher.getValues();
+        assertThat(updatedValues).hasSize(2);
+        verify(client, times(2)).fetch(eq(ConfigMap.class), any());
    }

    @Test
@ -74,10 +136,6 @@ class DefaultSettingFetcherTest {
        Optional<Sns> sns = settingFetcher.fetch("sns", Sns.class);
        assertThat(sns.isEmpty()).isFalse();
        JSONAssert.assertEquals(getSns(), JsonUtils.objectToJson(sns.get()), true);
-
-        when(extensionClient.fetch(eq(ConfigMap.class), any())).thenReturn(Mono.empty());
-        Optional<Sns> missing = settingFetcher.fetch("sns1", Sns.class);
-        assertThat(missing.isEmpty()).isTrue();
    }

    @Test
@ -101,14 +159,15 @@ class DefaultSettingFetcherTest {
        configMap.setMetadata(metadata);
        configMap.setKind("ConfigMap");
        configMap.setApiVersion("v1alpha1");
-        configMap.setData(Map.of("sns", getSns(),
-            "basic", """
-                {
-                     "color": "red",
-                     "width": "100"
-                 }
-                """)
-        );
+        var map = new HashMap<String, String>();
+        map.put("sns", getSns());
+        map.put("basic", """
+            {
+                "color": "red",
+                "width": "100"
+            }
+            """);
+        configMap.setData(map);
        return configMap;
    }