From 90da5a13a1e7556b00a8c9f3080e23e59b122704 Mon Sep 17 00:00:00 2001 From: guqing <38999863+guqing@users.noreply.github.com> Date: Thu, 14 Jul 2022 22:55:09 +0800 Subject: [PATCH] refactor: the way of plugin initialize load (#2242) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind improvement /area core /milestone 2.0 #### What this PR does / why we need it: 1. 优化插件初始化加载方式及 Plugin 自定义模型资源的更新 2. 插件 plugin.yaml 中 license 配置不再支持只配置字符串，而使用如下替代 ```yaml license: - name: "MIT" ``` 3. 可以在 application.yaml 中配置 ```yaml halo: initial-extension-locations: - "path/to/extensions/yaml" ``` 用于在系统启动时创建或更新自定义模型数据 #### Which issue(s) this PR fixes: Fixes # #### Special notes for your reviewer: /cc @halo-dev/sig-halo #### Does this PR introduce a user-facing change? ```release-note None ``` --- .../run/halo/app/core/extension/Plugin.java | 33 ++-- .../reconciler/PluginReconciler.java | 31 +++- .../infra/ExtensionResourceInitializer.java | 82 ++++++++++ .../app/infra/properties/HaloProperties.java | 6 +- .../halo/app/plugin/PluginLoadedListener.java | 35 ----- .../app/plugin/PluginStartedListener.java | 3 +- .../run/halo/app/plugin/YamlPluginFinder.java | 3 +- .../ExtensionResourceInitializerTest.java | 147 ++++++++++++++++++ .../halo/app/plugin/YamlPluginFinderTest.java | 46 +----- src/test/resources/plugin/plugin.yaml | 3 +- .../test-unstructured-resource-loader.jar | Bin 16472 -> 16768 bytes 11 files changed, 286 insertions(+), 103 deletions(-) create mode 100644 src/main/java/run/halo/app/infra/ExtensionResourceInitializer.java delete mode 100644 src/main/java/run/halo/app/plugin/PluginLoadedListener.java create mode 100644 src/test/java/run/halo/app/infra/ExtensionResourceInitializerTest.java diff --git a/src/main/java/run/halo/app/core/extension/Plugin.java b/src/main/java/run/halo/app/core/extension/Plugin.java index 2ef5daad1..53701e51b 100644 --- a/src/main/java/run/halo/app/core/extension/Plugin.java +++ b/src/main/java/run/halo/app/core/extension/Plugin.java @@ -1,17 +1,19 @@ package run.halo.app.core.extension; -import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.v3.oas.annotations.media.Schema; import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; import org.pf4j.PluginState; +import org.springframework.lang.NonNull; import run.halo.app.extension.AbstractExtension; import run.halo.app.extension.GVK; import run.halo.app.plugin.BasePlugin; @@ -34,6 +36,20 @@ public class Plugin extends AbstractExtension { private PluginStatus status; + /** + * Gets plugin status. + * + * @return empty object if status is null. + */ + @NonNull + @JsonIgnore + public PluginStatus statusNonNull() { + if (this.status == null) { + return new PluginStatus(); + } + return status; + } + @Data public static class PluginSpec { @@ -51,7 +67,6 @@ public class Plugin extends AbstractExtension { private String description; - @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) private List license; /** @@ -68,6 +83,12 @@ public class Plugin extends AbstractExtension { private String settingName; private String configMapName; + + @NonNull + @JsonIgnore + public List extensionLocationsNonNull() { + return Objects.requireNonNullElseGet(extensionLocations, List::of); + } } @Getter @@ -75,14 +96,6 @@ public class Plugin extends AbstractExtension { public static class License { private String name; private String url; - - public License() { - } - - public License(String name) { - this.name = name; - this.url = ""; - } } @Data diff --git a/src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java b/src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java index 1fe10cbff..0205a0596 100644 --- a/src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java +++ b/src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java @@ -10,12 +10,14 @@ import lombok.extern.slf4j.Slf4j; import org.pf4j.PluginRuntimeException; import org.pf4j.PluginState; import org.pf4j.PluginWrapper; +import org.pf4j.RuntimeMode; import run.halo.app.core.extension.Plugin; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.controller.Reconciler; import run.halo.app.infra.utils.JsonUtils; import run.halo.app.plugin.HaloPluginManager; import run.halo.app.plugin.PluginStartingError; +import run.halo.app.plugin.YamlPluginFinder; import run.halo.app.plugin.resources.JsBundleRuleProvider; import run.halo.app.plugin.resources.ReverseProxyRouterFunctionFactory; @@ -66,7 +68,7 @@ public class PluginReconciler implements Reconciler { } private void reconcilePluginState(Plugin plugin) { - Plugin.PluginStatus pluginStatus = plugin.getStatus(); + Plugin.PluginStatus pluginStatus = plugin.statusNonNull(); String name = plugin.getMetadata().getName(); PluginWrapper pluginWrapper = haloPluginManager.getPlugin(name); if (pluginWrapper == null) { @@ -81,6 +83,8 @@ public class PluginReconciler implements Reconciler { return; } + ensureSpecUpToDateWhenDevelopmentMode(pluginWrapper, plugin); + if (!Objects.equals(pluginStatus.getPhase(), pluginWrapper.getPluginState())) { // Set to the correct state pluginStatus.setPhase(pluginWrapper.getPluginState()); @@ -120,14 +124,14 @@ public class PluginReconciler implements Reconciler { private boolean shouldReconcileStartState(Plugin plugin) { return plugin.getSpec().getEnabled() - && plugin.getStatus().getPhase() != PluginState.STARTED; + && plugin.statusNonNull().getPhase() != PluginState.STARTED; } private void startPlugin(Plugin plugin) { String pluginName = plugin.getMetadata().getName(); PluginState currentState = haloPluginManager.startPlugin(pluginName); handleStatus(plugin, currentState, PluginState.STARTED); - Plugin.PluginStatus status = plugin.getStatus(); + Plugin.PluginStatus status = plugin.statusNonNull(); // TODO Check whether the JS bundle rule exists. If it does not exist, do not populate // populate stylesheet path String jsBundleRoute = ReverseProxyRouterFunctionFactory.buildRoutePath(pluginName, @@ -141,7 +145,7 @@ public class PluginReconciler implements Reconciler { private boolean shouldReconcileStopState(Plugin plugin) { return !plugin.getSpec().getEnabled() - && plugin.getStatus().getPhase() == PluginState.STARTED; + && plugin.statusNonNull().getPhase() == PluginState.STARTED; } private void stopPlugin(Plugin plugin) { @@ -152,10 +156,7 @@ public class PluginReconciler implements Reconciler { private void handleStatus(Plugin plugin, PluginState currentState, PluginState desiredState) { - Plugin.PluginStatus status = plugin.getStatus(); - if (status == null) { - status = new Plugin.PluginStatus(); - } + Plugin.PluginStatus status = plugin.statusNonNull(); status.setPhase(currentState); status.setLastTransitionTime(Instant.now()); if (desiredState.equals(currentState)) { @@ -169,4 +170,18 @@ public class PluginReconciler implements Reconciler { throw new PluginRuntimeException(startingError.getMessage()); } } + + private void ensureSpecUpToDateWhenDevelopmentMode(PluginWrapper pluginWrapper, + Plugin oldPlugin) { + if (!RuntimeMode.DEPLOYMENT.equals(pluginWrapper.getRuntimeMode())) { + return; + } + YamlPluginFinder yamlPluginFinder = new YamlPluginFinder(); + Plugin pluginFromPath = yamlPluginFinder.find(pluginWrapper.getPluginPath()); + // ensure plugin spec is up to date + Plugin.PluginSpec pluginSpec = JsonUtils.deepCopy(pluginFromPath.getSpec()); + if (!Objects.equals(oldPlugin.getSpec(), pluginSpec)) { + oldPlugin.setSpec(pluginSpec); + } + } } diff --git a/src/main/java/run/halo/app/infra/ExtensionResourceInitializer.java b/src/main/java/run/halo/app/infra/ExtensionResourceInitializer.java new file mode 100644 index 000000000..ee0657e0c --- /dev/null +++ b/src/main/java/run/halo/app/infra/ExtensionResourceInitializer.java @@ -0,0 +1,82 @@ +package run.halo.app.infra; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.thymeleaf.util.StringUtils; +import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.Unstructured; +import run.halo.app.infra.properties.HaloProperties; +import run.halo.app.infra.utils.YamlUnstructuredLoader; + +/** + *

Extension resources initializer.

+ *

Check whether {@link HaloProperties#getInitialExtensionLocations()} is configured + * When the system ready, and load resources according to it to creates {@link Unstructured}

+ * + * @author guqing + * @since 2.0.0 + */ +@Slf4j +@Component +public class ExtensionResourceInitializer implements ApplicationListener { + private final HaloProperties haloProperties; + private final ExtensionClient extensionClient; + + public ExtensionResourceInitializer(HaloProperties haloProperties, + ExtensionClient extensionClient) { + this.haloProperties = haloProperties; + this.extensionClient = extensionClient; + } + + @Override + public void onApplicationEvent(@NonNull ApplicationReadyEvent event) { + Set extensionLocations = haloProperties.getInitialExtensionLocations(); + if (!CollectionUtils.isEmpty(extensionLocations)) { + + Resource[] resources = extensionLocations.stream() + .map(this::listYamlFiles) + .flatMap(List::stream) + .toArray(Resource[]::new); + + log.debug("Initialization loaded [{}] resources to establish.", resources.length); + + new YamlUnstructuredLoader(resources).load() + .forEach(unstructured -> extensionClient.fetch(unstructured.groupVersionKind(), + unstructured.getMetadata().getName()) + .ifPresentOrElse(persisted -> { + unstructured.getMetadata() + .setVersion(persisted.getMetadata().getVersion()); + extensionClient.update(unstructured); + }, () -> extensionClient.create(unstructured))); + } + } + + private List listYamlFiles(String location) { + try (Stream walk = Files.walk(Paths.get(location))) { + return walk.filter(this::isYamlFile) + .map(path -> new FileSystemResource(path.toFile())) + .toList(); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private boolean isYamlFile(Path pathname) { + Path fileName = pathname.getFileName(); + return StringUtils.endsWith(fileName, ".yaml") + || StringUtils.endsWith(fileName, ".yml"); + } +} diff --git a/src/main/java/run/halo/app/infra/properties/HaloProperties.java b/src/main/java/run/halo/app/infra/properties/HaloProperties.java index b12e359ba..44abd46a0 100644 --- a/src/main/java/run/halo/app/infra/properties/HaloProperties.java +++ b/src/main/java/run/halo/app/infra/properties/HaloProperties.java @@ -1,12 +1,16 @@ package run.halo.app.infra.properties; +import java.util.Set; +import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; /** * @author guqing - * @date 2022-04-12 + * @since 2022-04-12 */ +@Data @ConfigurationProperties(prefix = "halo") public class HaloProperties { + private Set initialExtensionLocations; } diff --git a/src/main/java/run/halo/app/plugin/PluginLoadedListener.java b/src/main/java/run/halo/app/plugin/PluginLoadedListener.java deleted file mode 100644 index 64fbf7655..000000000 --- a/src/main/java/run/halo/app/plugin/PluginLoadedListener.java +++ /dev/null @@ -1,35 +0,0 @@ -package run.halo.app.plugin; - -import org.pf4j.PluginWrapper; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; -import run.halo.app.core.extension.Plugin; -import run.halo.app.extension.ExtensionClient; -import run.halo.app.plugin.event.HaloPluginLoadedEvent; - -/** - * @author guqing - * @since 2.0.0 - */ -@Component -public class PluginLoadedListener implements ApplicationListener { - private final ExtensionClient extensionClient; - - public PluginLoadedListener(ExtensionClient extensionClient) { - this.extensionClient = extensionClient; - } - - @Override - public void onApplicationEvent(HaloPluginLoadedEvent event) { - PluginWrapper pluginWrapper = event.getPluginWrapper(); - // TODO: Optimize plugin custom resource loading method - // load plugin.yaml - YamlPluginFinder yamlPluginFinder = new YamlPluginFinder(); - Plugin plugin = yamlPluginFinder.find(pluginWrapper.getPluginPath()); - extensionClient.fetch(Plugin.class, plugin.getMetadata().getName()) - .ifPresentOrElse(persisted -> { - plugin.getMetadata().setVersion(persisted.getMetadata().getVersion()); - extensionClient.update(plugin); - }, () -> extensionClient.create(plugin)); - } -} diff --git a/src/main/java/run/halo/app/plugin/PluginStartedListener.java b/src/main/java/run/halo/app/plugin/PluginStartedListener.java index 137ec4ff7..0c4f5985d 100644 --- a/src/main/java/run/halo/app/plugin/PluginStartedListener.java +++ b/src/main/java/run/halo/app/plugin/PluginStartedListener.java @@ -37,8 +37,7 @@ public class PluginStartedListener implements ApplicationListener new YamlUnstructuredLoader(resource).load()) diff --git a/src/main/java/run/halo/app/plugin/YamlPluginFinder.java b/src/main/java/run/halo/app/plugin/YamlPluginFinder.java index 3ec388ecd..c8f965abd 100644 --- a/src/main/java/run/halo/app/plugin/YamlPluginFinder.java +++ b/src/main/java/run/halo/app/plugin/YamlPluginFinder.java @@ -47,7 +47,8 @@ import run.halo.app.infra.utils.YamlUnstructuredLoader; * # 'displayName' explains what the plugin does in only a few words * displayName: "a name to show" * description: "Tell me more about this plugin." - * license: MIT + * license: + * - name: MIT * * * @author guqing diff --git a/src/test/java/run/halo/app/infra/ExtensionResourceInitializerTest.java b/src/test/java/run/halo/app/infra/ExtensionResourceInitializerTest.java new file mode 100644 index 000000000..4e7316e98 --- /dev/null +++ b/src/test/java/run/halo/app/infra/ExtensionResourceInitializerTest.java @@ -0,0 +1,147 @@ +package run.halo.app.infra; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.json.JSONException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.skyscreamer.jsonassert.JSONAssert; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.GroupVersionKind; +import run.halo.app.extension.Unstructured; +import run.halo.app.infra.properties.HaloProperties; +import run.halo.app.infra.utils.JsonUtils; + +/** + * Tests for {@link ExtensionResourceInitializer}. + * + * @author guqing + * @since 2.0.0 + */ +@ExtendWith(MockitoExtension.class) +class ExtensionResourceInitializerTest { + + @Mock + private ExtensionClient extensionClient; + @Mock + private HaloProperties haloProperties; + @Mock + private ApplicationReadyEvent applicationReadyEvent; + + private ExtensionResourceInitializer extensionResourceInitializer; + + @BeforeEach + void setUp() throws IOException { + extensionResourceInitializer = + new ExtensionResourceInitializer(haloProperties, extensionClient); + + Path tempDirectory = Files.createTempDirectory("extension-resource-initializer-test"); + Path multiDirectory = Files.createDirectories(tempDirectory.resolve("a/b/c")); + Files.writeString(tempDirectory.resolve("hello.yml"), """ + kind: FakeExtension + apiVersion: v1 + metadata: + name: fake-extension + spec: + hello: world + """, + StandardCharsets.UTF_8); + + Files.writeString(multiDirectory.getParent().resolve("fake-1.txt"), """ + kind: FakeExtension + name: fake-extension + """, + StandardCharsets.UTF_8); + Files.writeString(multiDirectory.resolve("fake.yaml"), """ + kind: FakeExtension + apiVersion: v1 + metadata: + name: fake-extension + spec: + hello: world + """, + StandardCharsets.UTF_8); + + // test file in directory + Path filePath = Files.createTempDirectory("extension-resource-file-test") + .resolve("good.yml"); + Files.writeString(filePath, """ + kind: FakeExtension + apiVersion: v1 + metadata: + name: config-file-is-ok + spec: + key: value + """, + StandardCharsets.UTF_8); + + when(haloProperties.getInitialExtensionLocations()) + .thenReturn(Set.of(tempDirectory.toString(), filePath.toString())); + } + + @Test + void onApplicationEvent() throws JSONException { + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Unstructured.class); + + when(extensionClient.fetch(any(GroupVersionKind.class), any())) + .thenReturn(Optional.empty()); + + extensionResourceInitializer.onApplicationEvent(applicationReadyEvent); + + verify(extensionClient, times(3)).create(argumentCaptor.capture()); + + List values = argumentCaptor.getAllValues(); + assertThat(values).isNotNull(); + assertThat(values).hasSize(3); + JSONAssert.assertEquals(""" + [ + { + "kind": "FakeExtension", + "apiVersion": "v1", + "metadata": { + "name": "config-file-is-ok" + }, + "spec": { + "key": "value" + } + }, + { + "kind": "FakeExtension", + "apiVersion": "v1", + "metadata": { + "name": "fake-extension" + }, + "spec": { + "hello": "world" + } + }, + { + "kind": "FakeExtension", + "apiVersion": "v1", + "metadata": { + "name": "fake-extension" + }, + "spec": { + "hello": "world" + } + } + ] + """, JsonUtils.objectToJson(values), false); + } +} \ No newline at end of file diff --git a/src/test/java/run/halo/app/plugin/YamlPluginFinderTest.java b/src/test/java/run/halo/app/plugin/YamlPluginFinderTest.java index 33ff0fd1b..8283c10d6 100644 --- a/src/test/java/run/halo/app/plugin/YamlPluginFinderTest.java +++ b/src/test/java/run/halo/app/plugin/YamlPluginFinderTest.java @@ -88,7 +88,7 @@ class YamlPluginFinderTest { "license": [ { "name": "MIT", - "url": "" + "url": null } ], "requires": ">=2.0.0", @@ -124,50 +124,6 @@ class YamlPluginFinderTest { .hasMessage("Unable to find plugin descriptor file: plugin.yaml"); } - @Test - void acceptArrayLicense() throws JSONException { - Resource pluginResource = new InMemoryResource(""" - apiVersion: v1 - kind: Plugin - metadata: - name: plugin-1 - spec: - license: "MIT" - """); - Plugin plugin = pluginFinder.unstructuredToPlugin(pluginResource); - assertThat(plugin.getSpec()).isNotNull(); - JSONAssert.assertEquals(""" - [{ - "name": "MIT", - "url": "" - }] - """, JsonUtils.objectToJson(plugin.getSpec().getLicense()), false); - } - - @Test - void acceptMultipleItemArrayLicense() throws JSONException { - Resource pluginResource = new InMemoryResource(""" - apiVersion: v1 - kind: Plugin - metadata: - name: plugin-1 - spec: - license: ["MIT", "Apache-2.0"] - """); - Plugin plugin = pluginFinder.unstructuredToPlugin(pluginResource); - assertThat(plugin.getSpec()).isNotNull(); - JSONAssert.assertEquals(""" - [{ - "name": "MIT", - "url": "" - }, - { - "name": "Apache-2.0", - "url": "" - }] - """, JsonUtils.objectToJson(plugin.getSpec().getLicense()), false); - } - @Test void acceptArrayObjectLicense() throws JSONException { Resource pluginResource = new InMemoryResource(""" diff --git a/src/test/resources/plugin/plugin.yaml b/src/test/resources/plugin/plugin.yaml index b29f43aab..939dba34a 100644 --- a/src/test/resources/plugin/plugin.yaml +++ b/src/test/resources/plugin/plugin.yaml @@ -13,4 +13,5 @@ spec: homepage: https://github.com/guqing/halo-plugin-1 displayName: "a name to show" description: "Tell me more about this plugin." - license: MIT \ No newline at end of file + license: + - name: MIT \ No newline at end of file diff --git a/src/test/resources/plugin/test-unstructured-resource-loader.jar b/src/test/resources/plugin/test-unstructured-resource-loader.jar index bca38cec250ed5e78afcc9e07acebba67c45305f..6825f73c062dce92d827263c60f5c25f8aa8de6a 100644 GIT binary patch delta 1336 zcmcc7z}V2tIKhm$>+<=@dzqyszcJ>S7_Y_-F14u+Z{HCb2O4XnFN+=dY< zsWth&gaw@KB54O_ua|U&uz{}Tmr4e)HEs?Wr%c-2}y`QJwpZU$JzRBn6#Y?AO&)F@t*IZ=viPz5Y zmI@PC={|c$YqDSxml5xG<%F-|+i$^%s}G9OmC826;ulzn}j1 z`Rco0XW#vLe)hQh@BI7T|Ic3E{@{;Ic*#tF}>r#*G_Gy z+#HetE}F?qUT1EPBrh{rz`_w~uFT{D3mB+TB zwlFy$|0i=+5J;OhJ5Vi1{bYWY3Xs_5E|xk*W;LKD9y6`Urff+tO`GSjo#JN(soX0j zHMv#H6v_^U&_D<1io1c>o6E#)7@?9{lkZDdz}YU6c5wE3NoNQfsF7bP8N}XPD%He< zB*-&aK{g63Q7an{me_n>{v!`qa zLPK~N*u5ARgS-d!qO^%@y+z?{@7=PZ0Sy{{Uy5I!tP)v&{NwRwPOgsbjV%sLDqbos z#iB1%xc>P}**(ATo`rhQ%{t@H_hzS;{Fzmgu#uOi*u?Mgo~TWsynBpRCP#l22~0e) zjH!a*`K$v;{tGe=PGic5bUw4-b~!V9g7t|>S01ZR&{-e!^8De)7a6?o7G9};x#-=c zuH5j&#cfWaGFx4ftVEnIPvl8-ILyWLHA8uOfre9uo&Vv(4klv8G2QRwuFg!_5&vS1 z-RkF(AGUbC(KIozocY9j*@~m)0Wv8EJk)t#NW^BOZEoIp`s{2;zRKeGoXnRe6WxP7 z6c-vw8{|x9T=a&y;?&2;T_>XanA|@~*H1VaXf*TjyaygDRW5qf3(Uvr$`SNi$o^LM}dFV%m3d-uQKVcnY(PR7~T zuYbo7!0XM1ByoYsE36?Y7otvJvZxI>sZWOSyKH<=)U(