refactor: optimizing plugin upgrade steps (#3838)

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.5.x

#### What this PR does / why we need it:
优化插件的升级流程

how to test it?
1. 测试正常的插件升级是否正常
2. 测试插件升级失败后插件是否会被卸载的问题
3. 测试没有 version 的插件安装是否能成功
4. 在插件目录不会多出一个名为 `{升级插件名称}-null.jar` 的文件

#### Which issue(s) this PR fixes:

Fixes #3839

#### Does this PR introduce a user-facing 
```release-note
优化插件的升级流程
```

api/src/main/java/run/halo/app/core/extension/Plugin.java

@@ -136,9 +136,4 @@ public class Plugin extends AbstractExtension {
 
         private String website;
     }
-
-    @JsonIgnore
-    public String generateFileName() {
-        return String.format("%s-%s.jar", getMetadata().getName(), spec.getVersion());
-    }
 }

application/src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java

@@ -1,23 +1,27 @@
 package run.halo.app.core.extension.reconciler;
 
 import static org.pf4j.util.FileUtils.isJarFile;
+import static run.halo.app.extension.MetadataUtil.nullSafeAnnotations;
+import static run.halo.app.extension.MetadataUtil.nullSafeLabels;
 import static run.halo.app.plugin.PluginConst.DELETE_STAGE;
+import static run.halo.app.plugin.PluginConst.PLUGIN_PATH;
+import static run.halo.app.plugin.PluginConst.RELOAD_ANNO;
 
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.BooleanUtils;
@@ -43,7 +47,6 @@ import run.halo.app.core.extension.theme.SettingUtils;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.GroupVersionKind;
 import run.halo.app.extension.Metadata;
-import run.halo.app.extension.MetadataUtil;
 import run.halo.app.extension.Unstructured;
 import run.halo.app.extension.controller.Controller;
 import run.halo.app.extension.controller.ControllerBuilder;
@@ -57,8 +60,8 @@ import run.halo.app.infra.utils.YamlUnstructuredLoader;
 import run.halo.app.plugin.HaloPluginManager;
 import run.halo.app.plugin.PluginConst;
 import run.halo.app.plugin.PluginExtensionLoaderUtils;
-import run.halo.app.plugin.PluginNotFoundException;
 import run.halo.app.plugin.PluginStartingError;
+import run.halo.app.plugin.YamlPluginFinder;
 import run.halo.app.plugin.event.PluginCreatedEvent;
 import run.halo.app.plugin.resources.BundleResourceUtils;
 
@@ -84,26 +87,52 @@ public class PluginReconciler implements Reconciler<Request> {
 
     @Override
     public Result reconcile(Request request) {
-        return client.fetch(Plugin.class, request.name())
+        try {
-            .map(plugin -> {
+            return client.fetch(Plugin.class, request.name())
-                if (plugin.getMetadata().getDeletionTimestamp() != null) {
+                .map(plugin -> {
-                    cleanUpResourcesAndRemoveFinalizer(request.name());
+                    if (plugin.getMetadata().getDeletionTimestamp() != null) {
+                        cleanUpResourcesAndRemoveFinalizer(request.name());
+                        return Result.doNotRetry();
+                    }
+                    addFinalizerIfNecessary(plugin);
+
+                    // if true returned, it means it is not ready
+                    if (readinessDetection(request.name())) {
+                        return new Result(true, null);
+                    }
+
+                    reconcilePluginState(plugin.getMetadata().getName());
                     return Result.doNotRetry();
-                }
+                })
-                addFinalizerIfNecessary(plugin);
+                .orElse(Result.doNotRetry());
+        } catch (DoNotRetryException e) {
+            persistenceFailureStatus(request.name(), e);
+            return Result.doNotRetry();
+        }
+    }
 
-                // if true returned, it means it is not ready
+    private void updatePluginPathAnno(String name) {
-                if (readinessDetection(request.name())) {
+        // TODO do it in a better way
-                    return new Result(true, null);
+        client.fetch(Plugin.class, name).ifPresent(plugin -> {
+            Map<String, String> annotations = nullSafeAnnotations(plugin);
+            String oldPluginPath = annotations.get(PLUGIN_PATH);
+            String pluginPath = oldPluginPath;
+            if (StringUtils.isBlank(oldPluginPath)) {
+                URI loadLocation = plugin.statusNonNull().getLoadLocation();
+                if (loadLocation == null) {
+                    throw new DoNotRetryException("Can not determine plugin path: " + name);
                 }
-
+                pluginPath = loadLocation.getPath();
-                reconcilePluginState(plugin.getMetadata().getName());
+            }
-                return Result.doNotRetry();
+            annotations.put(PLUGIN_PATH, pluginPath);
-            })
+            if (!StringUtils.equals(pluginPath, oldPluginPath)) {
-            .orElse(Result.doNotRetry());
+                client.update(plugin);
+            }
+        });
     }
 
     boolean readinessDetection(String name) {
+        updatePluginPathAnno(name);
         return client.fetch(Plugin.class, name)
             .map(plugin -> {
                 if (waitForSettingCreation(plugin)) {
@@ -115,8 +144,8 @@ public class PluginReconciler implements Reconciler<Request> {
                 generateAccessibleLogoUrl(plugin);
 
                 // update phase
-                PluginWrapper pluginWrapper = getPluginWrapper(name);
                 Plugin.PluginStatus status = plugin.statusNonNull();
+                PluginWrapper pluginWrapper = getPluginWrapper(name);
                 status.setPhase(pluginWrapper.getPluginState());
                 updateStatus(plugin.getMetadata().getName(), status);
                 return false;
@@ -186,7 +215,7 @@ public class PluginReconciler implements Reconciler<Request> {
         Optional<Setting> settingOption = lookupPluginSetting(pluginName, settingName)
             .map(setting -> {
                 // This annotation is added to prevent it from being deleted when stopped.
-                Map<String, String> settingAnnotations = MetadataUtil.nullSafeAnnotations(setting);
+                Map<String, String> settingAnnotations = nullSafeAnnotations(setting);
                 settingAnnotations.put(DELETE_STAGE, PluginConst.DeleteStage.UNINSTALL.name());
                 return setting;
             })
@@ -232,7 +261,7 @@ public class PluginReconciler implements Reconciler<Request> {
         stateTransition(name, currentState -> {
             boolean termination = false;
             switch (currentState) {
-                case CREATED -> ensurePluginLoaded();
+                case CREATED -> getPluginWrapper(name);
                 case STARTED -> termination = true;
                 // plugin can be started when it is stopped or failed
                 case RESOLVED, STOPPED, FAILED -> doStart(name);
@@ -247,7 +276,7 @@ public class PluginReconciler implements Reconciler<Request> {
         stateTransition(name, currentState -> {
             boolean termination = false;
             switch (currentState) {
-                case CREATED -> ensurePluginLoaded();
+                case CREATED -> getPluginWrapper(name);
                 case RESOLVED, STARTED -> doStop(name);
                 case FAILED, STOPPED -> termination = true;
                 default -> {
@@ -283,10 +312,8 @@ public class PluginReconciler implements Reconciler<Request> {
         if (currentState != desiredState) {
             log.error("Plugin [{}] state transition failed: {}", name,
                 haloPluginManager.getPluginStartingError(name));
-            var e = new PluginRuntimeException("Plugin [" + name + "] state transition from ["
+            throw new DoNotRetryException("Plugin [" + name + "] state transition from ["
                 + currentState + "] to [" + desiredState + "] failed");
-            persistenceFailureStatus(name, e);
-            throw e;
         }
     }
 
@@ -317,7 +344,7 @@ public class PluginReconciler implements Reconciler<Request> {
     private PluginWrapper getPluginWrapper(String name) {
         PluginWrapper pluginWrapper = haloPluginManager.getPlugin(name);
         if (pluginWrapper == null) {
-            ensurePluginLoaded();
+            ensurePluginLoaded(name);
             pluginWrapper = haloPluginManager.getPlugin(name);
         }
 
@@ -336,7 +363,7 @@ public class PluginReconciler implements Reconciler<Request> {
             Plugin.PluginStatus.nullSafeConditions(status)
                 .addAndEvictFIFO(condition);
             updateStatus(name, status);
-            throw new PluginNotFoundException(errorMsg);
+            throw new DoNotRetryException(errorMsg);
         }
         return pluginWrapper;
     }
@@ -348,6 +375,17 @@ public class PluginReconciler implements Reconciler<Request> {
         client.fetch(Plugin.class, name).ifPresent(plugin -> {
             Plugin.PluginStatus oldStatus = JsonUtils.deepCopy(plugin.statusNonNull());
             plugin.setStatus(status);
+            URI loadLocation = status.getLoadLocation();
+            if (loadLocation == null) {
+                String pluginPath = nullSafeAnnotations(plugin).get(PLUGIN_PATH);
+                if (StringUtils.isNotBlank(pluginPath)) {
+                    String absolutePath = buildPluginLocation(name, pluginPath);
+                    loadLocation = toUri(absolutePath);
+                } else {
+                    loadLocation = getPluginWrapper(name).getPluginPath().toUri();
+                }
+                status.setLoadLocation(loadLocation);
+            }
             if (!Objects.equals(oldStatus, status)) {
                 client.update(plugin);
             }
@@ -465,11 +503,15 @@ public class PluginReconciler implements Reconciler<Request> {
     }
 
     private void reconcilePluginState(String name) {
-        if (haloPluginManager.getPlugin(name) == null) {
-            ensurePluginLoaded();
-        }
-
         client.fetch(Plugin.class, name).ifPresent(plugin -> {
+            // reload detection
+            Map<String, String> annotations = nullSafeAnnotations(plugin);
+            if (annotations.containsKey(RELOAD_ANNO)) {
+                reload(plugin);
+                // update will requeue to make next reconciliation
+                return;
+            }
+
             // Transition plugin status if necessary
             if (shouldReconcileStartState(plugin)) {
                 startAction(name);
@@ -481,20 +523,117 @@ public class PluginReconciler implements Reconciler<Request> {
         });
     }
 
-    private void ensurePluginLoaded() {
+    void reload(Plugin plugin) {
-        // load plugin if exists in plugin root paths.
+        String newPluginPath = nullSafeAnnotations(plugin).get(RELOAD_ANNO);
-        List<PluginWrapper> loadedPlugins = haloPluginManager.getPlugins();
+        if (StringUtils.isBlank(newPluginPath)) {
-        Map<Path, PluginWrapper> loadedPluginWrapperMap = loadedPlugins.stream()
+            return;
-            .collect(Collectors.toMap(PluginWrapper::getPluginPath, item -> item));
+        }
-        haloPluginManager.getPluginRepository()
+        final String pluginName = plugin.getMetadata().getName();
-            .getPluginPaths()
+        URI oldPluginLocation = plugin.statusNonNull().getLoadLocation();
-            .forEach(path -> {
+        if (shouldDeleteFile(newPluginPath, oldPluginLocation)) {
-                if (!loadedPluginWrapperMap.containsKey(path)) {
+            try {
-                    haloPluginManager.loadPlugin(path);
+                // delete old plugin jar file
+                Files.deleteIfExists(Paths.get(oldPluginLocation.getPath()));
+            } catch (IOException e) {
+                throw new PluginRuntimeException(e);
+            }
+        }
+        final var pluginFinder = new YamlPluginFinder();
+        final var pluginInPath = pluginFinder.find(toPath(newPluginPath));
+        client.fetch(Plugin.class, plugin.getMetadata().getName())
+            .ifPresent(persisted -> {
+                if (!persisted.getMetadata().getName()
+                    .equals(pluginInPath.getMetadata().getName())) {
+                    throw new DoNotRetryException("Plugin name is not match, skip reload.");
                 }
+                persisted.setSpec(pluginInPath.getSpec());
+                // merge annotations and labels
+                Map<String, String> newAnnotations = nullSafeAnnotations(persisted);
+                newAnnotations.putAll(nullSafeAnnotations(pluginInPath));
+
+                newAnnotations.put(PLUGIN_PATH, resolvePluginPathAnnotation(newPluginPath));
+                newAnnotations.remove(RELOAD_ANNO);
+                nullSafeLabels(persisted).putAll(nullSafeLabels(pluginInPath));
+                persisted.statusNonNull().setLoadLocation(toUri(newPluginPath));
+
+                // reload
+                haloPluginManager.reloadPluginWithPath(pluginName, toPath(newPluginPath));
+                // update plugin
+                client.update(persisted);
             });
     }
 
+    String resolvePluginPathAnnotation(String pluginPathString) {
+        Path pluginsRoot = toPath(haloPluginManager.getPluginsRoot().toString());
+        Path pluginPath = toPath(pluginPathString);
+        if (pluginPath.startsWith(pluginsRoot)) {
+            return pluginsRoot.relativize(pluginPath).toString();
+        }
+        return pluginPath.toString();
+    }
+
+
+    /**
+     * Returns absolute plugin path.
+     * if plugin path is absolute, use it directly in development mode.
+     * otherwise, combine plugin path with plugin root path.
+     * Note: plugin location without scheme
+     */
+    String buildPluginLocation(String name, String pluginPathString) {
+        Assert.notNull(name, "Plugin name must not be null");
+        Assert.notNull(pluginPathString, "Plugin path must not be null");
+        Path pluginsRoot = toPath(haloPluginManager.getPluginsRoot().toString());
+        Path pluginPath = toPath(pluginPathString);
+        // if plugin path is absolute, use it directly in development mode
+        if (pluginPath.isAbsolute()) {
+            if (!isDevelopmentMode(name) && !pluginPath.startsWith(pluginsRoot)) {
+                throw new DoNotRetryException(
+                    "Plugin path must be relative path or relative to plugin root path.");
+            }
+            return pluginPath.toString();
+        }
+        return PathUtils.combinePath(pluginsRoot.toString(), pluginPath.toString());
+    }
+
+    boolean shouldDeleteFile(String newPluginPath, URI oldPluginLocation) {
+        if (oldPluginLocation == null) {
+            return false;
+        }
+
+        if (oldPluginLocation.equals(toUri(newPluginPath))) {
+            return false;
+        }
+        return isJarFile(Paths.get(oldPluginLocation));
+    }
+
+    private void ensurePluginLoaded(String name) {
+        client.fetch(Plugin.class, name).ifPresent(plugin -> {
+            PluginWrapper pluginWrapper = haloPluginManager.getPlugin(name);
+            if (pluginWrapper != null) {
+                return;
+            }
+            Path pluginLocation = determinePluginLocation(plugin);
+            if (!Files.exists(pluginLocation)) {
+                return;
+            }
+            haloPluginManager.loadPlugin(pluginLocation);
+        });
+    }
+
+    Path toPath(String pathString) {
+        if (StringUtils.isBlank(pathString)) {
+            return null;
+        }
+        return Paths.get(URI.create(pathString).getPath());
+    }
+
+    URI toUri(String pathString) {
+        if (StringUtils.isBlank(pathString)) {
+            throw new IllegalArgumentException("Path string must not be blank");
+        }
+        return Paths.get(pathString).toUri();
+    }
+
     private boolean shouldReconcileStartState(Plugin plugin) {
         PluginWrapper pluginWrapper = getPluginWrapper(plugin.getMetadata().getName());
         return BooleanUtils.isTrue(plugin.getSpec().getEnabled())
@@ -574,8 +713,11 @@ public class PluginReconciler implements Reconciler<Request> {
         }
 
         // delete plugin resources
-        Path pluginPath = determinePluginLocation(plugin);
+        Path pluginPath = Optional.ofNullable(plugin.statusNonNull().getLoadLocation())
-        if (pluginPath != null && !isDevelopmentMode(name) && isJarFile(pluginPath)) {
+            .map(URI::getPath)
+            .map(Paths::get)
+            .orElse(null);
+        if (pluginPath != null && isJarFile(pluginPath)) {
             // delete plugin file
             try {
                 Files.deleteIfExists(pluginPath);
@@ -585,19 +727,21 @@ public class PluginReconciler implements Reconciler<Request> {
         }
     }
 
-    @Nullable
+    @NonNull
     Path determinePluginLocation(Plugin plugin) {
-        final var name = plugin.getMetadata().getName();
+        String pluginPath = nullSafeAnnotations(plugin).get(PLUGIN_PATH);
-        PluginWrapper pluginWrapper = haloPluginManager.getPlugin(name);
+        String name = plugin.getMetadata().getName();
-        return Optional.ofNullable(pluginWrapper)
+        if (StringUtils.isBlank(pluginPath)) {
-            .map(PluginWrapper::getPluginPath)
+            URI loadLocation = plugin.statusNonNull().getLoadLocation();
-            .orElseGet(() -> {
+            if (loadLocation != null) {
-                var localtionUri = plugin.statusNonNull().getLoadLocation();
+                pluginPath = loadLocation.getPath();
-                if (localtionUri != null) {
+            } else {
-                    return Path.of(localtionUri);
+                throw new DoNotRetryException(
-                }
+                    "Cannot determine plugin path for plugin: " + name);
-                return null;
+            }
-            });
+        }
+        String pluginLocation = buildPluginLocation(name, pluginPath);
+        return Paths.get(pluginLocation);
     }
 
     void createInitialReverseProxyIfNotPresent(Plugin plugin) {
@@ -629,15 +773,22 @@ public class PluginReconciler implements Reconciler<Request> {
             }, () -> client.create(reverseProxy));
     }
 
+    static class DoNotRetryException extends PluginRuntimeException {
+        public DoNotRetryException(String message) {
+            super(message);
+        }
+    }
+
     static String initialReverseProxyName(String pluginName) {
         return pluginName + "-system-generated-reverse-proxy";
     }
 
     private boolean isDevelopmentMode(String name) {
         PluginWrapper pluginWrapper = haloPluginManager.getPlugin(name);
-        if (pluginWrapper == null) {
+        RuntimeMode runtimeMode = haloPluginManager.getRuntimeMode();
-            return false;
+        if (pluginWrapper != null) {
+            runtimeMode = pluginWrapper.getRuntimeMode();
         }
-        return RuntimeMode.DEVELOPMENT.equals(pluginWrapper.getRuntimeMode());
+        return RuntimeMode.DEVELOPMENT.equals(runtimeMode);
     }
 }

application/src/main/java/run/halo/app/core/extension/service/impl/PluginServiceImpl.java

@@ -7,25 +7,25 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.time.Duration;
+import java.util.Map;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
 import org.pf4j.PluginWrapper;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
-import org.springframework.retry.RetryException;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
-import org.springframework.web.server.ServerErrorException;
 import org.springframework.web.server.ServerWebInputException;
 import reactor.core.Exceptions;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
-import reactor.util.retry.Retry;
 import run.halo.app.core.extension.Plugin;
 import run.halo.app.core.extension.service.PluginService;
+import run.halo.app.extension.MetadataUtil;
 import run.halo.app.extension.ReactiveExtensionClient;
 import run.halo.app.infra.SystemVersionSupplier;
 import run.halo.app.infra.exception.PluginAlreadyExistsException;
@@ -33,6 +33,7 @@ import run.halo.app.infra.exception.UnsatisfiedAttributeValueException;
 import run.halo.app.infra.utils.FileUtils;
 import run.halo.app.infra.utils.VersionUtils;
 import run.halo.app.plugin.HaloPluginManager;
+import run.halo.app.plugin.PluginConst;
 import run.halo.app.plugin.PluginProperties;
 import run.halo.app.plugin.YamlPluginFinder;
 
@@ -96,29 +97,21 @@ public class PluginServiceImpl implements PluginService {
             // pre-check the plugin in the path
             final var pluginFinder = new YamlPluginFinder();
             final var pluginInPath = pluginFinder.find(path);
+            Validate.notNull(pluginInPath.statusNonNull().getLoadLocation());
             satisfiesRequiresVersion(pluginInPath);
             if (!Objects.equals(name, pluginInPath.getMetadata().getName())) {
                 return Mono.error(new ServerWebInputException(
                     "The provided plugin " + pluginInPath.getMetadata().getName()
-                    + " didn't match the given plugin " + name));
+                        + " didn't match the given plugin " + name));
             }
 
             // check if the plugin exists
             return client.fetch(Plugin.class, name)
                 .switchIfEmpty(Mono.error(() -> new ServerWebInputException(
                     "The given plugin with name " + name + " was not found.")))
-                // delete the plugin and wait for the deletion
-                .then(Mono.defer(() -> deletePluginAndWaitForComplete(name)))
                 // copy plugin into plugin home
-                .flatMap(prevPlugin -> copyToPluginHome(pluginInPath)
+                .flatMap(prevPlugin -> copyToPluginHome(pluginInPath))
-                    .map(pluginFinder::find)
+                .flatMap(pluginPath -> updateReloadAnno(name, pluginPath));
-                    // reset enabled spec
-                    .doOnNext(pluginToCreate -> {
-                        var enabled = prevPlugin.getSpec().getEnabled();
-                        pluginToCreate.getSpec().setEnabled(enabled);
-                    }))
-                // create the plugin
-                .flatMap(client::create);
         });
     }
 
@@ -129,15 +122,16 @@ public class PluginServiceImpl implements PluginService {
             return Mono.error(() -> new ServerWebInputException(
                 "The given plugin with name " + name + " was not found."));
         }
-        YamlPluginFinder yamlPluginFinder = new YamlPluginFinder();
+        return updateReloadAnno(name, pluginWrapper.getPluginPath());
-        Plugin newPlugin = yamlPluginFinder.find(pluginWrapper.getPluginPath());
+    }
-        // reload plugin
+
-        pluginManager.reloadPlugin(name);
+    private Mono<Plugin> updateReloadAnno(String name, Path pluginPath) {
         return client.get(Plugin.class, name)
             .flatMap(plugin -> {
-                newPlugin.getMetadata().setVersion(plugin.getMetadata().getVersion());
+                // add reload annotation to flag the plugin to be reloaded
-                newPlugin.getSpec().setEnabled(true);
+                Map<String, String> annotations = MetadataUtil.nullSafeAnnotations(plugin);
-                return client.update(newPlugin);
+                annotations.put(PluginConst.RELOAD_ANNO, pluginPath.toString());
+                return client.update(plugin);
             });
     }
 
@@ -150,7 +144,7 @@ public class PluginServiceImpl implements PluginService {
     private Mono<Path> copyToPluginHome(Plugin plugin) {
         return Mono.fromCallable(
                 () -> {
-                    var fileName = plugin.generateFileName();
+                    var fileName = generateFileName(plugin);
                     var pluginRoot = Paths.get(pluginProperties.getPluginsRoot());
                     try {
                         Files.createDirectories(pluginRoot);
@@ -168,25 +162,17 @@ public class PluginServiceImpl implements PluginService {
             .subscribeOn(Schedulers.boundedElastic());
     }
 
-    private Mono<Plugin> deletePluginAndWaitForComplete(String pluginName) {
+    static String generateFileName(Plugin plugin) {
-        return client.fetch(Plugin.class, pluginName)
+        Assert.notNull(plugin, "The plugin must not be null.");
-            .flatMap(client::delete)
+        Assert.notNull(plugin.getMetadata(), "The plugin metadata must not be null.");
-            .flatMap(plugin -> waitForDeleted(pluginName).thenReturn(plugin));
+        Assert.notNull(plugin.getSpec(), "The plugin spec must not be null.");
+        String version = plugin.getSpec().getVersion();
+        if (StringUtils.isBlank(version)) {
+            throw new ServerWebInputException("The plugin version must not be blank.");
+        }
+        return String.format("%s-%s.jar", plugin.getMetadata().getName(), version);
     }
 
-    private Mono<Void> waitForDeleted(String pluginName) {
-        return Mono.defer(() -> client.fetch(Plugin.class, pluginName)
-                .flatMap(plugin -> Mono.error(
-                    new RetryException("Re-check if the plugin is deleted successfully"))))
-            .retryWhen(Retry.fixedDelay(20, Duration.ofMillis(100))
-                .filter(t -> t instanceof RetryException)
-            )
-            .onErrorMap(Exceptions::isRetryExhausted,
-                t -> new ServerErrorException("Wait timeout for plugin deleted", t))
-            .then();
-    }
-
-
     private void satisfiesRequiresVersion(Plugin newPlugin) {
         Assert.notNull(newPlugin, "The plugin must not be null.");
         Version version = systemVersion.get();

application/src/main/java/run/halo/app/plugin/HaloPluginManager.java

@@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.pf4j.DefaultPluginManager;
 import org.pf4j.ExtensionFactory;
 import org.pf4j.ExtensionFinder;
+import org.pf4j.PluginAlreadyLoadedException;
 import org.pf4j.PluginDependency;
 import org.pf4j.PluginDescriptor;
 import org.pf4j.PluginDescriptorFinder;
@@ -378,6 +379,23 @@ public class HaloPluginManager extends DefaultPluginManager
         return getPlugin(pluginId).getPluginState();
     }
 
+    /**
+     * Reload plugin by name and path.
+     * Note: This method will ignore {@link PluginAlreadyLoadedException}.
+     *
+     * @param pluginName plugin name
+     * @param pluginPath a new plugin path
+     */
+    public void reloadPluginWithPath(String pluginName, Path pluginPath) {
+        stopPlugin(pluginName, false);
+        unloadPlugin(pluginName, false);
+        try {
+            loadPlugin(pluginPath);
+        } catch (PluginAlreadyLoadedException ex) {
+            // ignore
+        }
+    }
+
     /**
      * Release plugin holding release on stop.
      */

application/src/main/java/run/halo/app/plugin/PluginConst.java

@@ -16,6 +16,10 @@ public interface PluginConst {
 
     String SYSTEM_PLUGIN_NAME = "system";
 
+    String RELOAD_ANNO = "plugin.halo.run/reload";
+
+    String PLUGIN_PATH = "plugin.halo.run/plugin-path";
+
     static String assertsRoutePrefix(String pluginName) {
         return "/plugins/" + pluginName + "/assets/";
     }

application/src/main/java/run/halo/app/plugin/YamlPluginFinder.java

@@ -12,6 +12,7 @@ import org.pf4j.util.FileUtils;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
 import run.halo.app.core.extension.Plugin;
+import run.halo.app.extension.MetadataUtil;
 import run.halo.app.extension.Unstructured;
 import run.halo.app.infra.utils.YamlUnstructuredLoader;
 
@@ -68,6 +69,8 @@ public class YamlPluginFinder {
             pluginStatus.setLoadLocation(pluginPath.toUri());
             plugin.setStatus(pluginStatus);
         }
+        MetadataUtil.nullSafeAnnotations(plugin)
+            .put(PluginConst.PLUGIN_PATH, pluginPath.toString());
         return plugin;
     }
 

application/src/test/java/run/halo/app/core/extension/reconciler/PluginReconcilerTest.java

@@ -1,21 +1,33 @@
 package run.halo.app.core.extension.reconciler;
 
+import static java.util.Objects.requireNonNull;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isA;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static run.halo.app.core.extension.reconciler.PluginReconciler.initialReverseProxyName;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import org.json.JSONException;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Nested;
@@ -35,8 +47,10 @@ import run.halo.app.core.extension.ReverseProxy;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.Metadata;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.infra.utils.FileUtils;
 import run.halo.app.infra.utils.JsonUtils;
 import run.halo.app.plugin.HaloPluginManager;
+import run.halo.app.plugin.PluginConst;
 import run.halo.app.plugin.PluginStartingError;
 
 /**
@@ -84,7 +98,7 @@ class PluginReconcilerTest {
         });
 
         ArgumentCaptor<Plugin> pluginCaptor = doReconcileWithoutRequeue();
-        verify(extensionClient, times(2)).update(isA(Plugin.class));
+        verify(extensionClient, times(3)).update(isA(Plugin.class));
 
         Plugin updateArgs = pluginCaptor.getAllValues().get(1);
         assertThat(updateArgs).isNotNull();
@@ -144,7 +158,7 @@ class PluginReconcilerTest {
         when(pluginWrapper.getPluginState()).thenReturn(PluginState.STARTED);
 
         ArgumentCaptor<Plugin> pluginCaptor = doReconcileWithoutRequeue();
-        verify(extensionClient, times(2)).update(any(Plugin.class));
+        verify(extensionClient, times(3)).update(any(Plugin.class));
 
         Plugin updateArgs = pluginCaptor.getValue();
         assertThat(updateArgs).isNotNull();
@@ -169,7 +183,8 @@ class PluginReconcilerTest {
                     "enabled": false
                 },
                 "status": {
-                    "phase": "STOPPED"
+                    "phase": "STOPPED",
+                    "loadLocation": "/tmp/plugins/apples.jar"
                 }
             }
             """, Plugin.class);
@@ -182,7 +197,7 @@ class PluginReconcilerTest {
         when(pluginWrapper.getPluginState()).thenReturn(PluginState.STARTED);
 
         ArgumentCaptor<Plugin> pluginCaptor = doReconcileWithoutRequeue();
-        verify(extensionClient, times(2)).update(any(Plugin.class));
+        verify(extensionClient, times(3)).update(any(Plugin.class));
 
         Plugin updateArgs = pluginCaptor.getValue();
         assertThat(updateArgs).isNotNull();
@@ -360,6 +375,119 @@ class PluginReconcilerTest {
         }
     }
 
+    @Test
+    void resolvePluginPathAnnotation() {
+        when(haloPluginManager.getPluginsRoot()).thenReturn(Paths.get("/tmp/plugins"));
+        String path = pluginReconciler.resolvePluginPathAnnotation("/tmp/plugins/sitemap-1.0.jar");
+        assertThat(path).isEqualTo("sitemap-1.0.jar");
+
+        path = pluginReconciler.resolvePluginPathAnnotation("/abc/plugins/sitemap-1.0.jar");
+        assertThat(path).isEqualTo("/abc/plugins/sitemap-1.0.jar");
+    }
+
+    @Nested
+    class ReloadPluginTest {
+        private static final String PLUGIN_NAME = "fake-plugin";
+        private static final Path OLD_PLUGIN_PATH = Paths.get("/path/to/old/plugin.jar");
+
+        @Test
+        void reload() throws IOException, URISyntaxException {
+            var fakePluginUri = requireNonNull(
+                getClass().getClassLoader().getResource("plugin/plugin-0.0.2")).toURI();
+            Path tempDirectory = Files.createTempDirectory("halo-ut-plugin-service-impl-");
+            final Path fakePluginPath = tempDirectory.resolve("plugin-0.0.2.jar");
+            try {
+                FileUtils.jar(Paths.get(fakePluginUri), tempDirectory.resolve("plugin-0.0.2.jar"));
+                when(haloPluginManager.getPluginsRoot()).thenReturn(tempDirectory);
+                // mock plugin
+                Plugin plugin = mock(Plugin.class);
+                Metadata metadata = new Metadata();
+                metadata.setName(PLUGIN_NAME);
+                when(plugin.getMetadata()).thenReturn(metadata);
+                metadata.setAnnotations(new HashMap<>());
+                metadata.getAnnotations()
+                    .put(PluginConst.RELOAD_ANNO, fakePluginPath.toString());
+                Plugin.PluginStatus pluginStatus = mock(Plugin.PluginStatus.class);
+                when(pluginStatus.getLoadLocation()).thenReturn(OLD_PLUGIN_PATH.toUri());
+                when(plugin.statusNonNull()).thenReturn(pluginStatus);
+
+                when(extensionClient.fetch(Plugin.class, PLUGIN_NAME))
+                    .thenReturn(Optional.of(plugin));
+
+                // call reload method
+                pluginReconciler.reload(plugin);
+
+                // verify that the plugin is updated with the new plugin's spec, annotations, and
+                // labels
+                verify(plugin).setSpec(any(Plugin.PluginSpec.class));
+                verify(extensionClient).update(plugin);
+
+                // verify that the plugin's load location is updated to the new plugin path
+                verify(pluginStatus).setLoadLocation(fakePluginPath.toUri());
+
+                // verify that the new plugin is reloaded
+                verify(haloPluginManager).reloadPluginWithPath(PLUGIN_NAME, fakePluginPath);
+            } finally {
+                FileUtils.deleteRecursivelyAndSilently(tempDirectory);
+            }
+        }
+
+        @Test
+        void shouldDeleteFile() throws IOException {
+            String newPluginPath = "/path/to/new/plugin.jar";
+
+            // Case 1: oldPluginLocation is null
+            assertFalse(pluginReconciler.shouldDeleteFile(newPluginPath, null));
+
+            // Case 2: oldPluginLocation is the same as newPluginPath
+            assertFalse(pluginReconciler.shouldDeleteFile(newPluginPath,
+                pluginReconciler.toUri(newPluginPath)));
+
+            Path tempDirectory = Files.createTempDirectory("halo-ut-plugin-service-impl-");
+            try {
+                Path oldPluginPath = tempDirectory.resolve("plugin.jar");
+                final URI oldPluginLocation = oldPluginPath.toUri();
+                Files.createFile(oldPluginPath);
+                // Case 3: oldPluginLocation is different from newPluginPath and is a JAR file
+                assertTrue(pluginReconciler.shouldDeleteFile(newPluginPath, oldPluginLocation));
+            } finally {
+                FileUtils.deleteRecursivelyAndSilently(tempDirectory);
+            }
+
+            // Case 4: oldPluginLocation is different from newPluginPath and is not a JAR file
+            assertFalse(pluginReconciler.shouldDeleteFile(newPluginPath,
+                Paths.get("/path/to/old/plugin.txt").toUri()));
+        }
+
+        @Test
+        void toPath() {
+            assertThat(pluginReconciler.toPath(null)).isNull();
+            assertThat(pluginReconciler.toPath("")).isNull();
+            assertThat(pluginReconciler.toPath(" ")).isNull();
+
+            Path path = pluginReconciler.toPath("file:///path/to/file.txt");
+            assertThat(path).isNotNull();
+            assertThat(path.toString()).isEqualTo("/path/to/file.txt");
+        }
+
+        @Test
+        void toUri() {
+            // Test with null pathString
+            Assertions.assertThrows(IllegalArgumentException.class, () -> {
+                pluginReconciler.toUri(null);
+            });
+
+            // Test with empty pathString
+            Assertions.assertThrows(IllegalArgumentException.class, () -> {
+                pluginReconciler.toUri("");
+            });
+
+            // Test with non-empty pathString
+            URI uri = pluginReconciler.toUri("/path/to/file");
+            Assertions.assertEquals("file:///path/to/file", uri.toString());
+        }
+    }
+
     private ArgumentCaptor<Plugin> doReconcileNeedRequeue() {
         ArgumentCaptor<Plugin> pluginCaptor = ArgumentCaptor.forClass(Plugin.class);
         doNothing().when(extensionClient).update(pluginCaptor.capture());
@@ -397,7 +525,8 @@ class PluginReconcilerTest {
                     "enabled": true
                 },
                 "status": {
-                    "phase": "STOPPED"
+                    "phase": "STOPPED",
+                    "loadLocation": "/tmp/plugins/apples.jar"
                 }
             }
             """, Plugin.class);
@@ -416,7 +545,8 @@ class PluginReconcilerTest {
                     "enabled": false
                 },
                 "status": {
-                    "phase": "STARTED"
+                    "phase": "STARTED",
+                    "loadLocation": "/tmp/plugins/apples.jar"
                 }
             }
             """, Plugin.class);

application/src/test/java/run/halo/app/core/extension/service/impl/PluginServiceImplTest.java

@@ -2,11 +2,9 @@ package run.halo.app.core.extension.service.impl;
 
 import static java.util.Objects.requireNonNull;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
-import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isA;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
@@ -26,13 +24,11 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.pf4j.PluginState;
 import org.pf4j.PluginWrapper;
-import org.springframework.web.server.ServerErrorException;
 import org.springframework.web.server.ServerWebInputException;
 import reactor.core.publisher.Mono;
 import reactor.test.StepVerifier;
@@ -43,6 +39,7 @@ import run.halo.app.infra.SystemVersionSupplier;
 import run.halo.app.infra.exception.PluginAlreadyExistsException;
 import run.halo.app.infra.utils.FileUtils;
 import run.halo.app.plugin.HaloPluginManager;
+import run.halo.app.plugin.PluginConst;
 import run.halo.app.plugin.PluginProperties;
 import run.halo.app.plugin.YamlPluginFinder;
 
@@ -167,30 +164,15 @@ class PluginServiceImplTest {
             verify(client).fetch(Plugin.class, "fake-plugin");
         }
 
-        @Test
-        void upgradeWhenWaitingTimeoutForPluginDeletion() {
-            var prevPlugin = mock(Plugin.class);
-            when(client.fetch(Plugin.class, "fake-plugin"))
-                .thenReturn(Mono.just(prevPlugin));
-            when(client.delete(isA(Plugin.class))).thenReturn(Mono.just(prevPlugin));
-
-            var plugin = pluginService.upgrade("fake-plugin", fakePluginPath);
-            StepVerifier.create(plugin)
-                .consumeErrorWith(error -> {
-                    assertTrue(error instanceof ServerErrorException);
-                    assertEquals("Wait timeout for plugin deleted",
-                        ((ServerErrorException) error).getReason());
-                })
-                .verify();
-
-            verify(client, times(23)).fetch(Plugin.class, "fake-plugin");
-            verify(client).delete(isA(Plugin.class));
-        }
-
         @Test
         void upgradeNormally() {
-            var prevPlugin = mock(Plugin.class);
+            final var prevPlugin = mock(Plugin.class);
-            var spec = mock(Plugin.PluginSpec.class);
+            final var spec = mock(Plugin.PluginSpec.class);
+            final var updatedPlugin = mock(Plugin.class);
+            Metadata metadata = new Metadata();
+            metadata.setName("fake-plugin");
+            when(prevPlugin.getMetadata()).thenReturn(metadata);
+
             when(prevPlugin.getSpec()).thenReturn(spec);
             when(spec.getEnabled()).thenReturn(true);
 
@@ -198,61 +180,50 @@ class PluginServiceImplTest {
                 .thenReturn(Mono.just(prevPlugin))
                 .thenReturn(Mono.just(prevPlugin))
                 .thenReturn(Mono.empty());
-            when(client.delete(isA(Plugin.class))).thenReturn(Mono.just(prevPlugin));
 
-            var createdPlugin = mock(Plugin.class);
+            when(client.get(Plugin.class, "fake-plugin"))
-            when(client.create(isA(Plugin.class))).thenReturn(Mono.just(createdPlugin));
+                .thenReturn(Mono.just(prevPlugin));
+
+            when(client.update(isA(Plugin.class))).thenReturn(Mono.just(updatedPlugin));
 
             var plugin = pluginService.upgrade("fake-plugin", fakePluginPath);
 
             StepVerifier.create(plugin)
-                .expectNext(createdPlugin)
+                .expectNext(updatedPlugin).verifyComplete();
-                .verifyComplete();
 
-            verify(client, times(3)).fetch(Plugin.class, "fake-plugin");
+            verify(client, times(1)).fetch(Plugin.class, "fake-plugin");
-            verify(client).delete(isA(Plugin.class));
+            verify(client, times(0)).delete(isA(Plugin.class));
-            verify(client).<Plugin>create(argThat(p -> p.getSpec().getEnabled()));
+            verify(client).<Plugin>update(argThat(p -> p.getSpec().getEnabled()));
         }
     }
 
     @Test
-    void reload() throws IOException, URISyntaxException {
+    void reload() {
-        var fakePluginUri = requireNonNull(
+        // given
-            getClass().getClassLoader().getResource("plugin/plugin-0.0.2")).toURI();
+        String pluginName = "test-plugin";
-        Path tempDirectory = Files.createTempDirectory("halo-ut-plugin-service-impl-");
+        PluginWrapper pluginWrapper = mock(PluginWrapper.class);
-        Path fakePluginPath = tempDirectory.resolve("plugin-0.0.2.jar");
+        when(pluginManager.getPlugin(pluginName)).thenReturn(pluginWrapper);
-        try {
+        when(pluginWrapper.getPluginPath())
-            FileUtils.jar(Paths.get(fakePluginUri), tempDirectory.resolve("plugin-0.0.2.jar"));
+            .thenReturn(Paths.get("/tmp/plugins/fake-plugin.jar"));
+        Plugin plugin = new Plugin();
+        plugin.setMetadata(new Metadata());
+        plugin.getMetadata().setName(pluginName);
+        plugin.setSpec(new Plugin.PluginSpec());
+        when(client.get(Plugin.class, pluginName)).thenReturn(Mono.just(plugin));
+        when(client.update(plugin)).thenReturn(Mono.just(plugin));
 
-            final String pluginName = "fake-plugin";
+        // when
-            PluginWrapper pluginWrapper = mock(PluginWrapper.class);
+        Mono<Plugin> result = pluginService.reload(pluginName);
-            when(pluginManager.getPlugin(eq(pluginName))).thenReturn(pluginWrapper);
-            when(pluginWrapper.getPluginPath()).thenReturn(fakePluginPath);
 
-            Plugin plugin = new Plugin();
+        // then
-            plugin.setMetadata(new Metadata());
+        assertDoesNotThrow(() -> result.block());
-            plugin.getMetadata().setName(pluginName);
+        verify(client, times(1)).update(
-            plugin.setSpec(new Plugin.PluginSpec());
+            argThat(p -> {
-            plugin.getSpec().setEnabled(false);
+                String reloadPath = p.getMetadata().getAnnotations().get(PluginConst.RELOAD_ANNO);
-            plugin.getSpec().setDisplayName("Fake Plugin");
+                assertThat(reloadPath).isEqualTo("/tmp/plugins/fake-plugin.jar");
-
+                return true;
-            when(client.get(eq(Plugin.class), eq(pluginName))).thenReturn(Mono.just(plugin));
+            })
-            when(client.update(any(Plugin.class))).thenReturn(Mono.empty());
+        );
-
+        verify(pluginWrapper, times(1)).getPluginPath();
-            pluginService.reload(pluginName).block();
-
-            verify(pluginManager).reloadPlugin(eq(pluginName));
-            verify(client).get(eq(Plugin.class), eq(pluginName));
-
-            ArgumentCaptor<Plugin> captor = ArgumentCaptor.forClass(Plugin.class);
-            verify(client).update(captor.capture());
-
-            Plugin updatedPlugin = captor.getValue();
-            assertThat(updatedPlugin.getSpec().getEnabled()).isTrue();
-            assertThat(updatedPlugin.getSpec().getDisplayName()).isEqualTo("Fake Display Name");
-            assertThat(updatedPlugin.getSpec().getDescription()).isEqualTo("Fake description");
-        } finally {
-            FileUtils.deleteRecursivelyAndSilently(tempDirectory);
-        }
     }
-}
+}