feat: add plugin initial resource cleanup when plugin stop (#2262)

* refactor: the way of plugin extension update

* feat: add plugin initial resource cleanup when plugin stop

* refactor: remove role delete watcher
pull/2267/head
guqing 2022-07-20 14:35:24 +08:00 committed by GitHub
parent ba20f71504
commit bb0b5b26e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 315 additions and 107 deletions

View File

@ -6,7 +6,6 @@ import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@ -78,18 +77,9 @@ public class Plugin extends AbstractExtension {
private Boolean enabled = false;
private List<String> extensionLocations;
private String settingName;
private String configMapName;
@NonNull
@JsonIgnore
public List<String> extensionLocationsNonNull() {
this.extensionLocations = Objects.requireNonNullElseGet(extensionLocations, List::of);
return this.extensionLocations;
}
}
@Getter

View File

@ -25,6 +25,7 @@ import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.lang.NonNull;
import run.halo.app.plugin.event.HaloPluginBeforeStopEvent;
import run.halo.app.plugin.event.HaloPluginLoadedEvent;
import run.halo.app.plugin.event.HaloPluginStartedEvent;
import run.halo.app.plugin.event.HaloPluginStateChangedEvent;
@ -127,7 +128,6 @@ public class HaloPluginManager extends DefaultPluginManager
@Override
protected PluginState stopPlugin(String pluginId, boolean stopDependents) {
checkPluginId(pluginId);
PluginWrapper pluginWrapper = getPlugin(pluginId);
PluginDescriptor pluginDescriptor = pluginWrapper.getDescriptor();
PluginState pluginState = pluginWrapper.getPluginState();
@ -142,6 +142,8 @@ public class HaloPluginManager extends DefaultPluginManager
return pluginState;
}
rootApplicationContext.publishEvent(new HaloPluginBeforeStopEvent(this, pluginWrapper));
if (stopDependents) {
List<String> dependents = dependencyResolver.getDependents(pluginId);
while (!dependents.isEmpty()) {
@ -171,7 +173,7 @@ public class HaloPluginManager extends DefaultPluginManager
@Override
public PluginState stopPlugin(String pluginId) {
return stopPlugin(pluginId, true);
return this.stopPlugin(pluginId, true);
}
@Override
@ -294,6 +296,8 @@ public class HaloPluginManager extends DefaultPluginManager
PluginState pluginState = pluginWrapper.getPluginState();
if (PluginState.STARTED == pluginState) {
try {
rootApplicationContext.publishEvent(
new HaloPluginBeforeStopEvent(this, pluginWrapper));
log.info("Stop plugin '{}'", getPluginLabel(pluginWrapper.getDescriptor()));
pluginWrapper.getPlugin().stop();
pluginWrapper.setPluginState(PluginState.STOPPED);

View File

@ -1,6 +1,12 @@
package run.halo.app.plugin;
import java.util.List;
import java.util.concurrent.locks.StampedLock;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.util.Assert;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import run.halo.app.extension.GroupVersionKind;
/**
* The generic IOC container for plugins.
@ -12,6 +18,8 @@ import org.springframework.context.support.GenericApplicationContext;
*/
public class PluginApplicationContext extends GenericApplicationContext {
private final GvkExtensionMapping gvkExtensionMapping = new GvkExtensionMapping();
private String pluginId;
public String getPluginId() {
@ -21,4 +29,86 @@ public class PluginApplicationContext extends GenericApplicationContext {
public void setPluginId(String pluginId) {
this.pluginId = pluginId;
}
/**
* Gets the gvk-extension mapping.
* It is thread safe
*
* @param gvk the group-kind-version
* @param extensionName extension resources name
*/
public void addExtensionMapping(GroupVersionKind gvk, String extensionName) {
gvkExtensionMapping.addExtensionMapping(gvk, extensionName);
}
/**
* Gets the extension names by gvk.
* It is thread safe
*
* @param gvk the group-kind-version
* @return a immutable list of extension names
*/
public List<String> getExtensionNames(GroupVersionKind gvk) {
return List.copyOf(gvkExtensionMapping.getExtensionNames(gvk));
}
public MultiValueMap<GroupVersionKind, String> extensionNamesMapping() {
return gvkExtensionMapping.extensionNamesMapping();
}
static class GvkExtensionMapping {
private final StampedLock sl = new StampedLock();
private final MultiValueMap<GroupVersionKind, String> extensionNamesMapping =
new LinkedMultiValueMap<>();
public void addAllExtensionMapping(GroupVersionKind gvk, List<String> extensionNames) {
long stamp = sl.writeLock();
try {
extensionNamesMapping.addAll(gvk, extensionNames);
} finally {
sl.unlockWrite(stamp);
}
}
public void addExtensionMapping(GroupVersionKind gvk, String extensionName) {
long stamp = sl.writeLock();
try {
extensionNamesMapping.add(gvk, extensionName);
} finally {
sl.unlockWrite(stamp);
}
}
public List<String> getExtensionNames(GroupVersionKind gvk) {
Assert.notNull(gvk, "The gvk must not be null");
long stamp = sl.tryOptimisticRead();
List<String> values = extensionNamesMapping.get(gvk);
if (!sl.validate(stamp)) {
// Check if another write lock occurs after the optimistic read lock
// If so, escalate lock to a pessimistic lock
stamp = sl.readLock();
try {
return extensionNamesMapping.get(gvk);
} finally {
sl.unlockRead(stamp);
}
}
return values;
}
public MultiValueMap<GroupVersionKind, String> extensionNamesMapping() {
return new LinkedMultiValueMap<>(extensionNamesMapping);
}
public void clear() {
extensionNamesMapping.clear();
}
}
@Override
protected void onClose() {
// For subclasses: do nothing by default.
super.onClose();
gvkExtensionMapping.clear();
}
}

View File

@ -0,0 +1,44 @@
package run.halo.app.plugin;
import org.pf4j.PluginWrapper;
import org.springframework.context.ApplicationListener;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import org.springframework.util.MultiValueMap;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.plugin.event.HaloPluginBeforeStopEvent;
/**
* Synchronization listener executed by the plugin before it is stopped.
*
* @author guqing
* @since 2.0.0
*/
@Component
public class PluginBeforeStopSyncListener
implements ApplicationListener<HaloPluginBeforeStopEvent> {
private final ExtensionClient client;
public PluginBeforeStopSyncListener(ExtensionClient client) {
this.client = client;
}
@Override
public void onApplicationEvent(@NonNull HaloPluginBeforeStopEvent event) {
PluginWrapper pluginWrapper = event.getPlugin();
PluginApplicationContext pluginContext = ExtensionContextRegistry.getInstance()
.getByPluginId(pluginWrapper.getPluginId());
cleanUpPluginExtensionResources(pluginContext);
}
private void cleanUpPluginExtensionResources(PluginApplicationContext context) {
MultiValueMap<GroupVersionKind, String> gvkExtensionNames =
context.extensionNamesMapping();
gvkExtensionNames.forEach((gvk, extensionNames) ->
extensionNames.forEach(extensionName -> client.fetch(gvk, extensionName)
.ifPresent(client::delete)));
}
}

View File

@ -1,9 +1,24 @@
package run.halo.app.plugin;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import lombok.extern.slf4j.Slf4j;
import org.pf4j.DevelopmentPluginClasspath;
import org.pf4j.PluginRuntimeException;
import org.pf4j.PluginWrapper;
import org.pf4j.RuntimeMode;
import org.springframework.context.ApplicationListener;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.Resource;
@ -37,13 +52,22 @@ public class PluginStartedListener implements ApplicationListener<HaloPluginStar
// load unstructured
DefaultResourceLoader resourceLoader =
new DefaultResourceLoader(pluginWrapper.getPluginClassLoader());
plugin.getSpec().extensionLocationsNonNull().stream()
PluginApplicationContext pluginApplicationContext = ExtensionContextRegistry.getInstance()
.getByPluginId(pluginWrapper.getPluginId());
lookupExtensions(pluginWrapper.getPluginPath(),
pluginWrapper.getRuntimeMode())
.stream()
.map(resourceLoader::getResource)
.filter(Resource::exists)
.map(resource -> new YamlUnstructuredLoader(resource).load())
.flatMap(List::stream)
.forEach(unstructured -> {
MetadataOperator metadata = unstructured.getMetadata();
// collector plugin initialize extension resources
pluginApplicationContext.addExtensionMapping(unstructured.groupVersionKind(),
metadata.getName());
Map<String, String> labels = metadata.getLabels();
if (labels == null) {
labels = new HashMap<>();
@ -57,4 +81,69 @@ public class PluginStartedListener implements ApplicationListener<HaloPluginStar
}, () -> extensionClient.create(unstructured));
});
}
Set<String> lookupExtensions(Path pluginPath, RuntimeMode runtimeMode) {
if (RuntimeMode.DEVELOPMENT.equals(runtimeMode)) {
return PluginExtensionLoaderUtils.lookupFromClasses(pluginPath);
} else {
return PluginExtensionLoaderUtils.lookupFromJar(pluginPath);
}
}
@Slf4j
static class PluginExtensionLoaderUtils {
static final String EXTENSION_LOCATION = "extensions";
static final DevelopmentPluginClasspath PLUGIN_CLASSPATH = new DevelopmentPluginClasspath();
static Set<String> lookupFromClasses(Path pluginPath) {
Set<String> result = new HashSet<>();
for (String directory : PLUGIN_CLASSPATH.getClassesDirectories()) {
File file = pluginPath.resolve(directory).resolve(EXTENSION_LOCATION).toFile();
if (file.exists() && file.isDirectory()) {
result.addAll(walkExtensionFiles(file.toPath()));
}
}
return result;
}
private static Set<String> walkExtensionFiles(Path location) {
try (Stream<Path> stream = Files.walk(location)) {
return stream.map(Path::normalize)
.filter(Files::isRegularFile)
.filter(path -> isYamlFile(path.getFileName().toString()))
.map(path -> location.getParent().relativize(path).toString())
.collect(Collectors.toSet());
} catch (IOException e) {
log.debug("Failed to walk extension files from [{}]", location);
return Collections.emptySet();
}
}
static boolean isYamlFile(String path) {
return path.endsWith(".yaml") || path.endsWith(".yml");
}
/**
* <p>Lists the path of the unstructured yaml configuration file from the plugin jar.</p>
*
* @param pluginJarPath plugin jar path
* @return Unstructured file paths relative to plugin classpath
* @throws PluginRuntimeException If loading the file fails
*/
static Set<String> lookupFromJar(Path pluginJarPath) {
try (JarFile jarFile = new JarFile(pluginJarPath.toFile())) {
return jarFile.stream()
.filter(jarEntry -> {
String name = jarEntry.getName();
return name.startsWith(EXTENSION_LOCATION)
&& !jarEntry.isDirectory()
&& isYamlFile(name);
})
.map(ZipEntry::getName)
.collect(Collectors.toSet());
} catch (IOException e) {
throw new PluginRuntimeException(e);
}
}
}
}

View File

@ -4,20 +4,14 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import lombok.extern.slf4j.Slf4j;
import org.pf4j.DevelopmentPluginClasspath;
import org.pf4j.PluginRuntimeException;
import org.pf4j.PluginState;
import org.pf4j.util.FileUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.lang.Nullable;
import run.halo.app.core.extension.Plugin;
import run.halo.app.extension.Unstructured;
import run.halo.app.infra.utils.PathUtils;
@ -56,8 +50,8 @@ import run.halo.app.infra.utils.YamlUnstructuredLoader;
*/
@Slf4j
public class YamlPluginFinder {
static final DevelopmentPluginClasspath PLUGIN_CLASSPATH = new DevelopmentPluginClasspath();
public static final String DEFAULT_PROPERTIES_FILE_NAME = "plugin.yaml";
private static final String DEFAULT_RESOURCE_LOCATION = "extensions/";
private final String propertiesFileName;
public YamlPluginFinder() {
@ -75,12 +69,6 @@ public class YamlPluginFinder {
pluginStatus.setPhase(PluginState.RESOLVED);
plugin.setStatus(pluginStatus);
}
// read unstructured files
if (FileUtils.isJarFile(pluginPath)) {
plugin.getSpec().setExtensionLocations(getUnstructuredFilePathFromJar(pluginPath));
} else {
plugin.getSpec().setExtensionLocations(getUnstructuredFileFromClasspath(pluginPath));
}
return plugin;
}
@ -113,7 +101,7 @@ public class YamlPluginFinder {
protected Path getManifestPath(Path pluginPath, String propertiesFileName) {
if (Files.isDirectory(pluginPath)) {
for (String location : getSearchLocations()) {
for (String location : PLUGIN_CLASSPATH.getClassesDirectories()) {
String s = PathUtils.combinePath(pluginPath.toString(),
location, propertiesFileName);
Path path = Paths.get(s);
@ -133,66 +121,4 @@ public class YamlPluginFinder {
}
}
}
/**
* <p>Lists the path of the unstructured yaml configuration file from the plugin jar.</p>
*
* @param jarPath plugin jar path
* @return Unstructured file paths relative to plugin classpath
* @throws PluginRuntimeException If loading the file fails
*/
protected List<String> getUnstructuredFilePathFromJar(Path jarPath) {
try (JarFile jarFile = new JarFile(jarPath.toFile())) {
return jarFile.stream()
.filter(jarEntry -> {
String name = jarEntry.getName();
return name.startsWith(DEFAULT_RESOURCE_LOCATION)
&& !jarEntry.isDirectory()
&& isYamlFile(name);
})
.map(ZipEntry::getName)
.toList();
} catch (IOException e) {
throw new PluginRuntimeException(e);
}
}
private List<String> getUnstructuredFileFromClasspath(Path pluginPath) {
final Path unstructuredLocation = decisionUnstructuredLocation(pluginPath);
if (unstructuredLocation == null) {
return Collections.emptyList();
}
try (Stream<Path> stream = Files.walk(unstructuredLocation)) {
return stream.map(Path::normalize)
.filter(Files::isRegularFile)
.filter(path -> isYamlFile(path.getFileName().toString()))
.map(path -> unstructuredLocation.getParent().relativize(path).toString())
.collect(Collectors.toList());
} catch (IOException e) {
return Collections.emptyList();
}
}
@Nullable
private Path decisionUnstructuredLocation(Path pluginPath) {
for (String searchLocation : getSearchLocations()) {
String unstructuredLocationString = PathUtils.combinePath(pluginPath.toString(),
searchLocation, DEFAULT_RESOURCE_LOCATION);
Path path = Paths.get(unstructuredLocationString);
boolean exists = Files.exists(path);
if (exists) {
return path;
}
}
return null;
}
private boolean isYamlFile(String path) {
return path.endsWith(".yaml") || path.endsWith(".yml");
}
private Set<String> getSearchLocations() {
// TODO 优化路径获取
return Set.of("build/resources/main/", "target/classes/");
}
}

View File

@ -0,0 +1,21 @@
package run.halo.app.plugin.event;
import org.pf4j.PluginWrapper;
import org.springframework.context.ApplicationEvent;
/**
* @author guqing
* @since 2.0.0
*/
public class HaloPluginBeforeStopEvent extends ApplicationEvent {
private final PluginWrapper plugin;
public HaloPluginBeforeStopEvent(Object source, PluginWrapper plugin) {
super(source);
this.plugin = plugin;
}
public PluginWrapper getPlugin() {
return plugin;
}
}

View File

@ -89,7 +89,6 @@ class RoleReconcilerTest {
.get(Role.ROLE_DEPENDENCY_RULES), false);
}
@Test
void reconcileUiPermission() {
Role roleManage = TestRole.getRoleManage();

View File

@ -0,0 +1,51 @@
package run.halo.app.plugin;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Set;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.springframework.util.ResourceUtils;
/**
* Tests for {@link PluginStartedListener}.
*
* @author guqing
* @since 2.0.0
*/
class PluginStartedListenerTest {
@Nested
class PluginExtensionLoaderUtilsTest {
@Test
void lookupFromClasses() throws IOException {
Path tempPluginPath = Files.createTempDirectory("halo-test-plugin");
Path directories =
Files.createDirectories(tempPluginPath.resolve("build/resources/main"));
Path extensions = Files.createDirectory(directories.resolve("extensions"));
Files.createFile(extensions.resolve("roles.yaml"));
Set<String> extensionResources =
PluginStartedListener.PluginExtensionLoaderUtils.lookupFromClasses(tempPluginPath);
assertThat(extensionResources).containsAll(Set.of("extensions/roles.yaml"));
}
@Test
void lookupFromJar() throws FileNotFoundException {
File file =
ResourceUtils.getFile("classpath:plugin/test-unstructured-resource-loader.jar");
Set<String> unstructuredFilePathFromJar =
PluginStartedListener.PluginExtensionLoaderUtils.lookupFromJar(file.toPath());
assertThat(unstructuredFilePathFromJar).hasSize(3);
assertThat(unstructuredFilePathFromJar).containsAll(Set.of("extensions/roles.yaml",
"extensions/reverseProxy.yaml", "extensions/test.yml"));
}
}
}

View File

@ -10,7 +10,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.json.JSONException;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -48,8 +47,6 @@ class YamlPluginFinderTest {
Path directories = Files.createDirectories(tempDirectory.resolve("build/resources/main"));
FileCopyUtils.copy(testFile, directories.resolve("plugin.yaml").toFile());
Path extensions = Files.createDirectory(directories.resolve("extensions"));
Files.createFile(extensions.resolve("roles.yaml"));
Plugin plugin = pluginFinder.find(tempDirectory);
assertThat(plugin).isNotNull();
@ -66,11 +63,19 @@ class YamlPluginFinderTest {
""",
JsonUtils.objectToJson(plugin.getStatus()),
true);
assertThat(plugin.getSpec().getExtensionLocations()).contains("extensions/roles.yaml");
}
@Test
void unstructuredToPluginTest() throws JsonProcessingException, JSONException {
void findFromJar() throws FileNotFoundException {
File file =
ResourceUtils.getFile("classpath:plugin/test-unstructured-resource-loader.jar");
Plugin plugin = pluginFinder.find(file.toPath());
assertThat(plugin).isNotNull();
assertThat(plugin.getMetadata().getName()).isEqualTo("io.github.guqing.apples");
}
@Test
void unstructuredToPluginTest() throws JSONException {
Plugin plugin = pluginFinder.unstructuredToPlugin(new FileSystemResource(testFile));
assertThat(plugin).isNotNull();
JSONAssert.assertEquals("""
@ -94,7 +99,6 @@ class YamlPluginFinderTest {
"requires": ">=2.0.0",
"pluginClass": null,
"enabled": false,
"extensionLocations": null,
settingName: null,
configMapName: null
},
@ -169,14 +173,4 @@ class YamlPluginFinderTest {
assertThat(plugin.getSpec()).isNotNull();
JSONAssert.assertEquals(pluginJson, JsonUtils.objectToJson(plugin), false);
}
@Test
void getUnstructuredFilePathFromJar() throws FileNotFoundException {
File file = ResourceUtils.getFile("classpath:plugin/test-unstructured-resource-loader.jar");
List<String> unstructuredFilePathFromJar =
pluginFinder.getUnstructuredFilePathFromJar(file.toPath());
assertThat(unstructuredFilePathFromJar).hasSize(3);
assertThat(unstructuredFilePathFromJar).contains("extensions/roles.yaml",
"extensions/reverseProxy.yaml", "extensions/test.yml");
}
}