Implement simple garbage collector to collect deletable extensions (#2343)

#### What type of PR is this?

/kind feature
/area core
/milestone 2.0

#### What this PR does / why we need it:

- Add finalizers field into metadata to let controller do some works before deleting
- Implement a simple garbage collector to collect deletable extensions

Garbage collector controller is a special controller, which will watch extensions with any GVK instead of specific type. So we have to customize the controller parameters entirely.

#### Which issue(s) this PR fixes:

Fixes #2307

#### Special notes for your reviewer:

How to test?

- Delete without finalizers

    1. Create an extension and check it
    2. Delete it and check it again

- Delete with finalizers
    
    1. Create an extension and update it with finalizers
    2. Delete it and checkout it
    4. You will see the extension with finalizers not deleted
    5. Update it without finalizers and check it again

#### Does this PR introduce a user-facing change?

```release-note
None
```

src/main/java/run/halo/app/core/extension/reconciler/MenuItemReconciler.java

@@ -5,8 +5,9 @@ import run.halo.app.core.extension.MenuItem;
 import run.halo.app.core.extension.MenuItem.MenuItemStatus;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 
-public class MenuItemReconciler implements Reconciler {
+public class MenuItemReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
 

src/main/java/run/halo/app/core/extension/reconciler/MenuReconciler.java

@@ -3,7 +3,7 @@ package run.halo.app.core.extension.reconciler;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
 
-public class MenuReconciler implements Reconciler {
+public class MenuReconciler implements Reconciler<Reconciler.Request> {
 
     private final ExtensionClient client;
 

src/main/java/run/halo/app/core/extension/reconciler/PluginReconciler.java

@@ -13,6 +13,7 @@ import org.pf4j.PluginWrapper;
 import run.halo.app.core.extension.Plugin;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 import run.halo.app.infra.utils.JsonUtils;
 import run.halo.app.plugin.HaloPluginManager;
 import run.halo.app.plugin.PluginStartingError;
@@ -26,7 +27,7 @@ import run.halo.app.plugin.resources.ReverseProxyRouterFunctionFactory;
  * @since 2.0.0
  */
 @Slf4j
-public class PluginReconciler implements Reconciler {
+public class PluginReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
     private final HaloPluginManager haloPluginManager;

src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java

@@ -15,6 +15,7 @@ import run.halo.app.core.extension.Post;
 import run.halo.app.core.extension.Snapshot;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 import run.halo.app.infra.Condition;
 import run.halo.app.infra.ConditionStatus;
 import run.halo.app.infra.utils.JsonUtils;
@@ -31,7 +32,7 @@ import run.halo.app.infra.utils.JsonUtils;
  * @author guqing
  * @since 2.0.0
  */
-public class PostReconciler implements Reconciler {
+public class PostReconciler implements Reconciler<Request> {
     public static final String PERMALINK_PREFIX = "/permalink/posts/";
     private final ExtensionClient client;
     private final ContentService contentService;

src/main/java/run/halo/app/core/extension/reconciler/RoleBindingReconciler.java

@@ -15,10 +15,11 @@ import run.halo.app.core.extension.RoleBinding.Subject;
 import run.halo.app.core.extension.User;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 import run.halo.app.infra.utils.JsonUtils;
 
 @Slf4j
-public class RoleBindingReconciler implements Reconciler {
+public class RoleBindingReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
 

src/main/java/run/halo/app/core/extension/reconciler/RoleReconciler.java

@@ -13,6 +13,7 @@ import run.halo.app.core.extension.Role;
 import run.halo.app.core.extension.service.RoleService;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 import run.halo.app.infra.utils.JsonUtils;
 
 /**
@@ -22,7 +23,7 @@ import run.halo.app.infra.utils.JsonUtils;
  * @since 2.0.0
  */
 @Slf4j
-public class RoleReconciler implements Reconciler {
+public class RoleReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
 

src/main/java/run/halo/app/core/extension/reconciler/ThemeReconciler.java

@@ -8,6 +8,7 @@ import run.halo.app.core.extension.Setting;
 import run.halo.app.core.extension.Theme;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 import run.halo.app.infra.exception.ThemeUninstallException;
 import run.halo.app.infra.properties.HaloProperties;
 import run.halo.app.theme.ThemePathPolicy;
@@ -18,7 +19,7 @@ import run.halo.app.theme.ThemePathPolicy;
  * @author guqing
  * @since 2.0.0
  */
-public class ThemeReconciler implements Reconciler {
+public class ThemeReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
     private final ThemePathPolicy themePathPolicy;

src/main/java/run/halo/app/core/extension/reconciler/UserReconciler.java

@@ -3,9 +3,10 @@ package run.halo.app.core.extension.reconciler;
 import lombok.extern.slf4j.Slf4j;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
 
 @Slf4j
-public class UserReconciler implements Reconciler {
+public class UserReconciler implements Reconciler<Request> {
 
     private final ExtensionClient client;
 

src/main/java/run/halo/app/extension/Metadata.java

@@ -2,6 +2,7 @@ package run.halo.app.extension;
 
 import java.time.Instant;
 import java.util.Map;
+import java.util.Set;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
@@ -44,4 +45,6 @@ public class Metadata implements MetadataOperator {
      */
     private Instant deletionTimestamp;
 
+    private Set<String> finalizers;
+
 }

src/main/java/run/halo/app/extension/MetadataOperator.java

@@ -6,6 +6,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
 import java.time.Instant;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * MetadataOperator contains some getters and setters for required fields of metadata.
@@ -40,6 +41,9 @@ public interface MetadataOperator {
     @JsonProperty("deletionTimestamp")
     Instant getDeletionTimestamp();
 
+    @Schema(nullable = true)
+    Set<String> getFinalizers();
+
     void setName(String name);
 
     void setLabels(Map<String, String> labels);
@@ -52,6 +56,8 @@ public interface MetadataOperator {
 
     void setDeletionTimestamp(Instant deletionTimestamp);
 
+    void setFinalizers(Set<String> finalizers);
+
     static boolean metadataDeepEquals(MetadataOperator left, MetadataOperator right) {
         if (left == null && right == null) {
             return true;
@@ -77,6 +83,9 @@ public interface MetadataOperator {
         if (!Objects.equals(left.getVersion(), right.getVersion())) {
             return false;
         }
+        if (!Objects.equals(left.getFinalizers(), right.getFinalizers())) {
+            return false;
+        }
         return true;
     }
 }

src/main/java/run/halo/app/extension/Unstructured.java

@@ -13,10 +13,14 @@ import io.swagger.v3.core.util.Json;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Unstructured is a generic Extension, which wraps ObjectNode to maintain the Extension data, like
@@ -88,6 +92,11 @@ public class Unstructured implements Extension {
             return getNestedInstant(data, "metadata", "deletionTimestamp").orElse(null);
         }
 
+        @Override
+        public Set<String> getFinalizers() {
+            return getNestedStringSet(data, "metadata", "finalizers").orElse(null);
+        }
+
         @Override
         public void setName(String name) {
             setNestedValue(data, name, "metadata", "name");
@@ -117,8 +126,14 @@ public class Unstructured implements Extension {
         public void setDeletionTimestamp(Instant deletionTimestamp) {
             setNestedValue(data, deletionTimestamp, "metadata", "deletionTimestamp");
         }
+
+        @Override
+        public void setFinalizers(Set<String> finalizers) {
+            setNestedValue(data, finalizers, "metadata", "finalizers");
+        }
     }
 
+
     @Override
     public void setApiVersion(String apiVersion) {
         setNestedValue(data, apiVersion, "apiVersion");
@@ -151,6 +166,21 @@ public class Unstructured implements Extension {
         return Optional.ofNullable(tempMap.get(fields[fields.length - 1]));
     }
 
+    @SuppressWarnings("unchecked")
+    static Optional<List<String>> getNestedStringList(Map map, String... fields) {
+        return getNestedValue(map, fields).map(value -> (List<String>) value);
+    }
+
+    static Optional<Set<String>> getNestedStringSet(Map map, String... fields) {
+        return getNestedValue(map, fields).map(value -> {
+            if (value instanceof Collection collection) {
+                return new LinkedHashSet<>(collection);
+            }
+            throw new IllegalArgumentException(
+                "Incorrect value type: " + value.getClass() + ", expected: " + Set.class);
+        });
+    }
+
     @SuppressWarnings("unchecked")
     static void setNestedValue(Map map, Object value, String... fields) {
         if (fields == null || fields.length == 0) {

src/main/java/run/halo/app/extension/controller/ControllerBuilder.java

@@ -9,6 +9,7 @@ import org.springframework.util.Assert;
 import run.halo.app.extension.Extension;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.WatcherPredicates;
+import run.halo.app.extension.controller.Reconciler.Request;
 
 public class ControllerBuilder {
 
@@ -18,7 +19,7 @@ public class ControllerBuilder {
 
     private Duration maxDelay;
 
-    private Reconciler reconciler;
+    private Reconciler<Request> reconciler;
 
     private Supplier<Instant> nowSupplier;
 
@@ -51,7 +52,7 @@ public class ControllerBuilder {
         return this;
     }
 
-    public ControllerBuilder reconciler(Reconciler reconciler) {
+    public ControllerBuilder reconciler(Reconciler<Request> reconciler) {
         this.reconciler = reconciler;
         return this;
     }
@@ -102,7 +103,7 @@ public class ControllerBuilder {
         Assert.notNull(extension, "Extension must not be null");
         Assert.notNull(reconciler, "Reconciler must not be null");
 
-        var queue = new DefaultDelayQueue(nowSupplier, minDelay);
+        var queue = new DefaultDelayQueue<Request>(nowSupplier, minDelay);
         var predicates = new WatcherPredicates.Builder()
             .withGroupVersionKind(extension.groupVersionKind())
             .onAddPredicate(onAddPredicate)
@@ -115,6 +116,6 @@ public class ControllerBuilder {
             extension,
             watcher,
             predicates.onAddPredicate());
-        return new DefaultController(name, reconciler, queue, synchronizer, minDelay, maxDelay);
+        return new DefaultController<>(name, reconciler, queue, synchronizer, minDelay, maxDelay);
     }
 }

src/main/java/run/halo/app/extension/controller/DefaultController.java

@@ -11,15 +11,15 @@ import org.springframework.util.StopWatch;
 import run.halo.app.extension.controller.RequestQueue.DelayedEntry;
 
 @Slf4j
-class DefaultController implements Controller {
+public class DefaultController<R> implements Controller {
 
     private final String name;
 
-    private final Reconciler reconciler;
+    private final Reconciler<R> reconciler;
 
     private final Supplier<Instant> nowSupplier;
 
-    private final RequestQueue queue;
+    private final RequestQueue<R> queue;
 
     private volatile boolean disposed = false;
 
@@ -27,25 +27,25 @@ class DefaultController implements Controller {
 
     private final ExecutorService executor;
 
-    private final RequestSynchronizer synchronizer;
+    private final Synchronizer<R> synchronizer;
 
     private final Duration minDelay;
 
     private final Duration maxDelay;
 
     public DefaultController(String name,
-        Reconciler reconciler,
-        RequestQueue queue,
-        RequestSynchronizer synchronizer,
+        Reconciler<R> reconciler,
+        RequestQueue<R> queue,
+        Synchronizer<R> synchronizer,
         Duration minDelay,
         Duration maxDelay) {
         this(name, reconciler, queue, synchronizer, Instant::now, minDelay, maxDelay);
     }
 
     public DefaultController(String name,
-        Reconciler reconciler,
-        RequestQueue queue,
-        RequestSynchronizer synchronizer,
+        Reconciler<R> reconciler,
+        RequestQueue<R> queue,
+        Synchronizer<R> synchronizer,
         Supplier<Instant> nowSupplier,
         Duration minDelay,
         Duration maxDelay,
@@ -61,9 +61,9 @@ class DefaultController implements Controller {
     }
 
     public DefaultController(String name,
-        Reconciler reconciler,
-        RequestQueue queue,
-        RequestSynchronizer synchronizer,
+        Reconciler<R> reconciler,
+        RequestQueue<R> queue,
+        Synchronizer<R> synchronizer,
         Supplier<Instant> nowSupplier,
         Duration minDelay,
         Duration maxDelay) {
@@ -97,7 +97,7 @@ class DefaultController implements Controller {
                 Reconciler.Result result;
                 try {
                     log.debug("Reconciling request {} at {}", entry.getEntry(), nowSupplier.get());
-                    StopWatch watch = new StopWatch("Reconcile: " + entry.getEntry().name());
+                    StopWatch watch = new StopWatch("Reconcile: " + entry.getEntry());
                     watch.start("reconciliation");
                     result = this.reconciler.reconcile(entry.getEntry());
                     watch.stop();
@@ -111,6 +111,9 @@ class DefaultController implements Controller {
                 } finally {
                     queue.done(entry.getEntry());
                 }
+                if (result == null) {
+                    result = new Reconciler.Result(false, null);
+                }
                 if (!result.reEnqueue()) {
                     continue;
                 }

src/main/java/run/halo/app/extension/controller/DefaultDelayQueue.java

@@ -7,11 +7,10 @@ import java.util.Set;
 import java.util.concurrent.DelayQueue;
 import java.util.function.Supplier;
 import lombok.extern.slf4j.Slf4j;
-import run.halo.app.extension.controller.Reconciler.Request;
 
 @Slf4j
-public class DefaultDelayQueue
-    extends DelayQueue<DefaultDelayQueue.DelayedEntry<Request>> implements RequestQueue {
+public class DefaultDelayQueue<R>
+    extends DelayQueue<DefaultDelayQueue.DelayedEntry<R>> implements RequestQueue<R> {
 
     private final Supplier<Instant> nowSupplier;
 
@@ -19,7 +18,7 @@ public class DefaultDelayQueue
 
     private final Duration minDelay;
 
-    private final Set<Request> processing;
+    private final Set<R> processing;
 
     public DefaultDelayQueue(Supplier<Instant> nowSupplier) {
         this(nowSupplier, Duration.ZERO);
@@ -32,13 +31,14 @@ public class DefaultDelayQueue
     }
 
     @Override
-    public boolean addImmediately(Request request) {
+    public boolean addImmediately(R request) {
+        log.debug("Adding request {} immediately", request);
         var delayedEntry = new DelayedEntry<>(request, minDelay, nowSupplier);
         return offer(delayedEntry);
     }
 
     @Override
-    public boolean add(DelayedEntry<Request> entry) {
+    public boolean add(DelayedEntry<R> entry) {
         if (entry.getRetryAfter().compareTo(minDelay) < 0) {
             log.warn("Request {} will be retried after {} ms, but minimum delay is {} ms",
                 entry.getEntry(), entry.getRetryAfter().toMillis(), minDelay.toMillis());
@@ -48,19 +48,19 @@ public class DefaultDelayQueue
     }
 
     @Override
-    public DelayedEntry<Request> take() throws InterruptedException {
+    public DelayedEntry<R> take() throws InterruptedException {
         var entry = super.take();
         processing.add(entry.getEntry());
         return entry;
     }
 
     @Override
-    public void done(Request request) {
+    public void done(R request) {
         processing.remove(request);
     }
 
     @Override
-    public boolean offer(DelayedEntry<Request> entry) {
+    public boolean offer(DelayedEntry<R> entry) {
         if (this.isDisposed() || processing.contains(entry.getEntry())) {
             return false;
         }

src/main/java/run/halo/app/extension/controller/ExtensionWatcher.java

@@ -3,17 +3,18 @@ package run.halo.app.extension.controller;
 import run.halo.app.extension.Extension;
 import run.halo.app.extension.Watcher;
 import run.halo.app.extension.WatcherPredicates;
+import run.halo.app.extension.controller.Reconciler.Request;
 
 public class ExtensionWatcher implements Watcher {
 
-    private final RequestQueue queue;
+    private final RequestQueue<Request> queue;
 
     private volatile boolean disposed = false;
 
     private Runnable disposeHook;
     private final WatcherPredicates predicates;
 
-    public ExtensionWatcher(RequestQueue queue, WatcherPredicates predicates) {
+    public ExtensionWatcher(RequestQueue<Request> queue, WatcherPredicates predicates) {
         this.queue = queue;
         this.predicates = predicates;
     }
@@ -24,7 +25,7 @@ public class ExtensionWatcher implements Watcher {
             return;
         }
         // TODO filter the event
-        queue.addImmediately(new Reconciler.Request(extension.getMetadata().getName()));
+        queue.addImmediately(new Request(extension.getMetadata().getName()));
     }
 
     @Override
@@ -33,7 +34,7 @@ public class ExtensionWatcher implements Watcher {
             return;
         }
         // TODO filter the event
-        queue.addImmediately(new Reconciler.Request(newExtension.getMetadata().getName()));
+        queue.addImmediately(new Request(newExtension.getMetadata().getName()));
     }
 
     @Override
@@ -42,7 +43,7 @@ public class ExtensionWatcher implements Watcher {
             return;
         }
         // TODO filter the event
-        queue.addImmediately(new Reconciler.Request(extension.getMetadata().getName()));
+        queue.addImmediately(new Request(extension.getMetadata().getName()));
     }
 
     @Override

src/main/java/run/halo/app/extension/controller/Reconciler.java

@@ -2,14 +2,13 @@ package run.halo.app.extension.controller;
 
 import java.time.Duration;
 
-public interface Reconciler {
+public interface Reconciler<R> {
 
-    Result reconcile(Request request);
+    Result reconcile(R request);
 
     record Request(String name) {
     }
 
     record Result(boolean reEnqueue, Duration retryAfter) {
-
     }
 }

src/main/java/run/halo/app/extension/controller/RequestQueue.java

@@ -1,7 +1,5 @@
 package run.halo.app.extension.controller;
 
-import static run.halo.app.extension.controller.Reconciler.Request;
-
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Objects;
@@ -10,15 +8,15 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import reactor.core.Disposable;
 
-public interface RequestQueue extends Disposable {
+public interface RequestQueue<E> extends Disposable {
 
-    boolean addImmediately(Request request);
+    boolean addImmediately(E request);
 
-    boolean add(DelayedEntry<Request> entry);
+    boolean add(DelayedEntry<E> entry);
 
-    DelayedEntry<Request> take() throws InterruptedException;
+    DelayedEntry<E> take() throws InterruptedException;
 
-    void done(Request request);
+    void done(E request);
 
     class DelayedEntry<E> implements Delayed {
 

src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java

@@ -2,13 +2,13 @@ package run.halo.app.extension.controller;
 
 import java.util.function.Predicate;
 import lombok.extern.slf4j.Slf4j;
-import reactor.core.Disposable;
 import run.halo.app.extension.Extension;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.Watcher;
+import run.halo.app.extension.controller.Reconciler.Request;
 
 @Slf4j
-public class RequestSynchronizer implements Disposable {
+public class RequestSynchronizer implements Synchronizer<Request> {
 
     private final ExtensionClient client;
 
@@ -39,6 +39,7 @@ public class RequestSynchronizer implements Disposable {
         this.listPredicate = listPredicate;
     }
 
+    @Override
     public void start() {
         if (isDisposed() || started) {
             return;

src/main/java/run/halo/app/extension/controller/Synchronizer.java

@@ -0,0 +1,9 @@
+package run.halo.app.extension.controller;
+
+import reactor.core.Disposable;
+
+public interface Synchronizer<R> extends Disposable {
+
+    void start();
+
+}

src/main/java/run/halo/app/extension/gc/GarbageCollectorConfiguration.java

@@ -0,0 +1,36 @@
+package run.halo.app.extension.gc;
+
+import java.time.Duration;
+import java.time.Instant;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.ExtensionConverter;
+import run.halo.app.extension.SchemeManager;
+import run.halo.app.extension.controller.Controller;
+import run.halo.app.extension.controller.DefaultController;
+import run.halo.app.extension.controller.DefaultDelayQueue;
+import run.halo.app.extension.store.ExtensionStoreClient;
+
+@Configuration(proxyBeanMethods = false)
+public class GarbageCollectorConfiguration {
+
+
+    @Bean
+    Controller garbageCollector(ExtensionClient client,
+        ExtensionStoreClient storeClient,
+        ExtensionConverter converter,
+        SchemeManager schemeManager) {
+        var reconciler = new GcReconciler(client, storeClient, converter);
+        var queue = new DefaultDelayQueue<GcRequest>(Instant::now, Duration.ofMillis(500));
+        var synchronizer = new GcSynchronizer(client, queue, schemeManager);
+        return new DefaultController<>(
+            "garbage-collector-controller",
+            reconciler,
+            queue,
+            synchronizer,
+            Duration.ofMillis(500),
+            Duration.ofSeconds(1000)
+        );
+    }
+}

src/main/java/run/halo/app/extension/gc/GcReconciler.java

@@ -0,0 +1,48 @@
+package run.halo.app.extension.gc;
+
+import java.util.function.Predicate;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
+import run.halo.app.extension.Extension;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.ExtensionConverter;
+import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.store.ExtensionStoreClient;
+
+@Slf4j
+class GcReconciler implements Reconciler<GcRequest> {
+
+    private final ExtensionClient client;
+
+    private final ExtensionStoreClient storeClient;
+
+    private final ExtensionConverter converter;
+
+    GcReconciler(ExtensionClient client, ExtensionStoreClient storeClient,
+        ExtensionConverter converter) {
+        this.client = client;
+        this.storeClient = storeClient;
+        this.converter = converter;
+    }
+
+
+    @Override
+    public Result reconcile(GcRequest request) {
+        log.debug("Extension {} is being deleted", request);
+
+        client.fetch(request.gvk(), request.name())
+            .filter(deletable())
+            .ifPresent(extension -> {
+                var extensionStore = converter.convertTo(extension);
+                storeClient.delete(extensionStore.getName(), extensionStore.getVersion());
+                log.debug("Extension {} was deleted", request);
+            });
+
+        return null;
+    }
+
+    private Predicate<Extension> deletable() {
+        return extension -> CollectionUtils.isEmpty(extension.getMetadata().getFinalizers())
+            && extension.getMetadata().getDeletionTimestamp() != null;
+    }
+}

src/main/java/run/halo/app/extension/gc/GcRequest.java

@@ -0,0 +1,12 @@
+package run.halo.app.extension.gc;
+
+import org.springframework.util.Assert;
+import run.halo.app.extension.GroupVersionKind;
+
+record GcRequest(GroupVersionKind gvk, String name) {
+
+    public GcRequest {
+        Assert.notNull(gvk, "Group, version and kind must not be null");
+        Assert.hasText(name, "Extension name must not be blank");
+    }
+}

src/main/java/run/halo/app/extension/gc/GcSynchronizer.java

@@ -0,0 +1,65 @@
+package run.halo.app.extension.gc;
+
+import java.util.function.Predicate;
+import run.halo.app.extension.Extension;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.Scheme;
+import run.halo.app.extension.SchemeManager;
+import run.halo.app.extension.Watcher;
+import run.halo.app.extension.controller.RequestQueue;
+import run.halo.app.extension.controller.Synchronizer;
+
+class GcSynchronizer implements Synchronizer<GcRequest> {
+
+    private final ExtensionClient client;
+
+    private final RequestQueue<GcRequest> queue;
+
+    private final SchemeManager schemeManager;
+
+    private boolean disposed = false;
+
+    private boolean started = false;
+
+    private final Watcher watcher;
+
+    GcSynchronizer(ExtensionClient client, RequestQueue<GcRequest> queue,
+        SchemeManager schemeManager) {
+        this.client = client;
+        this.queue = queue;
+        this.schemeManager = schemeManager;
+        this.watcher = new GcWatcher(queue);
+    }
+
+    @Override
+    public void dispose() {
+        if (isDisposed()) {
+            return;
+        }
+        this.disposed = true;
+        this.watcher.dispose();
+    }
+
+    @Override
+    public boolean isDisposed() {
+        return disposed;
+    }
+
+    @Override
+    public void start() {
+        if (isDisposed() || started) {
+            return;
+        }
+        this.started = true;
+        client.watch(watcher);
+        schemeManager.schemes().stream()
+            .map(Scheme::type)
+            .forEach(type -> client.list(type, deleted(), null)
+                .forEach(watcher::onDelete));
+    }
+
+    private <E extends Extension> Predicate<E> deleted() {
+        return extension -> extension.getMetadata().getDeletionTimestamp() != null;
+    }
+
+}

src/main/java/run/halo/app/extension/gc/GcWatcher.java

@@ -0,0 +1,64 @@
+package run.halo.app.extension.gc;
+
+import run.halo.app.extension.Extension;
+import run.halo.app.extension.Watcher;
+import run.halo.app.extension.controller.RequestQueue;
+
+class GcWatcher implements Watcher {
+
+    private final RequestQueue<GcRequest> queue;
+
+    private Runnable disposeHook;
+
+    private boolean disposed = false;
+
+    GcWatcher(RequestQueue<GcRequest> queue) {
+        this.queue = queue;
+    }
+
+    @Override
+    public void onAdd(Extension extension) {
+        // TODO Should we ignore finalizers here?
+        if (!isDisposed() && extension.getMetadata().getDeletionTimestamp() != null) {
+            queue.addImmediately(
+                new GcRequest(extension.groupVersionKind(), extension.getMetadata().getName()));
+        }
+    }
+
+    @Override
+    public void onUpdate(Extension oldExt, Extension newExt) {
+        if (!isDisposed() && newExt.getMetadata().getDeletionTimestamp() != null) {
+            queue.addImmediately(
+                new GcRequest(newExt.groupVersionKind(), newExt.getMetadata().getName()));
+        }
+    }
+
+    @Override
+    public void onDelete(Extension extension) {
+        if (!isDisposed() && extension.getMetadata().getDeletionTimestamp() != null) {
+            queue.addImmediately(
+                new GcRequest(extension.groupVersionKind(), extension.getMetadata().getName()));
+        }
+    }
+
+    @Override
+    public void registerDisposeHook(Runnable dispose) {
+        this.disposeHook = dispose;
+    }
+
+    @Override
+    public void dispose() {
+        if (isDisposed()) {
+            return;
+        }
+        this.disposed = true;
+        if (this.disposeHook != null) {
+            this.disposeHook.run();
+        }
+    }
+
+    @Override
+    public boolean isDisposed() {
+        return disposed;
+    }
+}

src/main/java/run/halo/app/extension/store/ExtensionStoreClientJPAImpl.java

@@ -4,7 +4,7 @@ import jakarta.persistence.EntityNotFoundException;
 import java.util.List;
 import java.util.Optional;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
+import reactor.core.publisher.Mono;
 
 /**
  * An implementation of ExtensionStoreClient using JPA.
@@ -43,13 +43,15 @@ public class ExtensionStoreClientJPAImpl implements ExtensionStoreClient {
     }
 
     @Override
-    @Transactional
     public ExtensionStore delete(String name, Long version) {
-        var extensionStore =
-            repository.findById(name).blockOptional().orElseThrow(EntityNotFoundException::new);
-        extensionStore.setVersion(version);
-        repository.delete(extensionStore);
-        return extensionStore;
+        return repository.findById(name)
+            .switchIfEmpty(Mono.error(() -> new EntityNotFoundException(
+                "Extension store with name " + name + " was not found.")))
+            .flatMap(deleting -> {
+                deleting.setVersion(version);
+                return repository.delete(deleting).thenReturn(deleting);
+            })
+            .block();
     }
 
 }

src/main/java/run/halo/app/infra/utils/JsonUtils.java

@@ -2,13 +2,11 @@ package run.halo.app.infra.utils;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.swagger.v3.core.util.Json;
 import java.util.Map;
 import org.springframework.lang.NonNull;
-import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
 /**
@@ -19,40 +17,11 @@ import org.springframework.util.Assert;
  * @since 2.0.0
  */
 public class JsonUtils {
-    public static final ObjectMapper DEFAULT_JSON_MAPPER = createDefaultJsonMapper();
+    public static final ObjectMapper DEFAULT_JSON_MAPPER = Json.mapper();
 
     private JsonUtils() {
     }
 
-    /**
-     * Creates a default json mapper.
-     *
-     * @return object mapper
-     */
-    public static ObjectMapper createDefaultJsonMapper() {
-        return createDefaultJsonMapper(null);
-    }
-
-    /**
-     * Creates a default json mapper.
-     *
-     * @param strategy property naming strategy
-     * @return object mapper
-     */
-    @NonNull
-    public static ObjectMapper createDefaultJsonMapper(@Nullable PropertyNamingStrategy strategy) {
-        // Create object mapper
-        ObjectMapper mapper = new ObjectMapper();
-        // Configure
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        mapper.registerModule(new JavaTimeModule());
-        // Set property naming strategy
-        if (strategy != null) {
-            mapper.setPropertyNamingStrategy(strategy);
-        }
-        return mapper;
-    }
-
     /**
      * Converts a map to the object specified type.
      *

src/test/java/run/halo/app/content/ContentRequestTest.java

@@ -55,21 +55,13 @@ class ContentRequestTest {
                         "rawType": "MARKDOWN",
                         "rawPatch": "%s",
                         "contentPatch": "%s",
-                        "parentSnapshotName": null,
                         "displayVersion": "v1",
-                        "version": 1,
-                        "publishTime": null,
-                        "contributors": null
+                        "version": 1
                     },
                     "apiVersion": "content.halo.run/v1alpha1",
                     "kind": "Snapshot",
                     "metadata": {
-                        "name": "7b149646-ac60-4a5c-98ee-78b2dd0631b2",
-                        "labels": null,
-                        "annotations": null,
-                        "version": null,
-                        "creationTimestamp": null,
-                        "deletionTimestamp": null
+                        "name": "7b149646-ac60-4a5c-98ee-78b2dd0631b2"
                     }
                 }
                 """.formatted(expectedRawPatch, expectedContentPath),
@@ -88,16 +80,14 @@ class ContentRequestTest {
                     {
                         "source": {
                             "position": 3,
-                            "lines": [],
-                            "changePosition": null
+                            "lines": []
                         },
                         "target": {
                             "position": 3,
                             "lines": [
                                 "brought forth on this continent",
                                 ""
-                            ],
-                            "changePosition": null
+                            ]
                         },
                         "type": "INSERT"
                     }
@@ -116,16 +106,14 @@ class ContentRequestTest {
                  {
                      "source": {
                          "position": 2,
-                         "lines": [],
-                         "changePosition": null
+                         "lines": []
                      },
                      "target": {
                          "position": 2,
                          "lines": [
                              "<br/>",
                              "<p>brought forth on this continent</p>"
-                         ],
-                         "changePosition": null
+                         ]
                      },
                      "type": "INSERT"
                  }

src/test/java/run/halo/app/core/extension/SettingTest.java

@@ -90,12 +90,7 @@ class SettingTest {
                     "apiVersion": "v1alpha1",
                     "kind": "Setting",
                     "metadata": {
-                        "name": "setting-name",
-                        "labels": null,
-                        "annotations": null,
-                        "version": null,
-                        "creationTimestamp": null,
-                        "deletionTimestamp": null
+                        "name": "setting-name"
                     }
                 }
                 """,

src/test/java/run/halo/app/core/extension/ThemeTest.java

@@ -62,12 +62,7 @@ class ThemeTest {
                     "apiVersion": "theme.halo.run/v1alpha1",
                     "kind": "Theme",
                     "metadata": {
-                        "name": "test-theme",
-                        "labels": null,
-                        "annotations": null,
-                        "version": null,
-                        "creationTimestamp": null,
-                        "deletionTimestamp": null
+                        "name": "test-theme"
                     }
                 }
                 """,

src/test/java/run/halo/app/extension/MetadataOperatorTest.java

@@ -8,6 +8,7 @@ import static run.halo.app.extension.MetadataOperator.metadataDeepEquals;
 
 import java.time.Instant;
 import java.util.Map;
+import java.util.Set;
 import org.junit.jupiter.api.Test;
 
 class MetadataOperatorTest {
@@ -53,6 +54,11 @@ class MetadataOperatorTest {
         assertFalse(metadataDeepEquals(left, right));
         right.setName(null);
         assertTrue(metadataDeepEquals(left, right));
+
+        left.setFinalizers(null);
+        assertFalse(metadataDeepEquals(left, right));
+        right.setFinalizers(null);
+        assertTrue(metadataDeepEquals(left, right));
     }
 
     @Test
@@ -64,6 +70,8 @@ class MetadataOperatorTest {
         when(mockMetadata.getVersion()).thenReturn(123L);
         when(mockMetadata.getCreationTimestamp()).thenReturn(now);
         when(mockMetadata.getDeletionTimestamp()).thenReturn(now);
+        when(mockMetadata.getFinalizers())
+            .thenReturn(Set.of("fake-finalizer-1", "fake-finalizer-2"));
 
         var metadata = createFullMetadata();
         assertTrue(metadataDeepEquals(metadata, mockMetadata));
@@ -77,6 +85,7 @@ class MetadataOperatorTest {
         metadata.setVersion(123L);
         metadata.setCreationTimestamp(now);
         metadata.setDeletionTimestamp(now);
+        metadata.setFinalizers(Set.of("fake-finalizer-2", "fake-finalizer-1"));
         return metadata;
     }
 }

src/test/java/run/halo/app/extension/UnstructuredTest.java

@@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.time.Instant;
 import java.util.Map;
+import java.util.Set;
 import org.json.JSONException;
 import org.junit.jupiter.api.Test;
 import org.skyscreamer.jsonassert.JSONAssert;
@@ -28,7 +29,8 @@ class UnstructuredTest {
                 },
                 "name": "fake-extension",
                 "creationTimestamp": "2011-12-03T10:15:30Z",
-                "version": 12345
+                "version": 12345,
+                "finalizers": ["finalizer.1", "finalizer.2"]
             }
         }
         """;
@@ -95,6 +97,13 @@ class UnstructuredTest {
         assertNotEquals(createUnstructured(), another);
     }
 
+    @Test
+    void shouldGetFinalizersCorrectly() throws JsonProcessingException, JSONException {
+        var extension = objectMapper.readValue(extensionJson, Unstructured.class);
+
+        assertEquals(Set.of("finalizer.1", "finalizer.2"), extension.getMetadata().getFinalizers());
+    }
+
     Unstructured createUnstructured() {
         var unstructured = new Unstructured();
         unstructured.setApiVersion("fake.halo.run/v1alpha1");

src/test/java/run/halo/app/extension/controller/DefaultControllerTest.java

@@ -30,10 +30,10 @@ import run.halo.app.extension.controller.RequestQueue.DelayedEntry;
 class DefaultControllerTest {
 
     @Mock
-    RequestQueue queue;
+    RequestQueue<Request> queue;
 
     @Mock
-    Reconciler reconciler;
+    Reconciler<Request> reconciler;
 
     @Mock
     RequestSynchronizer synchronizer;
@@ -47,11 +47,11 @@ class DefaultControllerTest {
 
     Duration maxRetryAfter = Duration.ofSeconds(10);
 
-    DefaultController controller;
+    DefaultController<Request> controller;
 
     @BeforeEach
     void setUp() {
-        controller = new DefaultController("fake-controller", reconciler, queue, synchronizer,
+        controller = new DefaultController<>("fake-controller", reconciler, queue, synchronizer,
             () -> now, minRetryAfter, maxRetryAfter, executor);
 
         assertFalse(controller.isDisposed());

src/test/java/run/halo/app/extension/controller/DefaultDelayQueueTest.java

@@ -20,13 +20,13 @@ class DefaultDelayQueueTest {
 
     Instant now = Instant.now();
 
-    DefaultDelayQueue queue;
+    DefaultDelayQueue<Request> queue;
 
     final Duration minDelay = Duration.ofMillis(1);
 
     @BeforeEach
     void setUp() {
-        queue = new DefaultDelayQueue(() -> now, minDelay);
+        queue = new DefaultDelayQueue<>(() -> now, minDelay);
     }
 
     @Test

src/test/java/run/halo/app/extension/controller/ExtensionWatcherTest.java

@@ -14,12 +14,13 @@ import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import run.halo.app.extension.WatcherPredicates;
+import run.halo.app.extension.controller.Reconciler.Request;
 
 @ExtendWith(MockitoExtension.class)
 class ExtensionWatcherTest {
 
     @Mock
-    RequestQueue queue;
+    RequestQueue<Request> queue;
 
     @Mock
     WatcherPredicates predicates;

src/test/java/run/halo/app/extension/gc/GcReconcilerTest.java

@@ -0,0 +1,116 @@
+package run.halo.app.extension.gc;
+
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.Optional;
+import java.util.Set;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.ExtensionConverter;
+import run.halo.app.extension.FakeExtension;
+import run.halo.app.extension.Metadata;
+import run.halo.app.extension.Unstructured;
+import run.halo.app.extension.store.ExtensionStore;
+import run.halo.app.extension.store.ExtensionStoreClient;
+
+@ExtendWith(MockitoExtension.class)
+class GcReconcilerTest {
+
+    @Mock
+    ExtensionClient client;
+
+    @Mock
+    ExtensionStoreClient storeClient;
+
+    @Mock
+    ExtensionConverter converter;
+
+    @InjectMocks
+    GcReconciler reconciler;
+
+    @Test
+    void shouldDoNothingIfExtensionNotFound() {
+        var fake = createExtension();
+        when(client.fetch(fake.groupVersionKind(), fake.getMetadata().getName()))
+            .thenReturn(Optional.empty());
+
+        var result = reconciler.reconcile(createGcRequest());
+        assertNull(result);
+        verify(converter, never()).convertTo(any());
+        verify(storeClient, never()).delete(any(), any());
+    }
+
+    @Test
+    void shouldDoNothingIfFinalizersPresent() {
+        var fake = createExtension();
+        fake.getMetadata().setFinalizers(Set.of("fake-finalizer"));
+        fake.getMetadata().setDeletionTimestamp(null);
+        when(client.fetch(fake.groupVersionKind(), fake.getMetadata().getName()))
+            .thenReturn(Optional.of(convertTo(fake)));
+
+        var result = reconciler.reconcile(createGcRequest());
+        assertNull(result);
+        verify(converter, never()).convertTo(any());
+        verify(storeClient, never()).delete(any(), any());
+    }
+
+    @Test
+    void shouldDoNothingIfDeletionTimestampIsNull() {
+        var fake = createExtension();
+        fake.getMetadata().setDeletionTimestamp(null);
+        fake.getMetadata().setFinalizers(null);
+        when(client.fetch(fake.groupVersionKind(), fake.getMetadata().getName()))
+            .thenReturn(Optional.of(convertTo(fake)));
+
+        var result = reconciler.reconcile(createGcRequest());
+        assertNull(result);
+        verify(converter, never()).convertTo(any());
+        verify(storeClient, never()).delete(any(), any());
+    }
+
+    @Test
+    void shouldDeleteCorrectly() {
+        var fake = createExtension();
+        fake.getMetadata().setDeletionTimestamp(Instant.now());
+        fake.getMetadata().setFinalizers(null);
+        when(client.fetch(fake.groupVersionKind(), fake.getMetadata().getName()))
+            .thenReturn(Optional.of(convertTo(fake)));
+
+        ExtensionStore store = new ExtensionStore();
+        store.setName("fake-store-name");
+        store.setVersion(1L);
+
+        when(converter.convertTo(any())).thenReturn(store);
+
+        var result = reconciler.reconcile(createGcRequest());
+        assertNull(result);
+        verify(converter).convertTo(any());
+        verify(storeClient).delete("fake-store-name", 1L);
+    }
+
+    GcRequest createGcRequest() {
+        var fake = createExtension();
+        return new GcRequest(fake.groupVersionKind(), fake.getMetadata().getName());
+    }
+
+    Unstructured convertTo(FakeExtension fake) {
+        return Unstructured.OBJECT_MAPPER.convertValue(fake, Unstructured.class);
+    }
+
+    FakeExtension createExtension() {
+        var fake = new FakeExtension();
+        var metadata = new Metadata();
+        metadata.setName("fake");
+        fake.setMetadata(metadata);
+        return fake;
+    }
+}

src/test/java/run/halo/app/extension/gc/GcWatcherTest.java

@@ -0,0 +1,92 @@
+package run.halo.app.extension.gc;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.time.Instant;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import run.halo.app.extension.FakeExtension;
+import run.halo.app.extension.Metadata;
+import run.halo.app.extension.controller.RequestQueue;
+
+@ExtendWith(MockitoExtension.class)
+class GcWatcherTest {
+
+    @Mock
+    RequestQueue<GcRequest> queue;
+
+    @InjectMocks
+    GcWatcher watcher;
+
+    @Test
+    void shouldAddIntoQueueWhenDeletionTimestampSet() {
+        var fake = createExtension();
+        fake.getMetadata().setDeletionTimestamp(Instant.now());
+
+        watcher.onAdd(fake);
+        verify(queue).addImmediately(any(GcRequest.class));
+
+        watcher.onUpdate(fake, fake);
+        verify(queue, times(2)).addImmediately(any(GcRequest.class));
+
+        watcher.onDelete(fake);
+        verify(queue, times(3)).addImmediately(any(GcRequest.class));
+    }
+
+    @Test
+    void shouldNotAddIntoQueueWhenDeletionTimestampNotSet() {
+        var fake = createExtension();
+        watcher.onAdd(fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+
+        watcher.onUpdate(fake, fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+
+        watcher.onDelete(fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+    }
+
+    @Test
+    void shouldNotAddIntoQueueWhenDisposed() {
+        var fake = createExtension();
+        fake.getMetadata().setDeletionTimestamp(Instant.now());
+        watcher.dispose();
+
+        watcher.onAdd(fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+
+        watcher.onUpdate(fake, fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+
+        watcher.onDelete(fake);
+        verify(queue, never()).addImmediately(any(GcRequest.class));
+    }
+
+    @Test
+    void shouldDisposeHookCorrectly() {
+        var run = mock(Runnable.class);
+        watcher.registerDisposeHook(run);
+        assertFalse(watcher.isDisposed());
+        watcher.dispose();
+        assertTrue(watcher.isDisposed());
+        verify(run).run();
+    }
+
+
+    FakeExtension createExtension() {
+        var fake = new FakeExtension();
+        Metadata metadata = new Metadata();
+        metadata.setName("fake");
+        fake.setMetadata(metadata);
+        return fake;
+    }
+}

src/test/java/run/halo/app/plugin/YamlPluginFinderTest.java

@@ -52,13 +52,7 @@ class YamlPluginFinderTest {
         assertThat(plugin).isNotNull();
         JSONAssert.assertEquals("""
                 {
-                    "phase": "RESOLVED",
-                    "reason": null,
-                    "message": null,
-                    "lastStartTime": null,
-                    "lastTransitionTime": null,
-                    "entry": null,
-                    "stylesheet": null
+                    "phase": "RESOLVED"
                 }
                 """,
             JsonUtils.objectToJson(plugin.getStatus()),
@@ -92,26 +86,16 @@ class YamlPluginFinderTest {
                         "description": "Tell me more about this plugin.",
                         "license": [
                             {
-                                "name": "MIT",
-                                "url": null
+                                "name": "MIT"
                             }
                         ],
                         "requires": ">=2.0.0",
-                        "pluginClass": null,
-                        "enabled": false,
-                        settingName: null,
-                        configMapName: null
+                        "enabled": false
                     },
-                    "status": null,
                     "apiVersion": "plugin.halo.run/v1alpha1",
                     "kind": "Plugin",
                     "metadata": {
-                        "name": "plugin-1",
-                        "labels": null,
-                        "annotations": null,
-                        "version": null,
-                        "creationTimestamp": null,
-                        "deletionTimestamp": null
+                        "name": "plugin-1"
                     }
                 }
                   """,