refactor: optimize the update method for the number of tag associated with posts (#5422)

#### What type of PR is this?
/kind improvement
/area core

#### What this PR does / why we need it:
优化标签关联的文章数量的更新方式以降低标签太多时对 CPU 的消耗

how to test it?
测试文章关联标签、取消关联、更新关联、删除文章到回收站、彻底删除文章时标签对应的文章可见文章数量和所有文章数量是否正确

#### Does this PR introduce a user-facing change?
```release-note
优化标签关联的文章数量的更新方式以降低标签太多时对 CPU 的消耗
```

api/src/main/java/run/halo/app/core/extension/content/Post.java

@@ -41,7 +41,8 @@ public class Post extends AbstractExtension {
     public static final String CATEGORIES_ANNO = "content.halo.run/categories";
     public static final String LAST_RELEASED_SNAPSHOT_ANNO =
         "content.halo.run/last-released-snapshot";
-    public static final String TAGS_ANNO = "content.halo.run/tags";
+    public static final String LAST_ASSOCIATED_TAGS_ANNO = "content.halo.run/last-associated-tags";
+
     public static final String DELETED_LABEL = "content.halo.run/deleted";
     public static final String PUBLISHED_LABEL = "content.halo.run/published";
     public static final String OWNER_LABEL = "content.halo.run/owner";

api/src/main/java/run/halo/app/core/extension/content/Tag.java

@@ -27,6 +27,8 @@ public class Tag extends AbstractExtension {
 
     public static final GroupVersionKind GVK = GroupVersionKind.fromExtension(Tag.class);
 
+    public static final String REQUIRE_SYNC_ON_STARTUP_INDEX_NAME = "requireSyncOnStartup";
+
     @Schema(requiredMode = REQUIRED)
     private TagSpec spec;
 
@@ -77,5 +79,7 @@ public class Tag extends AbstractExtension {
         public Integer visiblePostCount;
 
         public Integer postCount;
+
+        private Long observedVersion;
     }
 }

application/src/main/java/run/halo/app/content/TagPostCountUpdater.java

@@ -0,0 +1,186 @@
+package run.halo.app.content;
+
+import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
+import static run.halo.app.extension.index.query.QueryFactory.and;
+import static run.halo.app.extension.index.query.QueryFactory.equal;
+import static run.halo.app.extension.index.query.QueryFactory.isNull;
+
+import com.google.common.collect.Sets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.context.SmartLifecycle;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+import run.halo.app.core.extension.content.Post;
+import run.halo.app.core.extension.content.Tag;
+import run.halo.app.core.extension.content.Tag.TagStatus;
+import run.halo.app.event.post.PostDeletedEvent;
+import run.halo.app.event.post.PostEvent;
+import run.halo.app.event.post.PostUpdatedEvent;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.ListOptions;
+import run.halo.app.extension.MetadataUtil;
+import run.halo.app.extension.PageRequestImpl;
+import run.halo.app.extension.controller.Controller;
+import run.halo.app.extension.controller.ControllerBuilder;
+import run.halo.app.extension.controller.DefaultController;
+import run.halo.app.extension.controller.DefaultQueue;
+import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.RequestQueue;
+import run.halo.app.extension.router.selector.FieldSelector;
+import run.halo.app.extension.router.selector.LabelSelector;
+import run.halo.app.infra.utils.JsonUtils;
+
+/**
+ * Update {@link TagStatus#postCount} when post related to tag is updated.
+ *
+ * @author guqing
+ * @since 2.13.0
+ */
+@Component
+public class TagPostCountUpdater
+    implements Reconciler<TagPostCountUpdater.PostRelatedTags>, SmartLifecycle {
+
+    private final RequestQueue<PostRelatedTags> tagQueue;
+
+    private final Controller postEventController;
+
+    private final ExtensionClient client;
+
+    private volatile boolean running = false;
+
+    /**
+     * Construct a {@link TagPostCountUpdater} with the given {@link ExtensionClient}.
+     */
+    public TagPostCountUpdater(ExtensionClient client) {
+        this.client = client;
+
+        this.tagQueue = new DefaultQueue<>(Instant::now);
+        this.postEventController = this.setupWith(null);
+    }
+
+    @Override
+    public Result reconcile(PostRelatedTags postRelatedTags) {
+        for (var tag : postRelatedTags.tags()) {
+            updateTagRelatedPostCount(tag);
+        }
+
+        // Update last associated tags when handled
+        client.fetch(Post.class, postRelatedTags.postName()).ifPresent(post -> {
+            var tags = defaultIfNull(post.getSpec().getTags(), List.<String>of());
+            var annotations = MetadataUtil.nullSafeAnnotations(post);
+            var tagAnno = JsonUtils.objectToJson(tags);
+            var oldTagAnno = annotations.get(Post.LAST_ASSOCIATED_TAGS_ANNO);
+
+            if (!tagAnno.equals(oldTagAnno)) {
+                annotations.put(Post.LAST_ASSOCIATED_TAGS_ANNO, tagAnno);
+                client.update(post);
+            }
+        });
+        return Result.doNotRetry();
+    }
+
+
+    @Override
+    public Controller setupWith(ControllerBuilder builder) {
+        return new DefaultController<>(
+            this.getClass().getName(),
+            this,
+            tagQueue,
+            null,
+            Duration.ofMillis(100),
+            Duration.ofMinutes(10)
+        );
+    }
+
+    @Override
+    public void start() {
+        postEventController.start();
+        running = true;
+    }
+
+    @Override
+    public void stop() {
+        running = false;
+        postEventController.dispose();
+    }
+
+    @Override
+    public boolean isRunning() {
+        return running;
+    }
+
+    /**
+     * Listen to post event to calculate post related to tag for updating.
+     */
+    @EventListener(PostEvent.class)
+    public void onPostUpdated(PostEvent postEvent) {
+        var postName = postEvent.getName();
+        if (postEvent instanceof PostUpdatedEvent) {
+            var tagsToUpdate = calcTagsToUpdate(postEvent.getName());
+            tagQueue.addImmediately(new PostRelatedTags(postName, tagsToUpdate));
+            return;
+        }
+
+        if (postEvent instanceof PostDeletedEvent deletedEvent) {
+            var tags = defaultIfNull(deletedEvent.getPost().getSpec().getTags(),
+                List.<String>of());
+            tagQueue.addImmediately(new PostRelatedTags(postName, Sets.newHashSet(tags)));
+        }
+    }
+
+    private Set<String> calcTagsToUpdate(String postName) {
+        var post = client.fetch(Post.class, postName).orElseThrow();
+        var annotations = MetadataUtil.nullSafeAnnotations(post);
+        var oldTags = Optional.ofNullable(annotations.get(Post.LAST_ASSOCIATED_TAGS_ANNO))
+            .filter(StringUtils::isNotBlank)
+            .map(tagsJson -> JsonUtils.jsonToObject(tagsJson, String[].class))
+            .orElse(new String[0]);
+
+        var tagsToUpdate = Sets.newHashSet(oldTags);
+        var newTags = post.getSpec().getTags();
+        if (newTags != null) {
+            tagsToUpdate.addAll(newTags);
+        }
+        return tagsToUpdate;
+    }
+
+    public record PostRelatedTags(String postName, Set<String> tags) {
+    }
+
+    private void updateTagRelatedPostCount(String tagName) {
+        client.fetch(Tag.class, tagName).ifPresent(tag -> {
+            var commonFieldQuery = and(
+                equal("spec.tags", tag.getMetadata().getName()),
+                isNull("metadata.deletionTimestamp")
+            );
+            // Update post count
+            var allPostOptions = new ListOptions();
+            allPostOptions.setFieldSelector(FieldSelector.of(commonFieldQuery));
+            var result = client.listBy(Post.class, allPostOptions, PageRequestImpl.ofSize(1));
+            tag.getStatusOrDefault().setPostCount((int) result.getTotal());
+
+            // Update visible post count
+            var publicPostOptions = new ListOptions();
+            publicPostOptions.setLabelSelector(LabelSelector.builder()
+                .eq(Post.PUBLISHED_LABEL, "true")
+                .build());
+            publicPostOptions.setFieldSelector(FieldSelector.of(
+                and(
+                    commonFieldQuery,
+                    equal("spec.deleted", "false"),
+                    equal("spec.visible", Post.VisibleEnum.PUBLIC.name())
+                )
+            ));
+            var publicPosts =
+                client.listBy(Post.class, publicPostOptions, PageRequestImpl.ofSize(1));
+            tag.getStatusOrDefault().setVisiblePostCount((int) publicPosts.getTotal());
+
+            client.update(tag);
+        });
+    }
+}

application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java

@@ -33,6 +33,7 @@ import run.halo.app.core.extension.content.Post.PostPhase;
 import run.halo.app.core.extension.content.Post.VisibleEnum;
 import run.halo.app.core.extension.content.Snapshot;
 import run.halo.app.core.extension.notification.Subscription;
+import run.halo.app.event.post.PostDeletedEvent;
 import run.halo.app.event.post.PostPublishedEvent;
 import run.halo.app.event.post.PostUnpublishedEvent;
 import run.halo.app.event.post.PostUpdatedEvent;
@@ -86,6 +87,7 @@ public class PostReconciler implements Reconciler<Reconciler.Request> {
                 if (ExtensionOperator.isDeleted(post)) {
                     removeFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME));
                     unPublishPost(post, events);
+                    events.add(new PostDeletedEvent(this, post));
                     cleanUpResources(post);
                     // update post to be able to be collected by gc collector.
                     client.update(post);
@@ -126,7 +128,7 @@ public class PostReconciler implements Reconciler<Reconciler.Request> {
 
                 // calculate the sha256sum
                 var configSha256sum = Hashing.sha256().hashString(post.getSpec().toString(), UTF_8)
-                        .toString();
+                    .toString();
 
                 var oldConfigChecksum = annotations.get(Constant.CHECKSUM_CONFIG_ANNO);
                 if (!Objects.equals(oldConfigChecksum, configSha256sum)) {

application/src/main/java/run/halo/app/core/extension/reconciler/TagReconciler.java

@@ -1,30 +1,25 @@
 package run.halo.app.core.extension.reconciler;
 
-import static org.apache.commons.lang3.BooleanUtils.isFalse;
+import static run.halo.app.extension.ExtensionUtil.addFinalizers;
-import static run.halo.app.extension.MetadataUtil.nullSafeLabels;
+import static run.halo.app.extension.ExtensionUtil.removeFinalizers;
 import static run.halo.app.extension.index.query.QueryFactory.equal;
 
-import java.time.Duration;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.data.domain.Sort;
 import org.springframework.stereotype.Component;
 import run.halo.app.content.permalinks.TagPermalinkPolicy;
 import run.halo.app.core.extension.content.Constant;
-import run.halo.app.core.extension.content.Post;
 import run.halo.app.core.extension.content.Tag;
+import run.halo.app.extension.DefaultExtensionMatcher;
 import run.halo.app.extension.ExtensionClient;
-import run.halo.app.extension.ListOptions;
+import run.halo.app.extension.ExtensionUtil;
 import run.halo.app.extension.MetadataUtil;
 import run.halo.app.extension.controller.Controller;
 import run.halo.app.extension.controller.ControllerBuilder;
 import run.halo.app.extension.controller.Reconciler;
 import run.halo.app.extension.router.selector.FieldSelector;
-import run.halo.app.infra.utils.JsonUtils;
 
 /**
  * Reconciler for {@link Tag}.
@@ -35,123 +30,50 @@ import run.halo.app.infra.utils.JsonUtils;
 @Component
 @RequiredArgsConstructor
 public class TagReconciler implements Reconciler<Reconciler.Request> {
-    private static final String FINALIZER_NAME = "tag-protection";
+    static final String FINALIZER_NAME = "tag-protection";
     private final ExtensionClient client;
     private final TagPermalinkPolicy tagPermalinkPolicy;
 
     @Override
     public Result reconcile(Request request) {
-        return client.fetch(Tag.class, request.name())
+        client.fetch(Tag.class, request.name())
-            .map(tag -> {
+            .ifPresent(tag -> {
-                if (isDeleted(tag)) {
+                if (ExtensionUtil.isDeleted(tag)) {
-                    cleanUpResourcesAndRemoveFinalizer(request.name());
+                    if (removeFinalizers(tag.getMetadata(), Set.of(FINALIZER_NAME))) {
-                    return Result.doNotRetry();
+                        client.update(tag);
+                    }
+                    return;
                 }
-                addFinalizerIfNecessary(tag);
 
-                reconcileMetadata(request.name());
+                addFinalizers(tag.getMetadata(), Set.of(FINALIZER_NAME));
 
-                this.reconcileStatusPermalink(request.name());
+                Map<String, String> annotations = MetadataUtil.nullSafeAnnotations(tag);
 
-                reconcileStatusPosts(request.name());
+                String newPattern = tagPermalinkPolicy.pattern();
-                return new Result(true, Duration.ofMinutes(1));
+                annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern);
-            })
+
-            .orElse(Result.doNotRetry());
+                String permalink = tagPermalinkPolicy.permalink(tag);
+                var status = tag.getStatusOrDefault();
+                status.setPermalink(permalink);
+
+                // Update the observed version.
+                status.setObservedVersion(tag.getMetadata().getVersion() + 1);
+
+                client.update(tag);
+            });
+        return Result.doNotRetry();
     }
 
     @Override
     public Controller setupWith(ControllerBuilder builder) {
         return builder
-            .syncAllOnStart(true)
             .extension(new Tag())
+            .onAddMatcher(DefaultExtensionMatcher.builder(client, Tag.GVK)
+                .fieldSelector(FieldSelector.of(
+                    equal(Tag.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME, BooleanUtils.TRUE))
+                )
+                .build()
+            )
             .build();
     }
-
-    void reconcileMetadata(String name) {
-        client.fetch(Tag.class, name).ifPresent(tag -> {
-            Map<String, String> annotations = MetadataUtil.nullSafeAnnotations(tag);
-            String oldPermalinkPattern = annotations.get(Constant.PERMALINK_PATTERN_ANNO);
-
-            String newPattern = tagPermalinkPolicy.pattern();
-            annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern);
-
-            if (!StringUtils.equals(oldPermalinkPattern, newPattern)) {
-                client.update(tag);
-            }
-        });
-    }
-
-    private void addFinalizerIfNecessary(Tag oldTag) {
-        Set<String> finalizers = oldTag.getMetadata().getFinalizers();
-        if (finalizers != null && finalizers.contains(FINALIZER_NAME)) {
-            return;
-        }
-        client.fetch(Tag.class, oldTag.getMetadata().getName())
-            .ifPresent(tag -> {
-                Set<String> newFinalizers = tag.getMetadata().getFinalizers();
-                if (newFinalizers == null) {
-                    newFinalizers = new HashSet<>();
-                    tag.getMetadata().setFinalizers(newFinalizers);
-                }
-                newFinalizers.add(FINALIZER_NAME);
-                client.update(tag);
-            });
-    }
-
-    private void cleanUpResourcesAndRemoveFinalizer(String tagName) {
-        client.fetch(Tag.class, tagName).ifPresent(tag -> {
-            if (tag.getMetadata().getFinalizers() != null) {
-                tag.getMetadata().getFinalizers().remove(FINALIZER_NAME);
-            }
-            client.update(tag);
-        });
-    }
-
-    private void reconcileStatusPermalink(String tagName) {
-        client.fetch(Tag.class, tagName)
-            .ifPresent(tag -> {
-                String oldPermalink = tag.getStatusOrDefault().getPermalink();
-                String permalink = tagPermalinkPolicy.permalink(tag);
-                tag.getStatusOrDefault().setPermalink(permalink);
-
-                if (!StringUtils.equals(permalink, oldPermalink)) {
-                    client.update(tag);
-                }
-            });
-    }
-
-    private void reconcileStatusPosts(String tagName) {
-        client.fetch(Tag.class, tagName).ifPresent(tag -> {
-            Tag oldTag = JsonUtils.deepCopy(tag);
-
-            populatePosts(tag);
-
-            if (!oldTag.equals(tag)) {
-                client.update(tag);
-            }
-        });
-    }
-
-    private void populatePosts(Tag tag) {
-        // populate post-count
-        var listOptions = new ListOptions();
-        listOptions.setFieldSelector(FieldSelector.of(
-            equal("spec.tags", tag.getMetadata().getName()))
-        );
-        var posts = client.listAll(Post.class, listOptions, Sort.unsorted());
-        tag.getStatusOrDefault().setPostCount(posts.size());
-
-        var publicPosts = posts.stream()
-            .filter(post -> post.getMetadata().getDeletionTimestamp() == null
-                && isFalse(post.getSpec().getDeleted())
-                && BooleanUtils.TRUE.equals(nullSafeLabels(post).get(Post.PUBLISHED_LABEL))
-                && Post.VisibleEnum.PUBLIC.equals(post.getSpec().getVisible())
-            )
-            .toList();
-        tag.getStatusOrDefault().setVisiblePostCount(publicPosts.size());
-    }
-
-    private boolean isDeleted(Tag tag) {
-        return tag.getMetadata().getDeletionTimestamp() != null;
-    }
 }

application/src/main/java/run/halo/app/event/post/PostDeletedEvent.java

@@ -0,0 +1,21 @@
+package run.halo.app.event.post;
+
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+import run.halo.app.core.extension.content.Post;
+
+@Getter
+public class PostDeletedEvent extends ApplicationEvent implements PostEvent {
+
+    private final Post post;
+
+    public PostDeletedEvent(Object source, Post post) {
+        super(source);
+        this.post = post;
+    }
+
+    @Override
+    public String getName() {
+        return post.getMetadata().getName();
+    }
+}

application/src/main/java/run/halo/app/infra/SchemeInitializer.java

@@ -199,6 +199,17 @@ public class SchemeInitializer implements ApplicationListener<ApplicationContext
                 .setName("spec.slug")
                 .setIndexFunc(simpleAttribute(Tag.class, tag -> tag.getSpec().getSlug()))
             );
+            indexSpecs.add(new IndexSpec()
+                .setName(Tag.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME)
+                .setIndexFunc(simpleAttribute(Tag.class, tag -> {
+                    var version = tag.getMetadata().getVersion();
+                    var observedVersion = tag.getStatusOrDefault().getObservedVersion();
+                    if (observedVersion == null || observedVersion < version) {
+                        return BooleanUtils.TRUE;
+                    }
+                    // do not care about the false case so return null to avoid indexing
+                    return null;
+                })));
         });
         schemeManager.register(Snapshot.class, indexSpecs -> {
             indexSpecs.add(new IndexSpec()

application/src/test/java/run/halo/app/core/extension/reconciler/TagReconcilerTest.java

@@ -8,8 +8,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.time.Instant;
-import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.ArgumentCaptor;
@@ -17,7 +17,6 @@ import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import run.halo.app.content.permalinks.TagPermalinkPolicy;
-import run.halo.app.core.extension.content.Post;
 import run.halo.app.core.extension.content.Tag;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.Metadata;
@@ -44,21 +43,20 @@ class TagReconcilerTest {
         Tag tag = tag();
         when(client.fetch(eq(Tag.class), eq("fake-tag")))
             .thenReturn(Optional.of(tag));
-        when(client.listAll(eq(Post.class), any(), any())).thenReturn(List.of());
         when(tagPermalinkPolicy.permalink(any()))
             .thenAnswer(arg -> "/tags/" + tag.getSpec().getSlug());
         ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
 
         tagReconciler.reconcile(new TagReconciler.Request("fake-tag"));
 
-        verify(client, times(3)).update(captor.capture());
+        verify(client).update(captor.capture());
         Tag capture = captor.getValue();
         assertThat(capture.getStatus().getPermalink()).isEqualTo("/tags/fake-slug");
 
         // change slug
         tag.getSpec().setSlug("new-slug");
         tagReconciler.reconcile(new TagReconciler.Request("fake-tag"));
-        verify(client, times(5)).update(captor.capture());
+        verify(client, times(2)).update(captor.capture());
         assertThat(capture.getStatus().getPermalink()).isEqualTo("/tags/new-slug");
     }
 
@@ -66,6 +64,7 @@ class TagReconcilerTest {
     void reconcileDelete() {
         Tag tag = tag();
         tag.getMetadata().setDeletionTimestamp(Instant.now());
+        tag.getMetadata().setFinalizers(Set.of(TagReconciler.FINALIZER_NAME));
         when(client.fetch(eq(Tag.class), eq("fake-tag")))
             .thenReturn(Optional.of(tag));
         ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
@@ -75,33 +74,10 @@ class TagReconcilerTest {
         verify(tagPermalinkPolicy, times(0)).permalink(any());
     }
 
-    @Test
-    void reconcileStatusPosts() {
-        Tag tag = tag();
-        when(client.fetch(eq(Tag.class), eq("fake-tag")))
-            .thenReturn(Optional.of(tag));
-        when(client.listAll(eq(Post.class), any(), any()))
-            .thenReturn(List.of(createPost("fake-post-1"), createPost("fake-post-2")));
-
-        ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
-        tagReconciler.reconcile(new TagReconciler.Request("fake-tag"));
-        verify(client, times(2)).update(captor.capture());
-        List<Tag> allValues = captor.getAllValues();
-        assertThat(allValues.get(1).getStatusOrDefault().getPostCount()).isEqualTo(2);
-        assertThat(allValues.get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
-    }
-
-    Post createPost(String name) {
-        var post = new Post();
-        post.setMetadata(new Metadata());
-        post.getMetadata().setName(name);
-        post.setSpec(new Post.PostSpec());
-        return post;
-    }
-
     Tag tag() {
         Tag tag = new Tag();
         tag.setMetadata(new Metadata());
+        tag.getMetadata().setVersion(0L);
         tag.getMetadata().setName("fake-tag");
 
         tag.setSpec(new Tag.TagSpec());