refactor: optimize query for category association posts count to prevent blocking due to large data (#5658)

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.15.x

#### What this PR does / why we need it:
优化分类关联文章数量的查询避免因查询数据量过大而导致的阻塞或内存溢出

#### Does this PR introduce a user-facing change?
```release-note
优化分类关联文章数量的查询避免因查询数据量过大而导致的阻塞或内存溢出
```

api/src/main/java/run/halo/app/core/extension/content/Post.java

@@ -222,54 +222,4 @@ public class Post extends AbstractExtension {
             return null;
         }
     }
-
-    @Data
-    public static class CompactPost {
-        private String name;
-
-        private VisibleEnum visible;
-
-        private Boolean published;
-
-        public static Builder builder() {
-            return new Builder();
-        }
-
-        /**
-         * <p>Compact post builder.</p>
-         * <p>Can not replace with lombok builder.</p>
-         * <p>The class used by subclasses of {@link AbstractExtension} must have a no-args
-         * constructor.</p>
-         */
-        public static class Builder {
-            private String name;
-
-            private VisibleEnum visible;
-
-            private Boolean published;
-
-            public Builder name(String name) {
-                this.name = name;
-                return this;
-            }
-
-            public Builder visible(VisibleEnum visible) {
-                this.visible = visible;
-                return this;
-            }
-
-            public Builder published(Boolean published) {
-                this.published = published;
-                return this;
-            }
-
-            public CompactPost build() {
-                CompactPost compactPost = new CompactPost();
-                compactPost.setName(name);
-                compactPost.setVisible(visible);
-                compactPost.setPublished(published);
-                return compactPost;
-            }
-        }
-    }
 }

application/src/main/java/run/halo/app/core/extension/reconciler/CategoryReconciler.java

@@ -2,6 +2,7 @@ package run.halo.app.core.extension.reconciler;
 
 import static run.halo.app.extension.index.query.QueryFactory.and;
 import static run.halo.app.extension.index.query.QueryFactory.equal;
+import static run.halo.app.extension.index.query.QueryFactory.in;
 import static run.halo.app.extension.index.query.QueryFactory.isNull;
 
 import java.time.Duration;
@@ -11,14 +12,13 @@ import java.util.Deque;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.AllArgsConstructor;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.data.domain.Sort;
-import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Component;
 import run.halo.app.content.permalinks.CategoryPermalinkPolicy;
 import run.halo.app.core.extension.content.Category;
@@ -27,10 +27,13 @@ import run.halo.app.core.extension.content.Post;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.ListOptions;
 import run.halo.app.extension.MetadataUtil;
+import run.halo.app.extension.PageRequestImpl;
 import run.halo.app.extension.controller.Controller;
 import run.halo.app.extension.controller.ControllerBuilder;
 import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.index.query.Query;
 import run.halo.app.extension.router.selector.FieldSelector;
+import run.halo.app.extension.router.selector.LabelSelector;
 import run.halo.app.infra.utils.JsonUtils;
 
 /**
@@ -42,7 +45,7 @@ import run.halo.app.infra.utils.JsonUtils;
 @Component
 @AllArgsConstructor
 public class CategoryReconciler implements Reconciler<Reconciler.Request> {
-    private static final String FINALIZER_NAME = "category-protection";
+    static final String FINALIZER_NAME = "category-protection";
     private final ExtensionClient client;
     private final CategoryPermalinkPolicy categoryPermalinkPolicy;
 
@@ -140,53 +143,46 @@ public class CategoryReconciler implements Reconciler<Reconciler.Request> {
 
     private void populatePosts(Category category) {
         String name = category.getMetadata().getName();
-        List<String> categoryNames = listChildrenByName(name)
+        Set<String> categoryNames = listChildrenByName(name)
             .stream()
             .map(item -> item.getMetadata().getName())
-            .toList();
+            .collect(Collectors.toSet());
 
-        var postListOptions = new ListOptions();
+        var totalPostCount = countTotalPosts(categoryNames);
-        postListOptions.setFieldSelector(FieldSelector.of(
+        category.getStatusOrDefault().setPostCount(totalPostCount);
-            and(isNull("metadata.deletionTimestamp"),
-                equal("spec.deleted", "false")))
-        );
-        var posts = client.listAll(Post.class, postListOptions, Sort.unsorted());
 
-        // populate post to status
+        var visiblePostCount = countVisiblePosts(categoryNames);
-        List<Post.CompactPost> compactPosts = posts.stream()
+        category.getStatusOrDefault().setVisiblePostCount(visiblePostCount);
-            .filter(post -> includes(post.getSpec().getCategories(), categoryNames))
-            .map(post -> Post.CompactPost.builder()
-                .name(post.getMetadata().getName())
-                .visible(post.getSpec().getVisible())
-                .published(post.isPublished())
-                .build())
-            .toList();
-        category.getStatusOrDefault().setPostCount(compactPosts.size());
-
-        long visiblePostCount = compactPosts.stream()
-            .filter(post -> Objects.equals(true, post.getPublished())
-                && Post.VisibleEnum.PUBLIC.equals(post.getVisible()))
-            .count();
-        category.getStatusOrDefault().setVisiblePostCount((int) visiblePostCount);
     }
 
-    /**
+    int countTotalPosts(Set<String> categoryNames) {
-     * whether {@code categoryRefs} contains elements in {@code categoryNames}.
+        var postListOptions = new ListOptions();
-     *
+        postListOptions.setFieldSelector(FieldSelector.of(
-     * @param categoryRefs category left to judge
+            basePostQuery(categoryNames)
-     * @param categoryNames category right to judge
+        ));
-     * @return true if {@code categoryRefs} contains elements in {@code categoryNames}.
+        return (int) client.listBy(Post.class, postListOptions, PageRequestImpl.ofSize(1))
-     */
+            .getTotal();
-    private boolean includes(@Nullable List<String> categoryRefs, List<String> categoryNames) {
+    }
-        if (categoryRefs == null || categoryNames == null) {
+
-            return false;
+    int countVisiblePosts(Set<String> categoryNames) {
-        }
+        var postListOptions = new ListOptions();
-        for (String categoryRef : categoryRefs) {
+        var fieldQuery = and(basePostQuery(categoryNames),
-            if (categoryNames.contains(categoryRef)) {
+            equal("spec.visible", Post.VisibleEnum.PUBLIC.name())
-                return true;
+        );
-            }
+        var labelSelector = LabelSelector.builder()
-        }
+            .eq(Post.PUBLISHED_LABEL, BooleanUtils.TRUE)
-        return false;
+            .build();
+        postListOptions.setFieldSelector(FieldSelector.of(fieldQuery));
+        postListOptions.setLabelSelector(labelSelector);
+        return (int) client.listBy(Post.class, postListOptions, PageRequestImpl.ofSize(1))
+            .getTotal();
+    }
+
+    private static Query basePostQuery(Set<String> categoryNames) {
+        return and(isNull("metadata.deletionTimestamp"),
+            equal("spec.deleted", BooleanUtils.FALSE),
+            in("spec.categories", categoryNames)
+        );
     }
 
     private List<Category> listChildrenByName(String name) {

application/src/test/java/run/halo/app/core/extension/reconciler/CategoryReconcilerIntegrationTest.java

@@ -0,0 +1,230 @@
+package run.halo.app.core.extension.reconciler;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Set;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.SpyBean;
+import org.springframework.test.annotation.DirtiesContext;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.test.StepVerifier;
+import run.halo.app.content.TestPost;
+import run.halo.app.core.extension.content.Category;
+import run.halo.app.core.extension.content.Post;
+import run.halo.app.extension.Extension;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.ExtensionStoreUtil;
+import run.halo.app.extension.Metadata;
+import run.halo.app.extension.ReactiveExtensionClient;
+import run.halo.app.extension.SchemeManager;
+import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.index.IndexerFactory;
+import run.halo.app.extension.store.ReactiveExtensionStoreClient;
+
+/**
+ * Tests for {@link CategoryReconciler}.
+ *
+ * @author guqing
+ * @since 2.0.0
+ */
+@DirtiesContext
+@SpringBootTest
+class CategoryReconcilerIntegrationTest {
+    private final List<Post> storedPosts = posts();
+    private final List<Category> storedCategories = categories();
+
+    @Autowired
+    private SchemeManager schemeManager;
+
+    @SpyBean
+    private ExtensionClient client;
+
+    @Autowired
+    private ReactiveExtensionClient reactiveClient;
+
+    @Autowired
+    private ReactiveExtensionStoreClient storeClient;
+
+    @Autowired
+    private IndexerFactory indexerFactory;
+
+    @Autowired
+    private CategoryReconciler categoryReconciler;
+
+    Mono<Extension> deleteImmediately(Extension extension) {
+        var name = extension.getMetadata().getName();
+        var scheme = schemeManager.get(extension.getClass());
+        // un-index
+        var indexer = indexerFactory.getIndexer(extension.groupVersionKind());
+        indexer.unIndexRecord(extension.getMetadata().getName());
+
+        // delete from db
+        var storeName = ExtensionStoreUtil.buildStoreName(scheme, name);
+        return storeClient.delete(storeName, extension.getMetadata().getVersion())
+            .thenReturn(extension);
+    }
+
+    @BeforeEach
+    void setUp() {
+        Flux.fromIterable(storedPosts)
+            .flatMap(post -> reactiveClient.create(post))
+            .as(StepVerifier::create)
+            .expectNextCount(storedPosts.size())
+            .verifyComplete();
+
+        Flux.fromIterable(storedCategories)
+            .flatMap(category -> reactiveClient.create(category))
+            .as(StepVerifier::create)
+            .expectNextCount(storedCategories.size())
+            .verifyComplete();
+    }
+
+    @AfterEach
+    void tearDown() {
+        Flux.fromIterable(storedPosts)
+            .flatMap(this::deleteImmediately)
+            .as(StepVerifier::create)
+            .expectNextCount(storedPosts.size())
+            .verifyComplete();
+
+        Flux.fromIterable(storedCategories)
+            .flatMap(this::deleteImmediately)
+            .as(StepVerifier::create)
+            .expectNextCount(storedCategories.size())
+            .verifyComplete();
+    }
+
+    @Test
+    void reconcileStatusPostForCategoryA() {
+        reconcileStatusPostPilling("category-A");
+
+        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
+        verify(client, times(3)).update(captor.capture());
+        var values = captor.getAllValues();
+        assertThat(values.get(2).getStatusOrDefault().getPostCount()).isEqualTo(4);
+        assertThat(
+            captor.getAllValues().get(2).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
+    }
+
+    @Test
+    void reconcileStatusPostForCategoryB() {
+        reconcileStatusPostPilling("category-B");
+        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
+        verify(client, times(3)).update(captor.capture());
+        Category category = captor.getAllValues().get(2);
+        assertThat(category.getStatusOrDefault().getPostCount()).isEqualTo(3);
+        assertThat(category.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
+    }
+
+    @Test
+    void reconcileStatusPostForCategoryC() {
+        reconcileStatusPostPilling("category-C");
+        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
+        verify(client, times(3)).update(captor.capture());
+        var value = captor.getAllValues().get(2);
+        assertThat(value.getStatusOrDefault().getPostCount()).isEqualTo(2);
+        assertThat(value.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
+    }
+
+    @Test
+    void reconcileStatusPostForCategoryD() {
+        reconcileStatusPostPilling("category-D");
+        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
+        verify(client, times(3)).update(captor.capture());
+        var value = captor.getAllValues().get(2);
+        assertThat(value.getStatusOrDefault().postCount).isEqualTo(1);
+        assertThat(value.getStatusOrDefault().visiblePostCount).isEqualTo(0);
+    }
+
+    private void reconcileStatusPostPilling(String reconcileCategoryName) {
+        Reconciler.Result result =
+            categoryReconciler.reconcile(new Reconciler.Request(reconcileCategoryName));
+        assertThat(result.reEnqueue()).isTrue();
+        assertThat(result.retryAfter()).isEqualTo(Duration.ofMinutes(1));
+    }
+
+    private List<Category> categories() {
+        /*
+         * |-A(post-4)
+         *   |-B(post-3)
+         *   |-|-C(post-2,post-1)
+         *   |-D(post-1)
+         */
+        Category categoryA = category("category-A");
+        categoryA.getSpec().setChildren(List.of("category-B", "category-D"));
+
+        Category categoryB = category("category-B");
+        categoryB.getSpec().setChildren(List.of("category-C"));
+
+        Category categoryC = category("category-C");
+        Category categoryD = category("category-D");
+        return List.of(categoryA, categoryB, categoryC, categoryD);
+    }
+
+    private Category category(String name) {
+        Category category = new Category();
+        Metadata metadata = new Metadata();
+        metadata.setName(name);
+        category.setMetadata(metadata);
+        category.getMetadata().setFinalizers(Set.of(CategoryReconciler.FINALIZER_NAME));
+        category.setSpec(new Category.CategorySpec());
+        category.setStatus(new Category.CategoryStatus());
+
+        category.getSpec().setDisplayName("display-name");
+        category.getSpec().setSlug("slug");
+        category.getSpec().setPriority(0);
+        return category;
+    }
+
+    private List<Post> posts() {
+        /*
+         * |-A(post-4)
+         *   |-B(post-3)
+         *   |-|-C(post-2,post-1)
+         *   |-D(post-1)
+         */
+        Post post1 = fakePost();
+        post1.getMetadata().setName("post-1");
+        post1.getSpec().setCategories(List.of("category-D", "category-C"));
+        post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
+
+        Post post2 = fakePost();
+        post2.getMetadata().setName("post-2");
+        post2.getSpec().setCategories(List.of("category-C"));
+        post2.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
+
+        Post post3 = fakePost();
+        post3.getMetadata().setName("post-3");
+        post3.getSpec().setCategories(List.of("category-B"));
+        post3.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
+
+        Post post4 = fakePost();
+        post4.getMetadata().setName("post-4");
+        post4.getSpec().setCategories(List.of("category-A"));
+        post4.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
+        return List.of(post1, post2, post3, post4);
+    }
+
+    Post fakePost() {
+        var post = TestPost.postV1();
+        post.getSpec().setAllowComment(true);
+        post.getSpec().setDeleted(false);
+        post.getSpec().setExcerpt(new Post.Excerpt());
+        post.getSpec().getExcerpt().setAutoGenerate(false);
+        post.getSpec().setPinned(false);
+        post.getSpec().setPriority(0);
+        post.getSpec().setPublish(false);
+        post.getSpec().setSlug("fake-post");
+        return post;
+    }
+}

application/src/test/java/run/halo/app/core/extension/reconciler/CategoryReconcilerTest.java

@@ -1,160 +0,0 @@
-package run.halo.app.core.extension.reconciler;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.lenient;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import java.time.Duration;
-import java.util.List;
-import java.util.Optional;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.ArgumentCaptor;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.springframework.data.domain.Sort;
-import run.halo.app.content.TestPost;
-import run.halo.app.content.permalinks.CategoryPermalinkPolicy;
-import run.halo.app.core.extension.content.Category;
-import run.halo.app.core.extension.content.Post;
-import run.halo.app.extension.ExtensionClient;
-import run.halo.app.extension.ListOptions;
-import run.halo.app.extension.Metadata;
-import run.halo.app.extension.controller.Reconciler;
-
-/**
- * Tests for {@link CategoryReconciler}.
- *
- * @author guqing
- * @since 2.0.0
- */
-@ExtendWith(MockitoExtension.class)
-class CategoryReconcilerTest {
-    @Mock
-    private ExtensionClient client;
-
-    @Mock
-    private CategoryPermalinkPolicy categoryPermalinkPolicy;
-
-    @InjectMocks
-    private CategoryReconciler categoryReconciler;
-
-    @Test
-    void reconcileStatusPostForCategoryA() {
-        reconcileStatusPostPilling("category-A");
-
-        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
-        verify(client, times(3)).update(captor.capture());
-        assertThat(captor.getAllValues().get(1).getStatusOrDefault().getPostCount()).isEqualTo(4);
-        assertThat(
-            captor.getAllValues().get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
-    }
-
-    @Test
-    void reconcileStatusPostForCategoryB() {
-        reconcileStatusPostPilling("category-B");
-        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
-        verify(client, times(3)).update(captor.capture());
-        Category category = captor.getAllValues().get(1);
-        assertThat(category.getStatusOrDefault().getPostCount()).isEqualTo(3);
-        assertThat(category.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
-    }
-
-    @Test
-    void reconcileStatusPostForCategoryC() {
-        reconcileStatusPostPilling("category-C");
-        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
-        verify(client, times(3)).update(captor.capture());
-        assertThat(captor.getAllValues().get(1).getStatusOrDefault().getPostCount()).isEqualTo(2);
-        assertThat(
-            captor.getAllValues().get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
-    }
-
-    @Test
-    void reconcileStatusPostForCategoryD() {
-        reconcileStatusPostPilling("category-D");
-        ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
-        verify(client, times(3)).update(captor.capture());
-        assertThat(captor.getAllValues().get(1).getStatusOrDefault().postCount).isEqualTo(1);
-        assertThat(captor.getAllValues().get(1).getStatusOrDefault().visiblePostCount).isEqualTo(0);
-    }
-
-
-    private void reconcileStatusPostPilling(String reconcileCategoryName) {
-        categories().forEach(category -> {
-            lenient().when(client.fetch(eq(Category.class), eq(category.getMetadata().getName())))
-                .thenReturn(Optional.of(category));
-        });
-
-        lenient().when(client.listAll(eq(Post.class), any(ListOptions.class), any(Sort.class)))
-            .thenReturn(posts());
-        lenient().when(client.listAll(eq(Category.class), any(), any()))
-            .thenReturn(categories());
-
-        Reconciler.Result result =
-            categoryReconciler.reconcile(new Reconciler.Request(reconcileCategoryName));
-        assertThat(result.reEnqueue()).isTrue();
-        assertThat(result.retryAfter()).isEqualTo(Duration.ofMinutes(1));
-    }
-
-    private List<Category> categories() {
-        /*
-         * |-A(post-4)
-         *   |-B(post-3)
-         *   |-|-C(post-2,post-1)
-         *   |-D(post-1)
-         */
-        Category categoryA = category("category-A");
-        categoryA.getSpec().setChildren(List.of("category-B", "category-D"));
-
-        Category categoryB = category("category-B");
-        categoryB.getSpec().setChildren(List.of("category-C"));
-
-        Category categoryC = category("category-C");
-        Category categoryD = category("category-D");
-        return List.of(categoryA, categoryB, categoryC, categoryD);
-    }
-
-    private Category category(String name) {
-        Category category = new Category();
-        Metadata metadata = new Metadata();
-        metadata.setName(name);
-        category.setMetadata(metadata);
-        category.setSpec(new Category.CategorySpec());
-        category.setStatus(new Category.CategoryStatus());
-        return category;
-    }
-
-    private List<Post> posts() {
-        /*
-         * |-A(post-4)
-         *   |-B(post-3)
-         *   |-|-C(post-2,post-1)
-         *   |-D(post-1)
-         */
-        Post post1 = TestPost.postV1();
-        post1.getMetadata().setName("post-1");
-        post1.getSpec().setCategories(List.of("category-D", "category-C"));
-        post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
-
-        Post post2 = TestPost.postV1();
-        post2.getMetadata().setName("post-2");
-        post2.getSpec().setCategories(List.of("category-C"));
-        post2.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
-
-        Post post3 = TestPost.postV1();
-        post3.getMetadata().setName("post-3");
-        post3.getSpec().setCategories(List.of("category-B"));
-        post3.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
-
-        Post post4 = TestPost.postV1();
-        post4.getMetadata().setName("post-4");
-        post4.getSpec().setCategories(List.of("category-A"));
-        post4.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
-        return List.of(post1, post2, post3, post4);
-    }
-}