refactor: optimize query for category association posts count to prevent blocking due to large data (#5658)

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.15.x

#### What this PR does / why we need it:
优化分类关联文章数量的查询避免因查询数据量过大而导致的阻塞或内存溢出

#### Does this PR introduce a user-facing change?
```release-note
优化分类关联文章数量的查询避免因查询数据量过大而导致的阻塞或内存溢出
```
pull/5614/head^2
guqing 2024-04-08 17:18:08 +08:00 committed by GitHub
parent c630a37eea
commit 52204d6487
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 270 additions and 254 deletions

View File

@ -222,54 +222,4 @@ public class Post extends AbstractExtension {
return null;
}
}
@Data
public static class CompactPost {
private String name;
private VisibleEnum visible;
private Boolean published;
public static Builder builder() {
return new Builder();
}
/**
* <p>Compact post builder.</p>
* <p>Can not replace with lombok builder.</p>
* <p>The class used by subclasses of {@link AbstractExtension} must have a no-args
* constructor.</p>
*/
public static class Builder {
private String name;
private VisibleEnum visible;
private Boolean published;
public Builder name(String name) {
this.name = name;
return this;
}
public Builder visible(VisibleEnum visible) {
this.visible = visible;
return this;
}
public Builder published(Boolean published) {
this.published = published;
return this;
}
public CompactPost build() {
CompactPost compactPost = new CompactPost();
compactPost.setName(name);
compactPost.setVisible(visible);
compactPost.setPublished(published);
return compactPost;
}
}
}
}

View File

@ -2,6 +2,7 @@ package run.halo.app.core.extension.reconciler;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.in;
import static run.halo.app.extension.index.query.QueryFactory.isNull;
import java.time.Duration;
@ -11,14 +12,13 @@ import java.util.Deque;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.AllArgsConstructor;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.Sort;
import org.springframework.lang.Nullable;
import org.springframework.stereotype.Component;
import run.halo.app.content.permalinks.CategoryPermalinkPolicy;
import run.halo.app.core.extension.content.Category;
@ -27,10 +27,13 @@ import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.MetadataUtil;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.controller.Controller;
import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.Reconciler;
import run.halo.app.extension.index.query.Query;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
import run.halo.app.infra.utils.JsonUtils;
/**
@ -42,7 +45,7 @@ import run.halo.app.infra.utils.JsonUtils;
@Component
@AllArgsConstructor
public class CategoryReconciler implements Reconciler<Reconciler.Request> {
private static final String FINALIZER_NAME = "category-protection";
static final String FINALIZER_NAME = "category-protection";
private final ExtensionClient client;
private final CategoryPermalinkPolicy categoryPermalinkPolicy;
@ -140,53 +143,46 @@ public class CategoryReconciler implements Reconciler<Reconciler.Request> {
private void populatePosts(Category category) {
String name = category.getMetadata().getName();
List<String> categoryNames = listChildrenByName(name)
Set<String> categoryNames = listChildrenByName(name)
.stream()
.map(item -> item.getMetadata().getName())
.toList();
.collect(Collectors.toSet());
var totalPostCount = countTotalPosts(categoryNames);
category.getStatusOrDefault().setPostCount(totalPostCount);
var visiblePostCount = countVisiblePosts(categoryNames);
category.getStatusOrDefault().setVisiblePostCount(visiblePostCount);
}
int countTotalPosts(Set<String> categoryNames) {
var postListOptions = new ListOptions();
postListOptions.setFieldSelector(FieldSelector.of(
and(isNull("metadata.deletionTimestamp"),
equal("spec.deleted", "false")))
basePostQuery(categoryNames)
));
return (int) client.listBy(Post.class, postListOptions, PageRequestImpl.ofSize(1))
.getTotal();
}
int countVisiblePosts(Set<String> categoryNames) {
var postListOptions = new ListOptions();
var fieldQuery = and(basePostQuery(categoryNames),
equal("spec.visible", Post.VisibleEnum.PUBLIC.name())
);
var posts = client.listAll(Post.class, postListOptions, Sort.unsorted());
// populate post to status
List<Post.CompactPost> compactPosts = posts.stream()
.filter(post -> includes(post.getSpec().getCategories(), categoryNames))
.map(post -> Post.CompactPost.builder()
.name(post.getMetadata().getName())
.visible(post.getSpec().getVisible())
.published(post.isPublished())
.build())
.toList();
category.getStatusOrDefault().setPostCount(compactPosts.size());
long visiblePostCount = compactPosts.stream()
.filter(post -> Objects.equals(true, post.getPublished())
&& Post.VisibleEnum.PUBLIC.equals(post.getVisible()))
.count();
category.getStatusOrDefault().setVisiblePostCount((int) visiblePostCount);
var labelSelector = LabelSelector.builder()
.eq(Post.PUBLISHED_LABEL, BooleanUtils.TRUE)
.build();
postListOptions.setFieldSelector(FieldSelector.of(fieldQuery));
postListOptions.setLabelSelector(labelSelector);
return (int) client.listBy(Post.class, postListOptions, PageRequestImpl.ofSize(1))
.getTotal();
}
/**
* whether {@code categoryRefs} contains elements in {@code categoryNames}.
*
* @param categoryRefs category left to judge
* @param categoryNames category right to judge
* @return true if {@code categoryRefs} contains elements in {@code categoryNames}.
*/
private boolean includes(@Nullable List<String> categoryRefs, List<String> categoryNames) {
if (categoryRefs == null || categoryNames == null) {
return false;
}
for (String categoryRef : categoryRefs) {
if (categoryNames.contains(categoryRef)) {
return true;
}
}
return false;
private static Query basePostQuery(Set<String> categoryNames) {
return and(isNull("metadata.deletionTimestamp"),
equal("spec.deleted", BooleanUtils.FALSE),
in("spec.categories", categoryNames)
);
}
private List<Category> listChildrenByName(String name) {

View File

@ -0,0 +1,230 @@
package run.halo.app.core.extension.reconciler;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.time.Duration;
import java.util.List;
import java.util.Set;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.annotation.DirtiesContext;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import run.halo.app.content.TestPost;
import run.halo.app.core.extension.content.Category;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionStoreUtil;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.extension.SchemeManager;
import run.halo.app.extension.controller.Reconciler;
import run.halo.app.extension.index.IndexerFactory;
import run.halo.app.extension.store.ReactiveExtensionStoreClient;
/**
* Tests for {@link CategoryReconciler}.
*
* @author guqing
* @since 2.0.0
*/
@DirtiesContext
@SpringBootTest
class CategoryReconcilerIntegrationTest {
private final List<Post> storedPosts = posts();
private final List<Category> storedCategories = categories();
@Autowired
private SchemeManager schemeManager;
@SpyBean
private ExtensionClient client;
@Autowired
private ReactiveExtensionClient reactiveClient;
@Autowired
private ReactiveExtensionStoreClient storeClient;
@Autowired
private IndexerFactory indexerFactory;
@Autowired
private CategoryReconciler categoryReconciler;
Mono<Extension> deleteImmediately(Extension extension) {
var name = extension.getMetadata().getName();
var scheme = schemeManager.get(extension.getClass());
// un-index
var indexer = indexerFactory.getIndexer(extension.groupVersionKind());
indexer.unIndexRecord(extension.getMetadata().getName());
// delete from db
var storeName = ExtensionStoreUtil.buildStoreName(scheme, name);
return storeClient.delete(storeName, extension.getMetadata().getVersion())
.thenReturn(extension);
}
@BeforeEach
void setUp() {
Flux.fromIterable(storedPosts)
.flatMap(post -> reactiveClient.create(post))
.as(StepVerifier::create)
.expectNextCount(storedPosts.size())
.verifyComplete();
Flux.fromIterable(storedCategories)
.flatMap(category -> reactiveClient.create(category))
.as(StepVerifier::create)
.expectNextCount(storedCategories.size())
.verifyComplete();
}
@AfterEach
void tearDown() {
Flux.fromIterable(storedPosts)
.flatMap(this::deleteImmediately)
.as(StepVerifier::create)
.expectNextCount(storedPosts.size())
.verifyComplete();
Flux.fromIterable(storedCategories)
.flatMap(this::deleteImmediately)
.as(StepVerifier::create)
.expectNextCount(storedCategories.size())
.verifyComplete();
}
@Test
void reconcileStatusPostForCategoryA() {
reconcileStatusPostPilling("category-A");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
var values = captor.getAllValues();
assertThat(values.get(2).getStatusOrDefault().getPostCount()).isEqualTo(4);
assertThat(
captor.getAllValues().get(2).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryB() {
reconcileStatusPostPilling("category-B");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
Category category = captor.getAllValues().get(2);
assertThat(category.getStatusOrDefault().getPostCount()).isEqualTo(3);
assertThat(category.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryC() {
reconcileStatusPostPilling("category-C");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
var value = captor.getAllValues().get(2);
assertThat(value.getStatusOrDefault().getPostCount()).isEqualTo(2);
assertThat(value.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryD() {
reconcileStatusPostPilling("category-D");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
var value = captor.getAllValues().get(2);
assertThat(value.getStatusOrDefault().postCount).isEqualTo(1);
assertThat(value.getStatusOrDefault().visiblePostCount).isEqualTo(0);
}
private void reconcileStatusPostPilling(String reconcileCategoryName) {
Reconciler.Result result =
categoryReconciler.reconcile(new Reconciler.Request(reconcileCategoryName));
assertThat(result.reEnqueue()).isTrue();
assertThat(result.retryAfter()).isEqualTo(Duration.ofMinutes(1));
}
private List<Category> categories() {
/*
* |-A(post-4)
* |-B(post-3)
* |-|-C(post-2,post-1)
* |-D(post-1)
*/
Category categoryA = category("category-A");
categoryA.getSpec().setChildren(List.of("category-B", "category-D"));
Category categoryB = category("category-B");
categoryB.getSpec().setChildren(List.of("category-C"));
Category categoryC = category("category-C");
Category categoryD = category("category-D");
return List.of(categoryA, categoryB, categoryC, categoryD);
}
private Category category(String name) {
Category category = new Category();
Metadata metadata = new Metadata();
metadata.setName(name);
category.setMetadata(metadata);
category.getMetadata().setFinalizers(Set.of(CategoryReconciler.FINALIZER_NAME));
category.setSpec(new Category.CategorySpec());
category.setStatus(new Category.CategoryStatus());
category.getSpec().setDisplayName("display-name");
category.getSpec().setSlug("slug");
category.getSpec().setPriority(0);
return category;
}
private List<Post> posts() {
/*
* |-A(post-4)
* |-B(post-3)
* |-|-C(post-2,post-1)
* |-D(post-1)
*/
Post post1 = fakePost();
post1.getMetadata().setName("post-1");
post1.getSpec().setCategories(List.of("category-D", "category-C"));
post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post2 = fakePost();
post2.getMetadata().setName("post-2");
post2.getSpec().setCategories(List.of("category-C"));
post2.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post3 = fakePost();
post3.getMetadata().setName("post-3");
post3.getSpec().setCategories(List.of("category-B"));
post3.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post4 = fakePost();
post4.getMetadata().setName("post-4");
post4.getSpec().setCategories(List.of("category-A"));
post4.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
return List.of(post1, post2, post3, post4);
}
Post fakePost() {
var post = TestPost.postV1();
post.getSpec().setAllowComment(true);
post.getSpec().setDeleted(false);
post.getSpec().setExcerpt(new Post.Excerpt());
post.getSpec().getExcerpt().setAutoGenerate(false);
post.getSpec().setPinned(false);
post.getSpec().setPriority(0);
post.getSpec().setPublish(false);
post.getSpec().setSlug("fake-post");
return post;
}
}

View File

@ -1,160 +0,0 @@
package run.halo.app.core.extension.reconciler;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.time.Duration;
import java.util.List;
import java.util.Optional;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.domain.Sort;
import run.halo.app.content.TestPost;
import run.halo.app.content.permalinks.CategoryPermalinkPolicy;
import run.halo.app.core.extension.content.Category;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.controller.Reconciler;
/**
* Tests for {@link CategoryReconciler}.
*
* @author guqing
* @since 2.0.0
*/
@ExtendWith(MockitoExtension.class)
class CategoryReconcilerTest {
@Mock
private ExtensionClient client;
@Mock
private CategoryPermalinkPolicy categoryPermalinkPolicy;
@InjectMocks
private CategoryReconciler categoryReconciler;
@Test
void reconcileStatusPostForCategoryA() {
reconcileStatusPostPilling("category-A");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
assertThat(captor.getAllValues().get(1).getStatusOrDefault().getPostCount()).isEqualTo(4);
assertThat(
captor.getAllValues().get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryB() {
reconcileStatusPostPilling("category-B");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
Category category = captor.getAllValues().get(1);
assertThat(category.getStatusOrDefault().getPostCount()).isEqualTo(3);
assertThat(category.getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryC() {
reconcileStatusPostPilling("category-C");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
assertThat(captor.getAllValues().get(1).getStatusOrDefault().getPostCount()).isEqualTo(2);
assertThat(
captor.getAllValues().get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0);
}
@Test
void reconcileStatusPostForCategoryD() {
reconcileStatusPostPilling("category-D");
ArgumentCaptor<Category> captor = ArgumentCaptor.forClass(Category.class);
verify(client, times(3)).update(captor.capture());
assertThat(captor.getAllValues().get(1).getStatusOrDefault().postCount).isEqualTo(1);
assertThat(captor.getAllValues().get(1).getStatusOrDefault().visiblePostCount).isEqualTo(0);
}
private void reconcileStatusPostPilling(String reconcileCategoryName) {
categories().forEach(category -> {
lenient().when(client.fetch(eq(Category.class), eq(category.getMetadata().getName())))
.thenReturn(Optional.of(category));
});
lenient().when(client.listAll(eq(Post.class), any(ListOptions.class), any(Sort.class)))
.thenReturn(posts());
lenient().when(client.listAll(eq(Category.class), any(), any()))
.thenReturn(categories());
Reconciler.Result result =
categoryReconciler.reconcile(new Reconciler.Request(reconcileCategoryName));
assertThat(result.reEnqueue()).isTrue();
assertThat(result.retryAfter()).isEqualTo(Duration.ofMinutes(1));
}
private List<Category> categories() {
/*
* |-A(post-4)
* |-B(post-3)
* |-|-C(post-2,post-1)
* |-D(post-1)
*/
Category categoryA = category("category-A");
categoryA.getSpec().setChildren(List.of("category-B", "category-D"));
Category categoryB = category("category-B");
categoryB.getSpec().setChildren(List.of("category-C"));
Category categoryC = category("category-C");
Category categoryD = category("category-D");
return List.of(categoryA, categoryB, categoryC, categoryD);
}
private Category category(String name) {
Category category = new Category();
Metadata metadata = new Metadata();
metadata.setName(name);
category.setMetadata(metadata);
category.setSpec(new Category.CategorySpec());
category.setStatus(new Category.CategoryStatus());
return category;
}
private List<Post> posts() {
/*
* |-A(post-4)
* |-B(post-3)
* |-|-C(post-2,post-1)
* |-D(post-1)
*/
Post post1 = TestPost.postV1();
post1.getMetadata().setName("post-1");
post1.getSpec().setCategories(List.of("category-D", "category-C"));
post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post2 = TestPost.postV1();
post2.getMetadata().setName("post-2");
post2.getSpec().setCategories(List.of("category-C"));
post2.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post3 = TestPost.postV1();
post3.getMetadata().setName("post-3");
post3.getSpec().setCategories(List.of("category-B"));
post3.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
Post post4 = TestPost.postV1();
post4.getMetadata().setName("post-4");
post4.getSpec().setCategories(List.of("category-A"));
post4.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
return List.of(post1, post2, post3, post4);
}
}