From 3a50fdc4e58e32c4c66c02665e21230845e2c373 Mon Sep 17 00:00:00 2001 From: John Niang Date: Thu, 7 Aug 2025 00:08:37 +0800 Subject: [PATCH] Optimize homepage post loading by eliminating N+1 queries for user data (#7668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind improvement /area core /milestone 2.21.x #### What this PR does / why we need it: This PR refactors posts query to reduce database queries significantly. #### Which issue(s) this PR fixes: Fixes https://github.com/halo-dev/halo/issues/7593 Supersedes https://github.com/halo-dev/halo/pull/7644 #### Does this PR introduce a user-facing change? ```release-note 优化首页、归档页加载速度 ``` --- .../app/core/user/service/UserService.java | 5 + .../halo/app/core/counter/CounterService.java | 4 + .../app/core/counter/CounterServiceImpl.java | 17 +++ .../user/service/impl/UserServiceImpl.java | 17 ++- .../app/theme/finders/CategoryFinder.java | 4 +- .../app/theme/finders/ContributorFinder.java | 5 +- .../theme/finders/PostPublicQueryService.java | 3 + .../run/halo/app/theme/finders/TagFinder.java | 3 +- .../finders/impl/CategoryFinderImpl.java | 14 ++- .../finders/impl/ContributorFinderImpl.java | 13 +- .../theme/finders/impl/PostFinderImpl.java | 4 +- .../impl/PostPublicQueryServiceImpl.java | 115 ++++++++++++++++-- .../app/theme/finders/impl/TagFinderImpl.java | 15 ++- .../app/theme/finders/vo/ContributorVo.java | 20 +++ 14 files changed, 204 insertions(+), 35 deletions(-) diff --git a/api/src/main/java/run/halo/app/core/user/service/UserService.java b/api/src/main/java/run/halo/app/core/user/service/UserService.java index d1061b8ba..ce1c049b3 100644 --- a/api/src/main/java/run/halo/app/core/user/service/UserService.java +++ b/api/src/main/java/run/halo/app/core/user/service/UserService.java @@ -1,5 +1,6 @@ package run.halo.app.core.user.service; +import java.util.Collection; import java.util.Set; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -7,10 +8,14 @@ import run.halo.app.core.extension.User; public interface UserService { + String GHOST_USER_NAME = "ghost"; + Mono getUser(String username); Mono getUserOrGhost(String username); + Flux getUsersOrGhosts(Collection names); + Mono updatePassword(String username, String newPassword); Mono updateWithRawPassword(String username, String rawPassword); diff --git a/application/src/main/java/run/halo/app/core/counter/CounterService.java b/application/src/main/java/run/halo/app/core/counter/CounterService.java index f3940c796..7b746216e 100644 --- a/application/src/main/java/run/halo/app/core/counter/CounterService.java +++ b/application/src/main/java/run/halo/app/core/counter/CounterService.java @@ -1,5 +1,7 @@ package run.halo.app.core.counter; +import java.util.Collection; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.Counter; @@ -11,5 +13,7 @@ public interface CounterService { Mono getByName(String counterName); + Flux getByNames(Collection names); + Mono deleteByName(String counterName); } diff --git a/application/src/main/java/run/halo/app/core/counter/CounterServiceImpl.java b/application/src/main/java/run/halo/app/core/counter/CounterServiceImpl.java index d1be85fc7..33aa1c29e 100644 --- a/application/src/main/java/run/halo/app/core/counter/CounterServiceImpl.java +++ b/application/src/main/java/run/halo/app/core/counter/CounterServiceImpl.java @@ -1,9 +1,15 @@ package run.halo.app.core.counter; +import java.util.Collection; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.Counter; +import run.halo.app.extension.ExtensionUtil; +import run.halo.app.extension.ListOptions; import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.extension.index.query.QueryFactory; /** * Counter service implementation. @@ -25,6 +31,17 @@ public class CounterServiceImpl implements CounterService { return client.fetch(Counter.class, counterName); } + @Override + public Flux getByNames(Collection names) { + if (CollectionUtils.isEmpty(names)) { + return Flux.empty(); + } + var options = ListOptions.builder() + .andQuery(QueryFactory.in("metadata.name", names)) + .build(); + return client.listAll(Counter.class, options, ExtensionUtil.defaultSort()); + } + @Override public Mono deleteByName(String counterName) { return client.fetch(Counter.class, counterName) diff --git a/application/src/main/java/run/halo/app/core/user/service/impl/UserServiceImpl.java b/application/src/main/java/run/halo/app/core/user/service/impl/UserServiceImpl.java index bc05a016d..905713a8c 100644 --- a/application/src/main/java/run/halo/app/core/user/service/impl/UserServiceImpl.java +++ b/application/src/main/java/run/halo/app/core/user/service/impl/UserServiceImpl.java @@ -5,6 +5,7 @@ import static run.halo.app.extension.index.query.QueryFactory.equal; import java.time.Clock; import java.time.Duration; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Objects; @@ -38,6 +39,7 @@ import run.halo.app.extension.ListOptions; import run.halo.app.extension.Metadata; import run.halo.app.extension.ReactiveExtensionClient; import run.halo.app.extension.exception.ExtensionNotFoundException; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.infra.SystemConfigurableEnvironmentFetcher; import run.halo.app.infra.SystemSetting; @@ -54,8 +56,6 @@ import run.halo.app.security.device.DeviceService; @RequiredArgsConstructor public class UserServiceImpl implements UserService { - public static final String GHOST_USER_NAME = "ghost"; - private final ReactiveExtensionClient client; private final PasswordEncoder passwordEncoder; @@ -92,6 +92,19 @@ public class UserServiceImpl implements UserService { .switchIfEmpty(Mono.defer(() -> client.get(User.class, GHOST_USER_NAME))); } + @Override + public Flux getUsersOrGhosts(Collection names) { + if (CollectionUtils.isEmpty(names)) { + return Flux.empty(); + } + var nameSet = new HashSet<>(names); + nameSet.add(GHOST_USER_NAME); + var options = ListOptions.builder() + .andQuery(QueryFactory.in("metadata.name", nameSet)) + .build(); + return client.listAll(User.class, options, defaultSort()); + } + @Override public Mono updatePassword(String username, String newPassword) { return getUser(username) diff --git a/application/src/main/java/run/halo/app/theme/finders/CategoryFinder.java b/application/src/main/java/run/halo/app/theme/finders/CategoryFinder.java index 80bd684bf..029225cc8 100644 --- a/application/src/main/java/run/halo/app/theme/finders/CategoryFinder.java +++ b/application/src/main/java/run/halo/app/theme/finders/CategoryFinder.java @@ -1,6 +1,6 @@ package run.halo.app.theme.finders; -import java.util.List; +import java.util.Collection; import org.springframework.lang.Nullable; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -19,7 +19,7 @@ public interface CategoryFinder { Mono getByName(String name); - Flux getByNames(List names); + Flux getByNames(Collection names); Mono> list(@Nullable Integer page, @Nullable Integer size); diff --git a/application/src/main/java/run/halo/app/theme/finders/ContributorFinder.java b/application/src/main/java/run/halo/app/theme/finders/ContributorFinder.java index 904bf57eb..c6b4fea35 100644 --- a/application/src/main/java/run/halo/app/theme/finders/ContributorFinder.java +++ b/application/src/main/java/run/halo/app/theme/finders/ContributorFinder.java @@ -1,6 +1,6 @@ package run.halo.app.theme.finders; -import java.util.List; +import java.util.Collection; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.User; @@ -13,5 +13,6 @@ public interface ContributorFinder { Mono getContributor(String name); - Flux getContributors(List names); + Flux getContributors(Collection names); + } diff --git a/application/src/main/java/run/halo/app/theme/finders/PostPublicQueryService.java b/application/src/main/java/run/halo/app/theme/finders/PostPublicQueryService.java index e1ef8e680..c41731d7a 100644 --- a/application/src/main/java/run/halo/app/theme/finders/PostPublicQueryService.java +++ b/application/src/main/java/run/halo/app/theme/finders/PostPublicQueryService.java @@ -1,5 +1,6 @@ package run.halo.app.theme.finders; +import java.util.List; import org.springframework.lang.NonNull; import reactor.core.publisher.Mono; import run.halo.app.core.extension.content.Post; @@ -30,6 +31,8 @@ public interface PostPublicQueryService { */ Mono convertToListedVo(@NonNull Post post); + Mono> convertToListedVos(List posts); + /** * Converts {@link Post} to post vo and populate post content by the given snapshot name. *

This method will get post content by {@code snapshotName} and try to find diff --git a/application/src/main/java/run/halo/app/theme/finders/TagFinder.java b/application/src/main/java/run/halo/app/theme/finders/TagFinder.java index b19b3916d..05f70fc5d 100644 --- a/application/src/main/java/run/halo/app/theme/finders/TagFinder.java +++ b/application/src/main/java/run/halo/app/theme/finders/TagFinder.java @@ -1,5 +1,6 @@ package run.halo.app.theme.finders; +import java.util.Collection; import java.util.List; import org.springframework.lang.Nullable; import reactor.core.publisher.Flux; @@ -18,7 +19,7 @@ public interface TagFinder { Mono getByName(String name); - Flux getByNames(List names); + Flux getByNames(Collection names); Mono> list(@Nullable Integer page, @Nullable Integer size); diff --git a/application/src/main/java/run/halo/app/theme/finders/impl/CategoryFinderImpl.java b/application/src/main/java/run/halo/app/theme/finders/impl/CategoryFinderImpl.java index 2bc65c080..8c77d5975 100644 --- a/application/src/main/java/run/halo/app/theme/finders/impl/CategoryFinderImpl.java +++ b/application/src/main/java/run/halo/app/theme/finders/impl/CategoryFinderImpl.java @@ -19,15 +19,18 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Sort; import org.springframework.lang.Nullable; import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.content.CategoryService; import run.halo.app.core.extension.content.Category; +import run.halo.app.extension.ExtensionUtil; import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.Metadata; import run.halo.app.extension.PageRequestImpl; import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.theme.finders.CategoryFinder; import run.halo.app.theme.finders.Finder; @@ -54,12 +57,15 @@ public class CategoryFinderImpl implements CategoryFinder { } @Override - public Flux getByNames(List names) { - if (names == null) { + public Flux getByNames(Collection names) { + if (CollectionUtils.isEmpty(names)) { return Flux.empty(); } - return Flux.fromIterable(names) - .flatMap(this::getByName); + var options = ListOptions.builder() + .andQuery(QueryFactory.in("metadata.name", names)) + .build(); + return client.listAll(Category.class, options, ExtensionUtil.defaultSort()) + .map(CategoryVo::from); } static Sort defaultSort() { diff --git a/application/src/main/java/run/halo/app/theme/finders/impl/ContributorFinderImpl.java b/application/src/main/java/run/halo/app/theme/finders/impl/ContributorFinderImpl.java index d01abc4b3..31e576c3e 100644 --- a/application/src/main/java/run/halo/app/theme/finders/impl/ContributorFinderImpl.java +++ b/application/src/main/java/run/halo/app/theme/finders/impl/ContributorFinderImpl.java @@ -1,6 +1,6 @@ package run.halo.app.theme.finders.impl; -import java.util.List; +import java.util.Collection; import lombok.RequiredArgsConstructor; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -23,16 +23,11 @@ public class ContributorFinderImpl implements ContributorFinder { @Override public Mono getContributor(String name) { - return userService.getUserOrGhost(name) - .map(ContributorVo::from); + return userService.getUserOrGhost(name).map(ContributorVo::from); } @Override - public Flux getContributors(List names) { - if (names == null) { - return Flux.empty(); - } - return Flux.fromIterable(names) - .flatMapSequential(this::getContributor); + public Flux getContributors(Collection names) { + return userService.getUsersOrGhosts(names).map(ContributorVo::from); } } diff --git a/application/src/main/java/run/halo/app/theme/finders/impl/PostFinderImpl.java b/application/src/main/java/run/halo/app/theme/finders/impl/PostFinderImpl.java index a18d0fe7d..28ab52671 100644 --- a/application/src/main/java/run/halo/app/theme/finders/impl/PostFinderImpl.java +++ b/application/src/main/java/run/halo/app/theme/finders/impl/PostFinderImpl.java @@ -291,7 +291,9 @@ public class PostFinderImpl implements PostFinder { public Flux listAll() { return postPredicateResolver.getListOptions() .flatMapMany(listOptions -> client.listAll(Post.class, listOptions, defaultSort())) - .flatMapSequential(postPublicQueryService::convertToListedVo); + .collectList() + .flatMap(postPublicQueryService::convertToListedVos) + .flatMapMany(Flux::fromIterable); } static int pageNullSafe(Integer page) { diff --git a/application/src/main/java/run/halo/app/theme/finders/impl/PostPublicQueryServiceImpl.java b/application/src/main/java/run/halo/app/theme/finders/impl/PostPublicQueryServiceImpl.java index d25320cb8..7f04a6493 100644 --- a/application/src/main/java/run/halo/app/theme/finders/impl/PostPublicQueryServiceImpl.java +++ b/application/src/main/java/run/halo/app/theme/finders/impl/PostPublicQueryServiceImpl.java @@ -1,18 +1,23 @@ package run.halo.app.theme.finders.impl; +import static java.util.Objects.requireNonNullElse; +import static java.util.Objects.requireNonNullElseGet; +import static run.halo.app.core.counter.MeterUtils.nameOf; +import static run.halo.app.core.user.service.UserService.GHOST_USER_NAME; + +import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.function.Function; import lombok.RequiredArgsConstructor; import org.springframework.lang.NonNull; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; -import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.content.ContentWrapper; import run.halo.app.content.PostService; import run.halo.app.core.counter.CounterService; -import run.halo.app.core.counter.MeterUtils; import run.halo.app.core.extension.content.Post; import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; @@ -25,6 +30,7 @@ import run.halo.app.theme.finders.ContributorFinder; import run.halo.app.theme.finders.PostPublicQueryService; import run.halo.app.theme.finders.TagFinder; import run.halo.app.theme.finders.vo.ContentVo; +import run.halo.app.theme.finders.vo.ContributorVo; import run.halo.app.theme.finders.vo.ListedPostVo; import run.halo.app.theme.finders.vo.PostVo; import run.halo.app.theme.finders.vo.StatsVo; @@ -66,16 +72,12 @@ public class PostPublicQueryServiceImpl implements PostPublicQueryService { return option; }) .flatMap(listOptions -> client.listBy(Post.class, listOptions, page)) - .flatMap(list -> Flux.fromStream(list.get()) - .flatMapSequential(post -> convertToListedVo(post) - .flatMap(postVo -> populateStats(postVo) - .doOnNext(postVo::setStats).thenReturn(postVo) + .flatMap(list -> convertToListedVos(list.getItems()) + .map( + postVos -> new ListResult<>( + list.getPage(), list.getSize(), list.getTotal(), postVos ) ) - .collectList() - .map(postVos -> new ListResult<>(list.getPage(), list.getSize(), list.getTotal(), - postVos) - ) ) .defaultIfEmpty(ListResult.emptyResult()); } @@ -128,6 +130,97 @@ public class PostPublicQueryServiceImpl implements PostPublicQueryService { .defaultIfEmpty(postVo); } + @Override + public Mono> convertToListedVos(List posts) { + var counterNames = new HashSet(posts.size()); + var userNames = new HashSet(); + var tagNames = new HashSet(); + var categoryNames = new HashSet(); + posts.forEach(post -> { + counterNames.add(nameOf(Post.class, post.getMetadata().getName())); + var spec = post.getSpec(); + userNames.add(spec.getOwner()); + var status = post.getStatus(); + if (status != null && status.getContributors() != null) { + userNames.addAll(status.getContributors()); + } + if (spec.getTags() != null) { + tagNames.addAll(spec.getTags()); + } + if (spec.getCategories() != null) { + categoryNames.addAll(spec.getCategories()); + } + }); + + var getCounters = counterService.getByNames(counterNames) + .collectMap(counter -> counter.getMetadata().getName()); + var getContributors = contributorFinder.getContributors(userNames) + .collectMap(ContributorVo::getName); + var getTags = tagFinder.getByNames(tagNames) + .collectMap(tagVo -> tagVo.getMetadata().getName()); + var getCategories = categoryFinder.getByNames(categoryNames) + .collectMap(categoryVo -> categoryVo.getMetadata().getName()); + + return Mono.zip(getCounters, getContributors, getTags, getCategories) + .map(tuple -> { + var counters = tuple.getT1(); + var contributors = tuple.getT2(); + var tags = tuple.getT3(); + var categories = tuple.getT4(); + return posts.stream() + .map(post -> { + var vo = ListedPostVo.from(post); + vo.setCategories(List.of()); + vo.setTags(List.of()); + vo.setContributors(List.of()); + + var spec = post.getSpec(); + var status = post.getStatus(); + var ghost = requireNonNullElseGet( + contributors.get(GHOST_USER_NAME), ContributorVo::ghost + ); + vo.setOwner(requireNonNullElse(contributors.get(spec.getOwner()), ghost)); + if (status != null && !CollectionUtils.isEmpty(status.getContributors())) { + vo.setContributors(status.getContributors() + .stream() + .map(name -> requireNonNullElse(contributors.get(name), ghost)) + .toList()); + } + + if (!CollectionUtils.isEmpty(spec.getTags())) { + vo.setTags(spec.getTags() + .stream() + .map(tags::get) + .filter(Objects::nonNull) + .toList()); + } + if (!CollectionUtils.isEmpty(spec.getCategories())) { + vo.setCategories(spec.getCategories() + .stream() + .map(categories::get) + .filter(Objects::nonNull) + .toList()); + } + + var counterName = nameOf(Post.class, post.getMetadata().getName()); + var counter = counters.get(counterName); + if (counter != null) { + vo.setStats(StatsVo.builder() + .visit(counter.getVisit()) + .upvote(counter.getUpvote()) + .comment(counter.getApprovedComment()) + .build() + ); + } else { + vo.setStats(StatsVo.empty()); + } + + return vo; + }) + .toList(); + }); + } + @Override public Mono convertToVo(Post post, String snapshotName) { final String baseSnapshotName = post.getSpec().getBaseSnapshot(); @@ -177,7 +270,7 @@ public class PostPublicQueryServiceImpl implements PostPublicQueryService { } private Mono populateStats(T postVo) { - return counterService.getByName(MeterUtils.nameOf(Post.class, postVo.getMetadata() + return counterService.getByName(nameOf(Post.class, postVo.getMetadata() .getName()) ) .map(counter -> StatsVo.builder() diff --git a/application/src/main/java/run/halo/app/theme/finders/impl/TagFinderImpl.java b/application/src/main/java/run/halo/app/theme/finders/impl/TagFinderImpl.java index 9a625bc98..c7956f998 100644 --- a/application/src/main/java/run/halo/app/theme/finders/impl/TagFinderImpl.java +++ b/application/src/main/java/run/halo/app/theme/finders/impl/TagFinderImpl.java @@ -1,5 +1,6 @@ package run.halo.app.theme.finders.impl; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; @@ -9,11 +10,13 @@ import org.springframework.util.CollectionUtils; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import run.halo.app.core.extension.content.Tag; +import run.halo.app.extension.ExtensionUtil; import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.PageRequest; import run.halo.app.extension.PageRequestImpl; import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.theme.finders.Finder; import run.halo.app.theme.finders.TagFinder; import run.halo.app.theme.finders.vo.TagVo; @@ -43,9 +46,15 @@ public class TagFinderImpl implements TagFinder { } @Override - public Flux getByNames(List names) { - return Flux.fromIterable(names) - .flatMapSequential(this::getByName); + public Flux getByNames(Collection names) { + if (CollectionUtils.isEmpty(names)) { + return Flux.empty(); + } + var options = ListOptions.builder() + .andQuery(QueryFactory.in("metadata.name", names)) + .build(); + return client.listAll(Tag.class, options, ExtensionUtil.defaultSort()) + .map(TagVo::from); } @Override diff --git a/application/src/main/java/run/halo/app/theme/finders/vo/ContributorVo.java b/application/src/main/java/run/halo/app/theme/finders/vo/ContributorVo.java index 06396bc75..1688d03ba 100644 --- a/application/src/main/java/run/halo/app/theme/finders/vo/ContributorVo.java +++ b/application/src/main/java/run/halo/app/theme/finders/vo/ContributorVo.java @@ -4,6 +4,8 @@ import lombok.Builder; import lombok.ToString; import lombok.Value; import run.halo.app.core.extension.User; +import run.halo.app.core.user.service.UserService; +import run.halo.app.extension.Metadata; import run.halo.app.extension.MetadataOperator; /** @@ -46,4 +48,22 @@ public class ContributorVo implements ExtensionVoOperator { .metadata(user.getMetadata()) .build(); } + + /** + * Create a ghost contributor. + * + * @return a ghost contributor value object + */ + public static ContributorVo ghost() { + var metadata = new Metadata(); + metadata.setName(UserService.GHOST_USER_NAME); + return builder() + .name("ghost") + .displayName("Ghost") + // .avatar("/images/ghost.png") + .bio("A ghost user.") + .permalink("/authors/ghost") + .metadata(metadata) + .build(); + } }