mirror of https://github.com/halo-dev/halo
feat: add indexer implementation to solve the slow tag stats (#3420)
#### What type of PR is this? /kind improvement /area core /milestone 2.3.x #### What this PR does / why we need it: 修复标签关联文章数量统计不正确的问题 1. 实现简要的 Indexer 可以通过 IndexFunc 来提取需要构建 indacates 的信息,比如对文章的标签和 labels 建立 indacates 2. Indexer 通过 Watch 文章数据来维护 Indicates 使其与数据库一致 3. TagReconciler 一分钟 requeue 一次,但直接通过 Indexer 获取文章名称来统计数量,无需 list 文章数据 how to test it? 1. 创建文章并绑定标签 2. 测试文章关联标签,关联多个标签、解除旧标签绑定等操作是否会在一分钟后在标签处正确显示文章数量 4. 查看主题端标签下文章数量统计是否正确 5. 重复上述操作多次 #### Which issue(s) this PR fixes: Fixes #3311, #3312 #### Does this PR introduce a user-facing change? ```release-note 修复标签关联文章数量统计不正确的问题,优化标签数量多时耗时长的问题 ```pull/3434/head
parent
aba151f54c
commit
af07b42b42
|
@ -0,0 +1,129 @@
|
|||
package run.halo.app.content;
|
||||
|
||||
import com.google.common.collect.HashMultimap;
|
||||
import com.google.common.collect.SetMultimap;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.util.Assert;
|
||||
import run.halo.app.extension.Extension;
|
||||
|
||||
/**
|
||||
* <p>A default implementation of {@link Indexer}.</p>
|
||||
* <p>Note that this Indexer is not thread-safe, If multiple threads access this indexer
|
||||
* concurrently and one of the threads modifies the indexer, it must be synchronized externally.</p>
|
||||
*
|
||||
* @param <T> the type of object to be indexed
|
||||
* @author guqing
|
||||
* @see
|
||||
* <a href="https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/tools/cache/index.go">kubernetes index</a>
|
||||
* @see <a href="https://juejin.cn/post/7132767272841510926">informer机制之cache.indexer机制</a>
|
||||
* @since 2.0.0
|
||||
*/
|
||||
public class DefaultIndexer<T extends Extension> implements Indexer<T> {
|
||||
private final Map<String, SetMultimap<String, String>> indices = new HashMap<>();
|
||||
private final Map<String, IndexFunc<T>> indexFuncMap = new HashMap<>();
|
||||
private final Map<String, SetMultimap<String, String>> indexValues = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public void addIndexFunc(String indexName, IndexFunc<T> indexFunc) {
|
||||
indexFuncMap.put(indexName, indexFunc);
|
||||
indices.put(indexName, HashMultimap.create());
|
||||
indexValues.put(indexName, HashMultimap.create());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> indexNames() {
|
||||
return Set.copyOf(indexFuncMap.keySet());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(String indexName, T obj) {
|
||||
IndexFunc<T> indexFunc = getIndexFunc(indexName);
|
||||
Set<String> indexKeys = indexFunc.apply(obj);
|
||||
for (String indexKey : indexKeys) {
|
||||
SetMultimap<String, String> index = indices.get(indexName);
|
||||
index.put(indexKey, getObjectKey(obj));
|
||||
|
||||
SetMultimap<String, String> indexValue = indexValues.get(indexName);
|
||||
indexValue.put(getObjectKey(obj), indexKey);
|
||||
}
|
||||
}
|
||||
|
||||
@NonNull
|
||||
private IndexFunc<T> getIndexFunc(String indexName) {
|
||||
IndexFunc<T> indexFunc = indexFuncMap.get(indexName);
|
||||
if (indexFunc == null) {
|
||||
throw new IllegalArgumentException(
|
||||
"Index function not found for index name: " + indexName);
|
||||
}
|
||||
return indexFunc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update(String indexName, T obj) {
|
||||
IndexFunc<T> indexFunc = getIndexFunc(indexName);
|
||||
Set<String> indexKeys = indexFunc.apply(obj);
|
||||
Set<String> oldIndexKeys = new HashSet<>();
|
||||
SetMultimap<String, String> indexValue = indexValues.get(indexName);
|
||||
if (indexValue.containsKey(getObjectKey(obj))) {
|
||||
oldIndexKeys.addAll(indexValue.get(getObjectKey(obj)));
|
||||
}
|
||||
// delete old index first
|
||||
for (String oldIndexKey : oldIndexKeys) {
|
||||
SetMultimap<String, String> index = indices.get(indexName);
|
||||
index.remove(oldIndexKey, getObjectKey(obj));
|
||||
indexValue.remove(getObjectKey(obj), oldIndexKey);
|
||||
}
|
||||
// add new index
|
||||
for (String indexKey : indexKeys) {
|
||||
SetMultimap<String, String> index = indices.get(indexName);
|
||||
index.put(indexKey, getObjectKey(obj));
|
||||
|
||||
indexValue.put(getObjectKey(obj), indexKey);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getByIndex(String indexName, String indexKey) {
|
||||
SetMultimap<String, String> index = indices.get(indexName);
|
||||
if (index != null) {
|
||||
return Set.copyOf(index.get(indexKey));
|
||||
}
|
||||
return Set.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String indexName, T obj) {
|
||||
IndexFunc<T> indexFunc = getIndexFunc(indexName);
|
||||
SetMultimap<String, String> indexValue = indexValues.get(indexName);
|
||||
Set<String> indexKeys = indexFunc.apply(obj);
|
||||
for (String indexKey : indexKeys) {
|
||||
String objectKey = getObjectKey(obj);
|
||||
SetMultimap<String, String> index = indices.get(indexName);
|
||||
index.remove(indexKey, objectKey);
|
||||
|
||||
indexValue.remove(indexKey, objectKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This method is only used for testing.
|
||||
*
|
||||
* @param indexName index name
|
||||
* @return all indices of the given index name
|
||||
*/
|
||||
public Map<String, Collection<String>> getIndices(String indexName) {
|
||||
return indices.get(indexName).asMap();
|
||||
}
|
||||
|
||||
private String getObjectKey(T obj) {
|
||||
Assert.notNull(obj, "Object must not be null");
|
||||
Assert.notNull(obj.getMetadata(), "Object metadata must not be null");
|
||||
Assert.notNull(obj.getMetadata().getName(), "Object name must not be null");
|
||||
return obj.getMetadata().getName();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,79 @@
|
|||
package run.halo.app.content;
|
||||
|
||||
import java.util.Set;
|
||||
import run.halo.app.extension.Extension;
|
||||
|
||||
/**
|
||||
* <p>Indexer is used to index objects by index name and index key.</p>
|
||||
* <p>For example, if you want to index posts by category, you can use the following code:</p>
|
||||
* <pre>
|
||||
* Indexer<Post> indexer = new Indexer<>();
|
||||
* indexer.addIndexFunc("category", post -> {
|
||||
* List<String> tags = post.getSpec().getTags();
|
||||
* return tags == null ? Set.of() : Set.copyOf(tags);
|
||||
* });
|
||||
* indexer.add("category", post);
|
||||
* indexer.getByIndex("category", "category-slug");
|
||||
* indexer.update("category", post);
|
||||
* indexer.delete("category", post);
|
||||
* </pre>
|
||||
*
|
||||
* @param <T> the type of object to be indexed
|
||||
* @author guqing
|
||||
* @since 2.0.0
|
||||
*/
|
||||
public interface Indexer<T extends Extension> {
|
||||
|
||||
/**
|
||||
* Adds an index function for a given index name.
|
||||
*
|
||||
* @param indexName The name of the index.
|
||||
* @param indexFunc The function to use for indexing.
|
||||
*/
|
||||
void addIndexFunc(String indexName, DefaultIndexer.IndexFunc<T> indexFunc);
|
||||
|
||||
Set<String> indexNames();
|
||||
|
||||
/**
|
||||
* The {@code add} method adds an object of type T to the index
|
||||
* with the given name. It does this by first getting the index function for the given index
|
||||
* name and applying it to the object to get a set of index keys. For each index key, it adds
|
||||
* the object key to the index and the index key to the object's index values.
|
||||
*
|
||||
* <p>For example, if you want to index Person objects by name and age, you can use the
|
||||
* following:</p>
|
||||
* <pre>
|
||||
* // Create an Indexer that indexes Person objects by name and age
|
||||
* Indexer<Person> indexer = new Indexer<>();
|
||||
* indexer.addIndexFunc("name", person -> Collections.singleton(person.getName()));
|
||||
* indexer.addIndexFunc("age", person -> Collections.singleton(String.valueOf(person
|
||||
* .getAge())));
|
||||
*
|
||||
* // Create some Person objects
|
||||
* Person alice = new Person("Alice", 25);
|
||||
* Person bob = new Person("Bob", 30);
|
||||
*
|
||||
* // Add the Person objects to the index
|
||||
* indexer.add("name", alice);
|
||||
* indexer.add("name", bob);
|
||||
* indexer.add("age", alice);
|
||||
* indexer.add("age", bob);
|
||||
* </pre>
|
||||
*
|
||||
* @param indexName The name of the index.
|
||||
* @param obj The function to use for indexing.
|
||||
* @throws IllegalArgumentException if the index name is not found.
|
||||
*/
|
||||
void add(String indexName, T obj);
|
||||
|
||||
void update(String indexName, T obj);
|
||||
|
||||
Set<String> getByIndex(String indexName, String indexKey);
|
||||
|
||||
void delete(String indexName, T obj);
|
||||
|
||||
@FunctionalInterface
|
||||
interface IndexFunc<T> {
|
||||
Set<String> apply(T obj);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,226 @@
|
|||
package run.halo.app.content;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.locks.StampedLock;
|
||||
import java.util.function.BiConsumer;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
import run.halo.app.core.extension.content.Post;
|
||||
import run.halo.app.extension.Extension;
|
||||
import run.halo.app.extension.ExtensionClient;
|
||||
import run.halo.app.extension.ExtensionUtil;
|
||||
import run.halo.app.extension.GroupVersionKind;
|
||||
import run.halo.app.extension.Metadata;
|
||||
import run.halo.app.extension.Unstructured;
|
||||
import run.halo.app.extension.Watcher;
|
||||
import run.halo.app.extension.controller.RequestSynchronizer;
|
||||
import run.halo.app.infra.SchemeInitializedEvent;
|
||||
|
||||
/**
|
||||
* <p>Monitor changes to {@link Post} resources and establish a local, in-memory cache in an
|
||||
* Indexer.
|
||||
* When changes to posts are detected, the Indexer is updated using the indexFunc to maintain
|
||||
* its integrity.
|
||||
* This enables quick retrieval of the unique identifier(It is usually {@link Metadata#getName()})
|
||||
* for article objects using the getByIndex method when needed.</p>
|
||||
*
|
||||
* @author guqing
|
||||
* @since 2.0.0
|
||||
*/
|
||||
@Component
|
||||
public class PostIndexInformer implements ApplicationListener<SchemeInitializedEvent>,
|
||||
DisposableBean {
|
||||
public static final String TAG_POST_INDEXER = "tag-post-indexer";
|
||||
public static final String LABEL_INDEXER_NAME = "post-label-indexer";
|
||||
|
||||
private final RequestSynchronizer synchronizer;
|
||||
|
||||
private final Indexer<Post> postIndexer;
|
||||
|
||||
private final PostWatcher postWatcher;
|
||||
|
||||
public PostIndexInformer(ExtensionClient client) {
|
||||
postIndexer = new DefaultIndexer<>();
|
||||
postIndexer.addIndexFunc(TAG_POST_INDEXER, post -> {
|
||||
List<String> tags = post.getSpec().getTags();
|
||||
return tags != null ? Set.copyOf(tags) : Set.of();
|
||||
});
|
||||
postIndexer.addIndexFunc(LABEL_INDEXER_NAME, labelIndexFunc());
|
||||
|
||||
this.postWatcher = new PostWatcher();
|
||||
this.synchronizer = new RequestSynchronizer(true,
|
||||
client,
|
||||
new Post(),
|
||||
postWatcher,
|
||||
this::checkExtension);
|
||||
}
|
||||
|
||||
private DefaultIndexer.IndexFunc<Post> labelIndexFunc() {
|
||||
return post -> {
|
||||
Map<String, String> labels = ExtensionUtil.nullSafeLabels(post);
|
||||
Set<String> indexKeys = new HashSet<>();
|
||||
for (Map.Entry<String, String> entry : labels.entrySet()) {
|
||||
indexKeys.add(labelKey(entry.getKey(), entry.getValue()));
|
||||
}
|
||||
return indexKeys;
|
||||
};
|
||||
}
|
||||
|
||||
public Set<String> getByIndex(String indexName, String indexKey) {
|
||||
return postIndexer.getByIndex(indexName, indexKey);
|
||||
}
|
||||
|
||||
public Set<String> getByTagName(String tagName) {
|
||||
return postIndexer.getByIndex(TAG_POST_INDEXER, tagName);
|
||||
}
|
||||
|
||||
public Set<String> getByLabels(Map<String, String> labels) {
|
||||
if (labels == null) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> result = new HashSet<>();
|
||||
for (Map.Entry<String, String> entry : labels.entrySet()) {
|
||||
Set<String> values = postIndexer.getByIndex(LABEL_INDEXER_NAME,
|
||||
labelKey(entry.getKey(), entry.getValue()));
|
||||
if (values == null) {
|
||||
// No objects have this label, no need to continue searching
|
||||
return Set.of();
|
||||
}
|
||||
if (result.isEmpty()) {
|
||||
result.addAll(values);
|
||||
} else {
|
||||
result.retainAll(values);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
String labelKey(String labelName, String labelValue) {
|
||||
return labelName + "=" + labelValue;
|
||||
}
|
||||
|
||||
public Set<String> getByLabel(String labelName, String labelValue) {
|
||||
return postIndexer.getByIndex(LABEL_INDEXER_NAME, labelKey(labelName, labelValue));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (postWatcher != null) {
|
||||
postWatcher.dispose();
|
||||
}
|
||||
if (synchronizer != null) {
|
||||
synchronizer.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onApplicationEvent(@NonNull SchemeInitializedEvent event) {
|
||||
if (!synchronizer.isStarted()) {
|
||||
synchronizer.start();
|
||||
}
|
||||
}
|
||||
|
||||
class PostWatcher implements Watcher {
|
||||
private Runnable disposeHook;
|
||||
private boolean disposed = false;
|
||||
private final StampedLock lock = new StampedLock();
|
||||
|
||||
@Override
|
||||
public void onAdd(Extension extension) {
|
||||
if (!checkExtension(extension)) {
|
||||
return;
|
||||
}
|
||||
handleIndicates(extension, postIndexer::add);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onUpdate(Extension oldExt, Extension newExt) {
|
||||
if (!checkExtension(newExt)) {
|
||||
return;
|
||||
}
|
||||
handleIndicates(newExt, postIndexer::update);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDelete(Extension extension) {
|
||||
if (!checkExtension(extension)) {
|
||||
return;
|
||||
}
|
||||
handleIndicates(extension, postIndexer::delete);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerDisposeHook(Runnable dispose) {
|
||||
this.disposeHook = dispose;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dispose() {
|
||||
if (isDisposed()) {
|
||||
return;
|
||||
}
|
||||
this.disposed = true;
|
||||
if (this.disposeHook != null) {
|
||||
this.disposeHook.run();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDisposed() {
|
||||
return this.disposed;
|
||||
}
|
||||
|
||||
void handleIndicates(Extension extension, BiConsumer<String, Post> consumer) {
|
||||
Post post = convertTo(extension);
|
||||
Set<String> indexNames = getIndexNames();
|
||||
for (String indexName : indexNames) {
|
||||
maintainIndicates(indexName, post, consumer);
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> getIndexNames() {
|
||||
long stamp = lock.tryOptimisticRead();
|
||||
Set<String> indexNames = postIndexer.indexNames();
|
||||
if (!lock.validate(stamp)) {
|
||||
stamp = lock.readLock();
|
||||
try {
|
||||
return postIndexer.indexNames();
|
||||
} finally {
|
||||
lock.unlockRead(stamp);
|
||||
}
|
||||
}
|
||||
return indexNames;
|
||||
}
|
||||
|
||||
void maintainIndicates(String indexName, Post post, BiConsumer<String, Post> consumer) {
|
||||
long stamp = lock.writeLock();
|
||||
try {
|
||||
consumer.accept(indexName, post);
|
||||
} finally {
|
||||
lock.unlockWrite(stamp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Post convertTo(Extension extension) {
|
||||
if (extension instanceof Post) {
|
||||
return (Post) extension;
|
||||
}
|
||||
return Unstructured.OBJECT_MAPPER.convertValue(extension, Post.class);
|
||||
}
|
||||
|
||||
private boolean checkExtension(Extension extension) {
|
||||
return !postWatcher.isDisposed()
|
||||
&& extension.getMetadata().getDeletionTimestamp() == null
|
||||
&& isPost(extension);
|
||||
}
|
||||
|
||||
private boolean isPost(Extension extension) {
|
||||
return GroupVersionKind.fromExtension(Post.class).equals(extension.groupVersionKind());
|
||||
}
|
||||
}
|
|
@ -1,12 +1,14 @@
|
|||
package run.halo.app.core.extension.reconciler;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Component;
|
||||
import run.halo.app.content.PostIndexInformer;
|
||||
import run.halo.app.content.permalinks.TagPermalinkPolicy;
|
||||
import run.halo.app.core.extension.content.Constant;
|
||||
import run.halo.app.core.extension.content.Post;
|
||||
|
@ -25,23 +27,20 @@ import run.halo.app.infra.utils.JsonUtils;
|
|||
* @since 2.0.0
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class TagReconciler implements Reconciler<Reconciler.Request> {
|
||||
private static final String FINALIZER_NAME = "tag-protection";
|
||||
private final ExtensionClient client;
|
||||
private final TagPermalinkPolicy tagPermalinkPolicy;
|
||||
|
||||
public TagReconciler(ExtensionClient client, TagPermalinkPolicy tagPermalinkPolicy) {
|
||||
this.client = client;
|
||||
this.tagPermalinkPolicy = tagPermalinkPolicy;
|
||||
}
|
||||
private final PostIndexInformer postIndexInformer;
|
||||
|
||||
@Override
|
||||
public Result reconcile(Request request) {
|
||||
client.fetch(Tag.class, request.name())
|
||||
.ifPresent(tag -> {
|
||||
return client.fetch(Tag.class, request.name())
|
||||
.map(tag -> {
|
||||
if (isDeleted(tag)) {
|
||||
cleanUpResourcesAndRemoveFinalizer(request.name());
|
||||
return;
|
||||
return Result.doNotRetry();
|
||||
}
|
||||
addFinalizerIfNecessary(tag);
|
||||
|
||||
|
@ -50,14 +49,15 @@ public class TagReconciler implements Reconciler<Reconciler.Request> {
|
|||
this.reconcileStatusPermalink(request.name());
|
||||
|
||||
reconcileStatusPosts(request.name());
|
||||
});
|
||||
return new Result(false, null);
|
||||
return new Result(true, Duration.ofMinutes(1));
|
||||
})
|
||||
.orElse(Result.doNotRetry());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Controller setupWith(ControllerBuilder builder) {
|
||||
return builder
|
||||
.syncAllOnStart(false)
|
||||
.syncAllOnStart(true)
|
||||
.extension(new Tag())
|
||||
.build();
|
||||
}
|
||||
|
@ -128,29 +128,20 @@ public class TagReconciler implements Reconciler<Reconciler.Request> {
|
|||
}
|
||||
|
||||
private void populatePosts(Tag tag) {
|
||||
List<Post.CompactPost> compactPosts = client.list(Post.class, null, null)
|
||||
.stream()
|
||||
.filter(post -> includes(post.getSpec().getTags(), tag.getMetadata().getName()))
|
||||
.map(post -> Post.CompactPost.builder()
|
||||
.name(post.getMetadata().getName())
|
||||
.published(post.isPublished())
|
||||
.visible(post.getSpec().getVisible())
|
||||
.build())
|
||||
.toList();
|
||||
tag.getStatusOrDefault().setPostCount(compactPosts.size());
|
||||
// populate post count
|
||||
Set<String> postNames = postIndexInformer.getByTagName(tag.getMetadata().getName());
|
||||
tag.getStatusOrDefault().setPostCount(postNames.size());
|
||||
|
||||
long visiblePostCount = compactPosts.stream()
|
||||
.filter(post -> Objects.equals(true, post.getPublished())
|
||||
&& Post.VisibleEnum.PUBLIC.equals(post.getVisible()))
|
||||
.count();
|
||||
tag.getStatusOrDefault().setVisiblePostCount((int) visiblePostCount);
|
||||
}
|
||||
// populate visible post count
|
||||
Map<String, String> labelToQuery = Map.of(Post.PUBLISHED_LABEL, BooleanUtils.TRUE,
|
||||
Post.VISIBLE_LABEL, Post.VisibleEnum.PUBLIC.name(),
|
||||
Post.DELETED_LABEL, BooleanUtils.FALSE);
|
||||
Set<String> hasAllLabelPosts = postIndexInformer.getByLabels(labelToQuery);
|
||||
|
||||
private boolean includes(List<String> tags, String tagName) {
|
||||
if (tags == null) {
|
||||
return false;
|
||||
}
|
||||
return tags.contains(tagName);
|
||||
// retain all posts that has all labels
|
||||
Set<String> postNamesWithTag = new HashSet<>(postNames);
|
||||
postNamesWithTag.retainAll(hasAllLabelPosts);
|
||||
tag.getStatusOrDefault().setVisiblePostCount(postNamesWithTag.size());
|
||||
}
|
||||
|
||||
private boolean isDeleted(Tag tag) {
|
||||
|
|
|
@ -162,11 +162,8 @@ public class DefaultController<R> implements Controller {
|
|||
watch.start("reconciliation");
|
||||
result = reconciler.reconcile(entry.getEntry());
|
||||
watch.stop();
|
||||
log.debug("{} >>> Reconciled request: {} with result: {}", this.name,
|
||||
entry.getEntry(), result);
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace(watch.toString());
|
||||
}
|
||||
log.debug("{} >>> Reconciled request: {} with result: {}, usage: {}",
|
||||
this.name, entry.getEntry(), result, watch.getTotalTimeMillis());
|
||||
} catch (Throwable t) {
|
||||
if (t instanceof OptimisticLockingFailureException) {
|
||||
log.warn("Optimistic locking failure when reconciling request: {}/{}",
|
||||
|
|
|
@ -0,0 +1,368 @@
|
|||
package run.halo.app.content;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.json.JSONException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.skyscreamer.jsonassert.JSONAssert;
|
||||
import run.halo.app.core.extension.content.Post;
|
||||
import run.halo.app.extension.Metadata;
|
||||
import run.halo.app.infra.utils.JsonUtils;
|
||||
|
||||
/**
|
||||
* Tests for {@link DefaultIndexer}.
|
||||
*
|
||||
* @author guqing
|
||||
* @since 2.0.0
|
||||
*/
|
||||
class DefaultIndexerTest {
|
||||
@Test
|
||||
public void testTagsIndexer() throws JSONException {
|
||||
// Create a new Indexer that indexes Post objects by tags.
|
||||
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
|
||||
String tagsIndexName = "tags";
|
||||
indexer.addIndexFunc(tagsIndexName, post -> {
|
||||
List<String> tags = post.getSpec().getTags();
|
||||
return tags == null ? Set.of() : Set.copyOf(tags);
|
||||
});
|
||||
|
||||
// Create some Post objects.
|
||||
Post post1 = new Post();
|
||||
post1.setMetadata(new Metadata());
|
||||
post1.getMetadata().setName("post-1");
|
||||
post1.setSpec(new Post.PostSpec());
|
||||
post1.getSpec().setTags(List.of("t1", "t2"));
|
||||
|
||||
Post post2 = new Post();
|
||||
post2.setMetadata(new Metadata());
|
||||
post2.getMetadata().setName("post-2");
|
||||
post2.setSpec(new Post.PostSpec());
|
||||
post2.getSpec().setTags(List.of("t2", "t3"));
|
||||
|
||||
Post post3 = new Post();
|
||||
post3.setMetadata(new Metadata());
|
||||
post3.getMetadata().setName("post-3");
|
||||
post3.setSpec(new Post.PostSpec());
|
||||
post3.getSpec().setTags(List.of("t3", "t4"));
|
||||
|
||||
// Add the Post objects to the Indexer.
|
||||
indexer.add(tagsIndexName, post1);
|
||||
indexer.add(tagsIndexName, post2);
|
||||
indexer.add(tagsIndexName, post3);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"t4": [
|
||||
"post-3"
|
||||
],
|
||||
"t3": [
|
||||
"post-2",
|
||||
"post-3"
|
||||
],
|
||||
"t2": [
|
||||
"post-1",
|
||||
"post-2"
|
||||
],
|
||||
"t1": [
|
||||
"post-1"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("tags")),
|
||||
true);
|
||||
|
||||
// Remove post2 from the Indexer.
|
||||
indexer.delete(tagsIndexName, post2);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"t1": [
|
||||
"post-1"
|
||||
],
|
||||
"t2": [
|
||||
"post-1"
|
||||
],
|
||||
"t3": [
|
||||
"post-3"
|
||||
],
|
||||
"t4": [
|
||||
"post-3"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("tags")),
|
||||
true);
|
||||
|
||||
// Update post3 in the Indexer.
|
||||
post3.getSpec().setTags(List.of("t4", "t5"));
|
||||
indexer.update(tagsIndexName, post3);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"t1": [
|
||||
"post-1"
|
||||
],
|
||||
"t2": [
|
||||
"post-1"
|
||||
],
|
||||
"t4": [
|
||||
"post-3"
|
||||
],
|
||||
"t5": [
|
||||
"post-3"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("tags")),
|
||||
true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLabelIndexer() throws JSONException {
|
||||
// Create a new Indexer.
|
||||
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
|
||||
|
||||
// Define the IndexFunc for labels.
|
||||
DefaultIndexer.IndexFunc<Post> labelIndexFunc = labelIndexFunc();
|
||||
|
||||
// Add the label IndexFunc to the Indexer.
|
||||
String labelsIndexName = "labels";
|
||||
indexer.addIndexFunc(labelsIndexName, labelIndexFunc);
|
||||
|
||||
// Create some posts with labels.
|
||||
Post post1 = new Post();
|
||||
post1.setMetadata(new Metadata());
|
||||
post1.getMetadata().setName("post-1");
|
||||
post1.getMetadata().setLabels(Map.of("app", "myapp", "env", "prod"));
|
||||
|
||||
Post post2 = new Post();
|
||||
post2.setMetadata(new Metadata());
|
||||
post2.getMetadata().setName("post-2");
|
||||
post2.getMetadata().setLabels(Map.of("app", "myapp", "env", "test"));
|
||||
|
||||
Post post3 = new Post();
|
||||
post3.setMetadata(new Metadata());
|
||||
post3.getMetadata().setName("post-3");
|
||||
post3.getMetadata().setLabels(Map.of("app", "otherapp", "env", "prod"));
|
||||
|
||||
// Add the posts to the Indexer.
|
||||
indexer.add(labelsIndexName, post1);
|
||||
indexer.add(labelsIndexName, post2);
|
||||
indexer.add(labelsIndexName, post3);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
assertEquals(
|
||||
Map.of(
|
||||
"app=myapp", Set.of("post-1", "post-2"),
|
||||
"app=otherapp", Set.of("post-3"),
|
||||
"env=test", Set.of("post-2"),
|
||||
"env=prod", Set.of("post-1", "post-3")
|
||||
),
|
||||
indexer.getIndices("labels"));
|
||||
|
||||
// Delete post2 from the Indexer.
|
||||
indexer.delete(labelsIndexName, post2);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"app=myapp": [
|
||||
"post-1"
|
||||
],
|
||||
"env=prod": [
|
||||
"post-1",
|
||||
"post-3"
|
||||
],
|
||||
"app=otherapp": [
|
||||
"post-3"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("labels")),
|
||||
true);
|
||||
|
||||
// Update post2 in the Indexer.
|
||||
post2.getMetadata().setLabels(Map.of("l1", "v1", "l2", "v2"));
|
||||
indexer.update(labelsIndexName, post2);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"app=myapp": [
|
||||
"post-1"
|
||||
],
|
||||
"env=prod": [
|
||||
"post-1",
|
||||
"post-3"
|
||||
],
|
||||
"app=otherapp": [
|
||||
"post-3"
|
||||
],
|
||||
"l1=v1": [
|
||||
"post-2"
|
||||
],
|
||||
"l2=v2": [
|
||||
"post-2"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("labels")),
|
||||
true);
|
||||
|
||||
// Update post1 in the Indexer.
|
||||
post1.getMetadata().setLabels(Map.of("l2", "v2", "l3", "v3"));
|
||||
indexer.update(labelsIndexName, post1);
|
||||
|
||||
// Verify that the Indexer has the correct indices.
|
||||
JSONAssert.assertEquals("""
|
||||
{
|
||||
"env=prod": [
|
||||
"post-3"
|
||||
],
|
||||
"app=otherapp": [
|
||||
"post-3"
|
||||
],
|
||||
"l1=v1": [
|
||||
"post-2"
|
||||
],
|
||||
"l2=v2": [
|
||||
"post-1",
|
||||
"post-2"
|
||||
],
|
||||
"l3=v3": [
|
||||
"post-1"
|
||||
]
|
||||
}
|
||||
""",
|
||||
JsonUtils.objectToJson(indexer.getIndices("labels")),
|
||||
true);
|
||||
}
|
||||
|
||||
@Test
|
||||
void multiIndexName() {
|
||||
// Create a new Indexer.
|
||||
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
|
||||
|
||||
// Define the IndexFunc for labels.
|
||||
String labelsIndexName = "labels";
|
||||
DefaultIndexer.IndexFunc<Post> labelIndexFunc = labelIndexFunc();
|
||||
indexer.addIndexFunc(labelsIndexName, labelIndexFunc);
|
||||
|
||||
String tagsIndexName = "tags";
|
||||
indexer.addIndexFunc(tagsIndexName, post -> {
|
||||
List<String> tags = post.getSpec().getTags();
|
||||
return tags == null ? Set.of() : Set.copyOf(tags);
|
||||
});
|
||||
|
||||
Post post1 = new Post();
|
||||
post1.setMetadata(new Metadata());
|
||||
post1.getMetadata().setName("post-1");
|
||||
post1.getMetadata().setLabels(Map.of("app", "myapp", "env", "prod"));
|
||||
post1.setSpec(new Post.PostSpec());
|
||||
post1.getSpec().setTags(List.of("t1", "t2"));
|
||||
|
||||
Post post2 = new Post();
|
||||
post2.setMetadata(new Metadata());
|
||||
post2.getMetadata().setName("post-2");
|
||||
post2.getMetadata().setLabels(Map.of("app", "myapp", "env", "test"));
|
||||
post2.setSpec(new Post.PostSpec());
|
||||
post2.getSpec().setTags(List.of("t2", "t3"));
|
||||
|
||||
indexer.add(labelsIndexName, post1);
|
||||
indexer.add(tagsIndexName, post1);
|
||||
|
||||
indexer.add(labelsIndexName, post2);
|
||||
indexer.add(tagsIndexName, post2);
|
||||
|
||||
assertThat(indexer.getByIndex(labelsIndexName, "app=myapp"))
|
||||
.containsExactlyInAnyOrder("post-1", "post-2");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
|
||||
.containsExactlyInAnyOrder("post-1");
|
||||
|
||||
assertThat(indexer.getByIndex(labelsIndexName, "env=test"))
|
||||
.containsExactlyInAnyOrder("post-2");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
|
||||
.containsExactlyInAnyOrder("post-1", "post-2");
|
||||
|
||||
post2.getSpec().setTags(List.of("t1", "t4"));
|
||||
indexer.update(tagsIndexName, post2);
|
||||
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
|
||||
.containsExactlyInAnyOrder("post-1", "post-2");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
|
||||
.containsExactlyInAnyOrder("post-1");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t4"))
|
||||
.containsExactlyInAnyOrder("post-2");
|
||||
}
|
||||
|
||||
private static DefaultIndexer.IndexFunc<Post> labelIndexFunc() {
|
||||
return post -> {
|
||||
Map<String, String> labels = post.getMetadata().getLabels();
|
||||
Set<String> indexKeys = new HashSet<>();
|
||||
if (labels != null) {
|
||||
for (Map.Entry<String, String> entry : labels.entrySet()) {
|
||||
indexKeys.add(entry.getKey() + "=" + entry.getValue());
|
||||
}
|
||||
}
|
||||
return indexKeys;
|
||||
};
|
||||
}
|
||||
|
||||
@Test
|
||||
void getByIndex() {
|
||||
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
|
||||
String tagsIndexName = "tags";
|
||||
indexer.addIndexFunc(tagsIndexName, post -> {
|
||||
List<String> tags = post.getSpec().getTags();
|
||||
return tags == null ? Set.of() : Set.copyOf(tags);
|
||||
});
|
||||
|
||||
// Create some Post objects.
|
||||
Post post1 = new Post();
|
||||
post1.setMetadata(new Metadata());
|
||||
post1.getMetadata().setName("post-1");
|
||||
post1.setSpec(new Post.PostSpec());
|
||||
post1.getSpec().setTags(List.of("t1", "t2"));
|
||||
|
||||
Post post2 = new Post();
|
||||
post2.setMetadata(new Metadata());
|
||||
post2.getMetadata().setName("post-2");
|
||||
post2.setSpec(new Post.PostSpec());
|
||||
post2.getSpec().setTags(List.of("t2", "t3"));
|
||||
|
||||
indexer.add(tagsIndexName, post1);
|
||||
indexer.add(tagsIndexName, post2);
|
||||
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
|
||||
.containsExactlyInAnyOrder("post-1");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
|
||||
.containsExactlyInAnyOrder("post-1", "post-2");
|
||||
assertThat(indexer.getByIndex(tagsIndexName, "t3"))
|
||||
.containsExactlyInAnyOrder("post-2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void addButNotIndexFunc() {
|
||||
// Create some Post objects.
|
||||
Post post1 = new Post();
|
||||
post1.setMetadata(new Metadata());
|
||||
post1.getMetadata().setName("post-1");
|
||||
post1.setSpec(new Post.PostSpec());
|
||||
post1.getSpec().setTags(List.of("t1", "t2"));
|
||||
|
||||
// Create a new Indexer that indexes Post objects by tags.
|
||||
final DefaultIndexer<Post> indexer = new DefaultIndexer<>();
|
||||
assertThrows(IllegalArgumentException.class, () -> {
|
||||
indexer.add("fake-index-name", post1);
|
||||
}, "Index function not found for index name 'fake-index-name'");
|
||||
}
|
||||
}
|
|
@ -10,15 +10,15 @@ import static org.mockito.Mockito.when;
|
|||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import run.halo.app.content.TestPost;
|
||||
import run.halo.app.content.PostIndexInformer;
|
||||
import run.halo.app.content.permalinks.TagPermalinkPolicy;
|
||||
import run.halo.app.core.extension.content.Post;
|
||||
import run.halo.app.core.extension.content.Tag;
|
||||
import run.halo.app.extension.ExtensionClient;
|
||||
import run.halo.app.extension.Metadata;
|
||||
|
@ -37,6 +37,9 @@ class TagReconcilerTest {
|
|||
@Mock
|
||||
private TagPermalinkPolicy tagPermalinkPolicy;
|
||||
|
||||
@Mock
|
||||
private PostIndexInformer postIndexInformer;
|
||||
|
||||
@InjectMocks
|
||||
private TagReconciler tagReconciler;
|
||||
|
||||
|
@ -45,6 +48,8 @@ class TagReconcilerTest {
|
|||
Tag tag = tag();
|
||||
when(client.fetch(eq(Tag.class), eq("fake-tag")))
|
||||
.thenReturn(Optional.of(tag));
|
||||
when(postIndexInformer.getByTagName(eq("fake-tag")))
|
||||
.thenReturn(Set.of());
|
||||
when(tagPermalinkPolicy.permalink(any()))
|
||||
.thenAnswer(arg -> "/tags/" + tag.getSpec().getSlug());
|
||||
ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
|
||||
|
@ -80,7 +85,8 @@ class TagReconcilerTest {
|
|||
Tag tag = tag();
|
||||
when(client.fetch(eq(Tag.class), eq("fake-tag")))
|
||||
.thenReturn(Optional.of(tag));
|
||||
when(client.list(eq(Post.class), any(), any())).thenReturn(posts());
|
||||
when(postIndexInformer.getByTagName(eq("fake-tag")))
|
||||
.thenReturn(Set.of("fake-post-1", "fake-post-3"));
|
||||
|
||||
ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
|
||||
tagReconciler.reconcile(new TagReconciler.Request("fake-tag"));
|
||||
|
@ -101,23 +107,4 @@ class TagReconcilerTest {
|
|||
tag.setStatus(new Tag.TagStatus());
|
||||
return tag;
|
||||
}
|
||||
|
||||
private List<Post> posts() {
|
||||
Post post1 = TestPost.postV1();
|
||||
post1.getMetadata().setName("fake-post-1");
|
||||
post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
|
||||
post1.getSpec().setTags(List.of("fake-tag", "tag-A", "tag-B"));
|
||||
|
||||
Post post2 = TestPost.postV1();
|
||||
post2.getMetadata().setName("fake-post-2");
|
||||
post2.getSpec().setVisible(Post.VisibleEnum.INTERNAL);
|
||||
post2.getSpec().setTags(List.of("tag-A", "tag-C"));
|
||||
|
||||
Post post3 = TestPost.postV1();
|
||||
post3.getMetadata().setName("fake-post-3");
|
||||
post3.getSpec().setVisible(Post.VisibleEnum.PRIVATE);
|
||||
post3.getSpec().setTags(List.of("tag-A", "fake-tag"));
|
||||
return List.of(post1, post2, post3);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue