feat: add indexer implementation to solve the slow tag stats (#3420)

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.3.x

#### What this PR does / why we need it:
修复标签关联文章数量统计不正确的问题
1. 实现简要的 Indexer 可以通过 IndexFunc 来提取需要构建 indacates 的信息,比如对文章的标签和 labels 建立 indacates
2. Indexer 通过 Watch 文章数据来维护 Indicates 使其与数据库一致
3. TagReconciler 一分钟 requeue 一次,但直接通过 Indexer 获取文章名称来统计数量,无需 list 文章数据

how to test it?
1. 创建文章并绑定标签
2. 测试文章关联标签,关联多个标签、解除旧标签绑定等操作是否会在一分钟后在标签处正确显示文章数量
4. 查看主题端标签下文章数量统计是否正确
5. 重复上述操作多次

#### Which issue(s) this PR fixes:
Fixes #3311, #3312

#### Does this PR introduce a user-facing change?

```release-note
修复标签关联文章数量统计不正确的问题,优化标签数量多时耗时长的问题
```
pull/3434/head
guqing 2023-03-01 11:44:25 +08:00 committed by GitHub
parent aba151f54c
commit af07b42b42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 839 additions and 62 deletions

View File

@ -0,0 +1,129 @@
package run.halo.app.content;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.SetMultimap;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.springframework.lang.NonNull;
import org.springframework.util.Assert;
import run.halo.app.extension.Extension;
/**
* <p>A default implementation of {@link Indexer}.</p>
* <p>Note that this Indexer is not thread-safe, If multiple threads access this indexer
* concurrently and one of the threads modifies the indexer, it must be synchronized externally.</p>
*
* @param <T> the type of object to be indexed
* @author guqing
* @see
* <a href="https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/tools/cache/index.go">kubernetes index</a>
* @see <a href="https://juejin.cn/post/7132767272841510926">informercache.indexer</a>
* @since 2.0.0
*/
public class DefaultIndexer<T extends Extension> implements Indexer<T> {
private final Map<String, SetMultimap<String, String>> indices = new HashMap<>();
private final Map<String, IndexFunc<T>> indexFuncMap = new HashMap<>();
private final Map<String, SetMultimap<String, String>> indexValues = new HashMap<>();
@Override
public void addIndexFunc(String indexName, IndexFunc<T> indexFunc) {
indexFuncMap.put(indexName, indexFunc);
indices.put(indexName, HashMultimap.create());
indexValues.put(indexName, HashMultimap.create());
}
@Override
public Set<String> indexNames() {
return Set.copyOf(indexFuncMap.keySet());
}
@Override
public void add(String indexName, T obj) {
IndexFunc<T> indexFunc = getIndexFunc(indexName);
Set<String> indexKeys = indexFunc.apply(obj);
for (String indexKey : indexKeys) {
SetMultimap<String, String> index = indices.get(indexName);
index.put(indexKey, getObjectKey(obj));
SetMultimap<String, String> indexValue = indexValues.get(indexName);
indexValue.put(getObjectKey(obj), indexKey);
}
}
@NonNull
private IndexFunc<T> getIndexFunc(String indexName) {
IndexFunc<T> indexFunc = indexFuncMap.get(indexName);
if (indexFunc == null) {
throw new IllegalArgumentException(
"Index function not found for index name: " + indexName);
}
return indexFunc;
}
@Override
public void update(String indexName, T obj) {
IndexFunc<T> indexFunc = getIndexFunc(indexName);
Set<String> indexKeys = indexFunc.apply(obj);
Set<String> oldIndexKeys = new HashSet<>();
SetMultimap<String, String> indexValue = indexValues.get(indexName);
if (indexValue.containsKey(getObjectKey(obj))) {
oldIndexKeys.addAll(indexValue.get(getObjectKey(obj)));
}
// delete old index first
for (String oldIndexKey : oldIndexKeys) {
SetMultimap<String, String> index = indices.get(indexName);
index.remove(oldIndexKey, getObjectKey(obj));
indexValue.remove(getObjectKey(obj), oldIndexKey);
}
// add new index
for (String indexKey : indexKeys) {
SetMultimap<String, String> index = indices.get(indexName);
index.put(indexKey, getObjectKey(obj));
indexValue.put(getObjectKey(obj), indexKey);
}
}
@Override
public Set<String> getByIndex(String indexName, String indexKey) {
SetMultimap<String, String> index = indices.get(indexName);
if (index != null) {
return Set.copyOf(index.get(indexKey));
}
return Set.of();
}
@Override
public void delete(String indexName, T obj) {
IndexFunc<T> indexFunc = getIndexFunc(indexName);
SetMultimap<String, String> indexValue = indexValues.get(indexName);
Set<String> indexKeys = indexFunc.apply(obj);
for (String indexKey : indexKeys) {
String objectKey = getObjectKey(obj);
SetMultimap<String, String> index = indices.get(indexName);
index.remove(indexKey, objectKey);
indexValue.remove(indexKey, objectKey);
}
}
/**
* This method is only used for testing.
*
* @param indexName index name
* @return all indices of the given index name
*/
public Map<String, Collection<String>> getIndices(String indexName) {
return indices.get(indexName).asMap();
}
private String getObjectKey(T obj) {
Assert.notNull(obj, "Object must not be null");
Assert.notNull(obj.getMetadata(), "Object metadata must not be null");
Assert.notNull(obj.getMetadata().getName(), "Object name must not be null");
return obj.getMetadata().getName();
}
}

View File

@ -0,0 +1,79 @@
package run.halo.app.content;
import java.util.Set;
import run.halo.app.extension.Extension;
/**
* <p>Indexer is used to index objects by index name and index key.</p>
* <p>For example, if you want to index posts by category, you can use the following code:</p>
* <pre>
* Indexer&lt;Post&gt; indexer = new Indexer&lt;&gt;();
* indexer.addIndexFunc("category", post -&gt; {
* List&lt;String&gt; tags = post.getSpec().getTags();
* return tags == null ? Set.of() : Set.copyOf(tags);
* });
* indexer.add("category", post);
* indexer.getByIndex("category", "category-slug");
* indexer.update("category", post);
* indexer.delete("category", post);
* </pre>
*
* @param <T> the type of object to be indexed
* @author guqing
* @since 2.0.0
*/
public interface Indexer<T extends Extension> {
/**
* Adds an index function for a given index name.
*
* @param indexName The name of the index.
* @param indexFunc The function to use for indexing.
*/
void addIndexFunc(String indexName, DefaultIndexer.IndexFunc<T> indexFunc);
Set<String> indexNames();
/**
* The {@code add} method adds an object of type T to the index
* with the given name. It does this by first getting the index function for the given index
* name and applying it to the object to get a set of index keys. For each index key, it adds
* the object key to the index and the index key to the object's index values.
*
* <p>For example, if you want to index Person objects by name and age, you can use the
* following:</p>
* <pre>
* // Create an Indexer that indexes Person objects by name and age
* Indexer&lt;Person&gt; indexer = new Indexer<>();
* indexer.addIndexFunc("name", person -> Collections.singleton(person.getName()));
* indexer.addIndexFunc("age", person -> Collections.singleton(String.valueOf(person
* .getAge())));
*
* // Create some Person objects
* Person alice = new Person("Alice", 25);
* Person bob = new Person("Bob", 30);
*
* // Add the Person objects to the index
* indexer.add("name", alice);
* indexer.add("name", bob);
* indexer.add("age", alice);
* indexer.add("age", bob);
* </pre>
*
* @param indexName The name of the index.
* @param obj The function to use for indexing.
* @throws IllegalArgumentException if the index name is not found.
*/
void add(String indexName, T obj);
void update(String indexName, T obj);
Set<String> getByIndex(String indexName, String indexKey);
void delete(String indexName, T obj);
@FunctionalInterface
interface IndexFunc<T> {
Set<String> apply(T obj);
}
}

View File

@ -0,0 +1,226 @@
package run.halo.app.content;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.StampedLock;
import java.util.function.BiConsumer;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.ApplicationListener;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionUtil;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.Unstructured;
import run.halo.app.extension.Watcher;
import run.halo.app.extension.controller.RequestSynchronizer;
import run.halo.app.infra.SchemeInitializedEvent;
/**
* <p>Monitor changes to {@link Post} resources and establish a local, in-memory cache in an
* Indexer.
* When changes to posts are detected, the Indexer is updated using the indexFunc to maintain
* its integrity.
* This enables quick retrieval of the unique identifier(It is usually {@link Metadata#getName()})
* for article objects using the getByIndex method when needed.</p>
*
* @author guqing
* @since 2.0.0
*/
@Component
public class PostIndexInformer implements ApplicationListener<SchemeInitializedEvent>,
DisposableBean {
public static final String TAG_POST_INDEXER = "tag-post-indexer";
public static final String LABEL_INDEXER_NAME = "post-label-indexer";
private final RequestSynchronizer synchronizer;
private final Indexer<Post> postIndexer;
private final PostWatcher postWatcher;
public PostIndexInformer(ExtensionClient client) {
postIndexer = new DefaultIndexer<>();
postIndexer.addIndexFunc(TAG_POST_INDEXER, post -> {
List<String> tags = post.getSpec().getTags();
return tags != null ? Set.copyOf(tags) : Set.of();
});
postIndexer.addIndexFunc(LABEL_INDEXER_NAME, labelIndexFunc());
this.postWatcher = new PostWatcher();
this.synchronizer = new RequestSynchronizer(true,
client,
new Post(),
postWatcher,
this::checkExtension);
}
private DefaultIndexer.IndexFunc<Post> labelIndexFunc() {
return post -> {
Map<String, String> labels = ExtensionUtil.nullSafeLabels(post);
Set<String> indexKeys = new HashSet<>();
for (Map.Entry<String, String> entry : labels.entrySet()) {
indexKeys.add(labelKey(entry.getKey(), entry.getValue()));
}
return indexKeys;
};
}
public Set<String> getByIndex(String indexName, String indexKey) {
return postIndexer.getByIndex(indexName, indexKey);
}
public Set<String> getByTagName(String tagName) {
return postIndexer.getByIndex(TAG_POST_INDEXER, tagName);
}
public Set<String> getByLabels(Map<String, String> labels) {
if (labels == null) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (Map.Entry<String, String> entry : labels.entrySet()) {
Set<String> values = postIndexer.getByIndex(LABEL_INDEXER_NAME,
labelKey(entry.getKey(), entry.getValue()));
if (values == null) {
// No objects have this label, no need to continue searching
return Set.of();
}
if (result.isEmpty()) {
result.addAll(values);
} else {
result.retainAll(values);
}
}
return result;
}
String labelKey(String labelName, String labelValue) {
return labelName + "=" + labelValue;
}
public Set<String> getByLabel(String labelName, String labelValue) {
return postIndexer.getByIndex(LABEL_INDEXER_NAME, labelKey(labelName, labelValue));
}
@Override
public void destroy() throws Exception {
if (postWatcher != null) {
postWatcher.dispose();
}
if (synchronizer != null) {
synchronizer.dispose();
}
}
@Override
public void onApplicationEvent(@NonNull SchemeInitializedEvent event) {
if (!synchronizer.isStarted()) {
synchronizer.start();
}
}
class PostWatcher implements Watcher {
private Runnable disposeHook;
private boolean disposed = false;
private final StampedLock lock = new StampedLock();
@Override
public void onAdd(Extension extension) {
if (!checkExtension(extension)) {
return;
}
handleIndicates(extension, postIndexer::add);
}
@Override
public void onUpdate(Extension oldExt, Extension newExt) {
if (!checkExtension(newExt)) {
return;
}
handleIndicates(newExt, postIndexer::update);
}
@Override
public void onDelete(Extension extension) {
if (!checkExtension(extension)) {
return;
}
handleIndicates(extension, postIndexer::delete);
}
@Override
public void registerDisposeHook(Runnable dispose) {
this.disposeHook = dispose;
}
@Override
public void dispose() {
if (isDisposed()) {
return;
}
this.disposed = true;
if (this.disposeHook != null) {
this.disposeHook.run();
}
}
@Override
public boolean isDisposed() {
return this.disposed;
}
void handleIndicates(Extension extension, BiConsumer<String, Post> consumer) {
Post post = convertTo(extension);
Set<String> indexNames = getIndexNames();
for (String indexName : indexNames) {
maintainIndicates(indexName, post, consumer);
}
}
Set<String> getIndexNames() {
long stamp = lock.tryOptimisticRead();
Set<String> indexNames = postIndexer.indexNames();
if (!lock.validate(stamp)) {
stamp = lock.readLock();
try {
return postIndexer.indexNames();
} finally {
lock.unlockRead(stamp);
}
}
return indexNames;
}
void maintainIndicates(String indexName, Post post, BiConsumer<String, Post> consumer) {
long stamp = lock.writeLock();
try {
consumer.accept(indexName, post);
} finally {
lock.unlockWrite(stamp);
}
}
}
private Post convertTo(Extension extension) {
if (extension instanceof Post) {
return (Post) extension;
}
return Unstructured.OBJECT_MAPPER.convertValue(extension, Post.class);
}
private boolean checkExtension(Extension extension) {
return !postWatcher.isDisposed()
&& extension.getMetadata().getDeletionTimestamp() == null
&& isPost(extension);
}
private boolean isPost(Extension extension) {
return GroupVersionKind.fromExtension(Post.class).equals(extension.groupVersionKind());
}
}

View File

@ -1,12 +1,14 @@
package run.halo.app.core.extension.reconciler;
import java.time.Duration;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import run.halo.app.content.PostIndexInformer;
import run.halo.app.content.permalinks.TagPermalinkPolicy;
import run.halo.app.core.extension.content.Constant;
import run.halo.app.core.extension.content.Post;
@ -25,23 +27,20 @@ import run.halo.app.infra.utils.JsonUtils;
* @since 2.0.0
*/
@Component
@RequiredArgsConstructor
public class TagReconciler implements Reconciler<Reconciler.Request> {
private static final String FINALIZER_NAME = "tag-protection";
private final ExtensionClient client;
private final TagPermalinkPolicy tagPermalinkPolicy;
public TagReconciler(ExtensionClient client, TagPermalinkPolicy tagPermalinkPolicy) {
this.client = client;
this.tagPermalinkPolicy = tagPermalinkPolicy;
}
private final PostIndexInformer postIndexInformer;
@Override
public Result reconcile(Request request) {
client.fetch(Tag.class, request.name())
.ifPresent(tag -> {
return client.fetch(Tag.class, request.name())
.map(tag -> {
if (isDeleted(tag)) {
cleanUpResourcesAndRemoveFinalizer(request.name());
return;
return Result.doNotRetry();
}
addFinalizerIfNecessary(tag);
@ -50,14 +49,15 @@ public class TagReconciler implements Reconciler<Reconciler.Request> {
this.reconcileStatusPermalink(request.name());
reconcileStatusPosts(request.name());
});
return new Result(false, null);
return new Result(true, Duration.ofMinutes(1));
})
.orElse(Result.doNotRetry());
}
@Override
public Controller setupWith(ControllerBuilder builder) {
return builder
.syncAllOnStart(false)
.syncAllOnStart(true)
.extension(new Tag())
.build();
}
@ -128,29 +128,20 @@ public class TagReconciler implements Reconciler<Reconciler.Request> {
}
private void populatePosts(Tag tag) {
List<Post.CompactPost> compactPosts = client.list(Post.class, null, null)
.stream()
.filter(post -> includes(post.getSpec().getTags(), tag.getMetadata().getName()))
.map(post -> Post.CompactPost.builder()
.name(post.getMetadata().getName())
.published(post.isPublished())
.visible(post.getSpec().getVisible())
.build())
.toList();
tag.getStatusOrDefault().setPostCount(compactPosts.size());
// populate post count
Set<String> postNames = postIndexInformer.getByTagName(tag.getMetadata().getName());
tag.getStatusOrDefault().setPostCount(postNames.size());
long visiblePostCount = compactPosts.stream()
.filter(post -> Objects.equals(true, post.getPublished())
&& Post.VisibleEnum.PUBLIC.equals(post.getVisible()))
.count();
tag.getStatusOrDefault().setVisiblePostCount((int) visiblePostCount);
}
// populate visible post count
Map<String, String> labelToQuery = Map.of(Post.PUBLISHED_LABEL, BooleanUtils.TRUE,
Post.VISIBLE_LABEL, Post.VisibleEnum.PUBLIC.name(),
Post.DELETED_LABEL, BooleanUtils.FALSE);
Set<String> hasAllLabelPosts = postIndexInformer.getByLabels(labelToQuery);
private boolean includes(List<String> tags, String tagName) {
if (tags == null) {
return false;
}
return tags.contains(tagName);
// retain all posts that has all labels
Set<String> postNamesWithTag = new HashSet<>(postNames);
postNamesWithTag.retainAll(hasAllLabelPosts);
tag.getStatusOrDefault().setVisiblePostCount(postNamesWithTag.size());
}
private boolean isDeleted(Tag tag) {

View File

@ -162,11 +162,8 @@ public class DefaultController<R> implements Controller {
watch.start("reconciliation");
result = reconciler.reconcile(entry.getEntry());
watch.stop();
log.debug("{} >>> Reconciled request: {} with result: {}", this.name,
entry.getEntry(), result);
if (log.isTraceEnabled()) {
log.trace(watch.toString());
}
log.debug("{} >>> Reconciled request: {} with result: {}, usage: {}",
this.name, entry.getEntry(), result, watch.getTotalTimeMillis());
} catch (Throwable t) {
if (t instanceof OptimisticLockingFailureException) {
log.warn("Optimistic locking failure when reconciling request: {}/{}",

View File

@ -0,0 +1,368 @@
package run.halo.app.content;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONException;
import org.junit.jupiter.api.Test;
import org.skyscreamer.jsonassert.JSONAssert;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.Metadata;
import run.halo.app.infra.utils.JsonUtils;
/**
* Tests for {@link DefaultIndexer}.
*
* @author guqing
* @since 2.0.0
*/
class DefaultIndexerTest {
@Test
public void testTagsIndexer() throws JSONException {
// Create a new Indexer that indexes Post objects by tags.
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
String tagsIndexName = "tags";
indexer.addIndexFunc(tagsIndexName, post -> {
List<String> tags = post.getSpec().getTags();
return tags == null ? Set.of() : Set.copyOf(tags);
});
// Create some Post objects.
Post post1 = new Post();
post1.setMetadata(new Metadata());
post1.getMetadata().setName("post-1");
post1.setSpec(new Post.PostSpec());
post1.getSpec().setTags(List.of("t1", "t2"));
Post post2 = new Post();
post2.setMetadata(new Metadata());
post2.getMetadata().setName("post-2");
post2.setSpec(new Post.PostSpec());
post2.getSpec().setTags(List.of("t2", "t3"));
Post post3 = new Post();
post3.setMetadata(new Metadata());
post3.getMetadata().setName("post-3");
post3.setSpec(new Post.PostSpec());
post3.getSpec().setTags(List.of("t3", "t4"));
// Add the Post objects to the Indexer.
indexer.add(tagsIndexName, post1);
indexer.add(tagsIndexName, post2);
indexer.add(tagsIndexName, post3);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"t4": [
"post-3"
],
"t3": [
"post-2",
"post-3"
],
"t2": [
"post-1",
"post-2"
],
"t1": [
"post-1"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("tags")),
true);
// Remove post2 from the Indexer.
indexer.delete(tagsIndexName, post2);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"t1": [
"post-1"
],
"t2": [
"post-1"
],
"t3": [
"post-3"
],
"t4": [
"post-3"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("tags")),
true);
// Update post3 in the Indexer.
post3.getSpec().setTags(List.of("t4", "t5"));
indexer.update(tagsIndexName, post3);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"t1": [
"post-1"
],
"t2": [
"post-1"
],
"t4": [
"post-3"
],
"t5": [
"post-3"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("tags")),
true);
}
@Test
public void testLabelIndexer() throws JSONException {
// Create a new Indexer.
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
// Define the IndexFunc for labels.
DefaultIndexer.IndexFunc<Post> labelIndexFunc = labelIndexFunc();
// Add the label IndexFunc to the Indexer.
String labelsIndexName = "labels";
indexer.addIndexFunc(labelsIndexName, labelIndexFunc);
// Create some posts with labels.
Post post1 = new Post();
post1.setMetadata(new Metadata());
post1.getMetadata().setName("post-1");
post1.getMetadata().setLabels(Map.of("app", "myapp", "env", "prod"));
Post post2 = new Post();
post2.setMetadata(new Metadata());
post2.getMetadata().setName("post-2");
post2.getMetadata().setLabels(Map.of("app", "myapp", "env", "test"));
Post post3 = new Post();
post3.setMetadata(new Metadata());
post3.getMetadata().setName("post-3");
post3.getMetadata().setLabels(Map.of("app", "otherapp", "env", "prod"));
// Add the posts to the Indexer.
indexer.add(labelsIndexName, post1);
indexer.add(labelsIndexName, post2);
indexer.add(labelsIndexName, post3);
// Verify that the Indexer has the correct indices.
assertEquals(
Map.of(
"app=myapp", Set.of("post-1", "post-2"),
"app=otherapp", Set.of("post-3"),
"env=test", Set.of("post-2"),
"env=prod", Set.of("post-1", "post-3")
),
indexer.getIndices("labels"));
// Delete post2 from the Indexer.
indexer.delete(labelsIndexName, post2);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"app=myapp": [
"post-1"
],
"env=prod": [
"post-1",
"post-3"
],
"app=otherapp": [
"post-3"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("labels")),
true);
// Update post2 in the Indexer.
post2.getMetadata().setLabels(Map.of("l1", "v1", "l2", "v2"));
indexer.update(labelsIndexName, post2);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"app=myapp": [
"post-1"
],
"env=prod": [
"post-1",
"post-3"
],
"app=otherapp": [
"post-3"
],
"l1=v1": [
"post-2"
],
"l2=v2": [
"post-2"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("labels")),
true);
// Update post1 in the Indexer.
post1.getMetadata().setLabels(Map.of("l2", "v2", "l3", "v3"));
indexer.update(labelsIndexName, post1);
// Verify that the Indexer has the correct indices.
JSONAssert.assertEquals("""
{
"env=prod": [
"post-3"
],
"app=otherapp": [
"post-3"
],
"l1=v1": [
"post-2"
],
"l2=v2": [
"post-1",
"post-2"
],
"l3=v3": [
"post-1"
]
}
""",
JsonUtils.objectToJson(indexer.getIndices("labels")),
true);
}
@Test
void multiIndexName() {
// Create a new Indexer.
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
// Define the IndexFunc for labels.
String labelsIndexName = "labels";
DefaultIndexer.IndexFunc<Post> labelIndexFunc = labelIndexFunc();
indexer.addIndexFunc(labelsIndexName, labelIndexFunc);
String tagsIndexName = "tags";
indexer.addIndexFunc(tagsIndexName, post -> {
List<String> tags = post.getSpec().getTags();
return tags == null ? Set.of() : Set.copyOf(tags);
});
Post post1 = new Post();
post1.setMetadata(new Metadata());
post1.getMetadata().setName("post-1");
post1.getMetadata().setLabels(Map.of("app", "myapp", "env", "prod"));
post1.setSpec(new Post.PostSpec());
post1.getSpec().setTags(List.of("t1", "t2"));
Post post2 = new Post();
post2.setMetadata(new Metadata());
post2.getMetadata().setName("post-2");
post2.getMetadata().setLabels(Map.of("app", "myapp", "env", "test"));
post2.setSpec(new Post.PostSpec());
post2.getSpec().setTags(List.of("t2", "t3"));
indexer.add(labelsIndexName, post1);
indexer.add(tagsIndexName, post1);
indexer.add(labelsIndexName, post2);
indexer.add(tagsIndexName, post2);
assertThat(indexer.getByIndex(labelsIndexName, "app=myapp"))
.containsExactlyInAnyOrder("post-1", "post-2");
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
.containsExactlyInAnyOrder("post-1");
assertThat(indexer.getByIndex(labelsIndexName, "env=test"))
.containsExactlyInAnyOrder("post-2");
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
.containsExactlyInAnyOrder("post-1", "post-2");
post2.getSpec().setTags(List.of("t1", "t4"));
indexer.update(tagsIndexName, post2);
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
.containsExactlyInAnyOrder("post-1", "post-2");
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
.containsExactlyInAnyOrder("post-1");
assertThat(indexer.getByIndex(tagsIndexName, "t4"))
.containsExactlyInAnyOrder("post-2");
}
private static DefaultIndexer.IndexFunc<Post> labelIndexFunc() {
return post -> {
Map<String, String> labels = post.getMetadata().getLabels();
Set<String> indexKeys = new HashSet<>();
if (labels != null) {
for (Map.Entry<String, String> entry : labels.entrySet()) {
indexKeys.add(entry.getKey() + "=" + entry.getValue());
}
}
return indexKeys;
};
}
@Test
void getByIndex() {
DefaultIndexer<Post> indexer = new DefaultIndexer<>();
String tagsIndexName = "tags";
indexer.addIndexFunc(tagsIndexName, post -> {
List<String> tags = post.getSpec().getTags();
return tags == null ? Set.of() : Set.copyOf(tags);
});
// Create some Post objects.
Post post1 = new Post();
post1.setMetadata(new Metadata());
post1.getMetadata().setName("post-1");
post1.setSpec(new Post.PostSpec());
post1.getSpec().setTags(List.of("t1", "t2"));
Post post2 = new Post();
post2.setMetadata(new Metadata());
post2.getMetadata().setName("post-2");
post2.setSpec(new Post.PostSpec());
post2.getSpec().setTags(List.of("t2", "t3"));
indexer.add(tagsIndexName, post1);
indexer.add(tagsIndexName, post2);
assertThat(indexer.getByIndex(tagsIndexName, "t1"))
.containsExactlyInAnyOrder("post-1");
assertThat(indexer.getByIndex(tagsIndexName, "t2"))
.containsExactlyInAnyOrder("post-1", "post-2");
assertThat(indexer.getByIndex(tagsIndexName, "t3"))
.containsExactlyInAnyOrder("post-2");
}
@Test
void addButNotIndexFunc() {
// Create some Post objects.
Post post1 = new Post();
post1.setMetadata(new Metadata());
post1.getMetadata().setName("post-1");
post1.setSpec(new Post.PostSpec());
post1.getSpec().setTags(List.of("t1", "t2"));
// Create a new Indexer that indexes Post objects by tags.
final DefaultIndexer<Post> indexer = new DefaultIndexer<>();
assertThrows(IllegalArgumentException.class, () -> {
indexer.add("fake-index-name", post1);
}, "Index function not found for index name 'fake-index-name'");
}
}

View File

@ -10,15 +10,15 @@ import static org.mockito.Mockito.when;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.content.TestPost;
import run.halo.app.content.PostIndexInformer;
import run.halo.app.content.permalinks.TagPermalinkPolicy;
import run.halo.app.core.extension.content.Post;
import run.halo.app.core.extension.content.Tag;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.Metadata;
@ -37,6 +37,9 @@ class TagReconcilerTest {
@Mock
private TagPermalinkPolicy tagPermalinkPolicy;
@Mock
private PostIndexInformer postIndexInformer;
@InjectMocks
private TagReconciler tagReconciler;
@ -45,6 +48,8 @@ class TagReconcilerTest {
Tag tag = tag();
when(client.fetch(eq(Tag.class), eq("fake-tag")))
.thenReturn(Optional.of(tag));
when(postIndexInformer.getByTagName(eq("fake-tag")))
.thenReturn(Set.of());
when(tagPermalinkPolicy.permalink(any()))
.thenAnswer(arg -> "/tags/" + tag.getSpec().getSlug());
ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
@ -80,7 +85,8 @@ class TagReconcilerTest {
Tag tag = tag();
when(client.fetch(eq(Tag.class), eq("fake-tag")))
.thenReturn(Optional.of(tag));
when(client.list(eq(Post.class), any(), any())).thenReturn(posts());
when(postIndexInformer.getByTagName(eq("fake-tag")))
.thenReturn(Set.of("fake-post-1", "fake-post-3"));
ArgumentCaptor<Tag> captor = ArgumentCaptor.forClass(Tag.class);
tagReconciler.reconcile(new TagReconciler.Request("fake-tag"));
@ -101,23 +107,4 @@ class TagReconcilerTest {
tag.setStatus(new Tag.TagStatus());
return tag;
}
private List<Post> posts() {
Post post1 = TestPost.postV1();
post1.getMetadata().setName("fake-post-1");
post1.getSpec().setVisible(Post.VisibleEnum.PUBLIC);
post1.getSpec().setTags(List.of("fake-tag", "tag-A", "tag-B"));
Post post2 = TestPost.postV1();
post2.getMetadata().setName("fake-post-2");
post2.getSpec().setVisible(Post.VisibleEnum.INTERNAL);
post2.getSpec().setTags(List.of("tag-A", "tag-C"));
Post post3 = TestPost.postV1();
post3.getMetadata().setName("fake-post-3");
post3.getSpec().setVisible(Post.VisibleEnum.PRIVATE);
post3.getSpec().setTags(List.of("tag-A", "fake-tag"));
return List.of(post1, post2, post3);
}
}
}