feat: add index mechanism for extension (#5121)

#### What type of PR is this?
/kind feature
/area core
/milestone 2.12.x

#### What this PR does / why we need it:
新增自定义模型索引机制

默认为所有的自定义模型都添加了以下索引:
- metadata.name
- metadata.labels
- metadata.creationTimestamp
- metadata.deletionTimestamp

**how to test it?**
1. 测试应用的启动和停止
2. 测试 Reconciler 被正确执行,如创建文章发布文章,测试删除文章的某个 label 数据启动后能被 PostReconciler 恢复(即Reconciler 被正确执行)
3. 测试自定义模型自动生成的 list APIs
	1. 能根据 labels 正确过滤数据和分页
	2. 能根据 creationTimestamp 正确排序
	3. 测试插件启用后也能正确使用 list APIs 根据 labels 过滤数据和 creationTimestamp 排序
4. 能正确删除数据(则表示 GcReconciler 使用索引正确)
5. 测试在插件中为自定义模型注册索引
```java
public class DemoPlugin extension BasePlugin {
    private final SchemeManager schemeManager;

    public MomentsPlugin(PluginContext pluginContext, SchemeManager schemeManager) {
        super(pluginContext);
        this.schemeManager = schemeManager;
    }

    @Override
    public void start() {
        schemeManager.register(Moment.class, indexSpecs -> {
            indexSpecs.add(new IndexSpec()
                .setName("spec.tags")
                .setIndexFunc(multiValueAttribute(Moment.class, moment -> {
                    var tags = moment.getSpec().getTags();
                    return tags == null ? Set.of() : tags;
                }))
            );
            indexSpecs.add(new IndexSpec()
                .setName("spec.owner")
                .setIndexFunc(simpleAttribute(Moment.class,
                    moment -> moment.getSpec().getOwner())
                )
            );
            indexSpecs.add(new IndexSpec()
                .setName("spec.releaseTime")
                .setIndexFunc(simpleAttribute(Moment.class, moment -> {
                    var releaseTime = moment.getSpec().getReleaseTime();
                    return releaseTime == null ? null : releaseTime.toString();
                }))
            );

            indexSpecs.add(new IndexSpec()
                .setName("spec.visible")
                .setIndexFunc(simpleAttribute(Moment.class, moment -> {
                    var visible = moment.getSpec().getVisible();
                    return visible == null ? null : visible.toString();
                }))
            );
        });
    }

    @Override
    public void stop() {
        // unregister scheme 即可,不需要手动删除索引
    }
}
```
可以正确在自动生成的 list APIs 使用 fieldSelector 来过滤 `spec.slug` 和排序,可以自己添加其他的 indexSpec 测试
6. 测试唯一索引并添加重复数据,期望无法添加进去

#### Which issue(s) this PR fixes:
Fixes #5058

#### Does this PR introduce a user-facing change?
```release-note
新增自定义模型索引机制
```
pull/5210/head
guqing 2024-01-19 14:36:09 +08:00 committed by GitHub
parent 3ebb45c266
commit 6a37df07a8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
115 changed files with 6609 additions and 170 deletions

View File

@ -0,0 +1,57 @@
package run.halo.app.extension;
import lombok.Builder;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.springframework.util.CollectionUtils;
import run.halo.app.extension.index.query.QueryFactory;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
@Getter
@RequiredArgsConstructor
@Builder(builderMethodName = "internalBuilder")
public class DefaultExtensionMatcher implements ExtensionMatcher {
private final ExtensionClient client;
private final GroupVersionKind gvk;
private final LabelSelector labelSelector;
private final FieldSelector fieldSelector;
public static DefaultExtensionMatcherBuilder builder(ExtensionClient client,
GroupVersionKind gvk) {
return internalBuilder().client(client).gvk(gvk);
}
/**
* Match the given extension with the current matcher.
*
* @param extension extension to match
* @return true if the extension matches the current matcher
*/
@Override
public boolean match(Extension extension) {
if (!gvk.equals(extension.groupVersionKind())) {
return false;
}
if (!hasFieldSelector() && !hasLabelSelector()) {
return true;
}
var listOptions = new ListOptions();
listOptions.setLabelSelector(labelSelector);
var fieldQuery = QueryFactory.all();
if (hasFieldSelector()) {
fieldQuery = QueryFactory.and(fieldQuery, fieldSelector.query());
}
listOptions.setFieldSelector(new FieldSelector(fieldQuery));
return client.indexedQueryEngine().retrieve(getGvk(),
listOptions, PageRequestImpl.ofSize(1)).getTotal() > 0;
}
boolean hasFieldSelector() {
return fieldSelector != null && fieldSelector.query() != null;
}
boolean hasLabelSelector() {
return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers());
}
}

View File

@ -4,6 +4,8 @@ import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.index.IndexedQueryEngine;
/**
* ExtensionClient is an interface which contains some operations on Extension instead of
@ -42,6 +44,11 @@ public interface ExtensionClient {
<E extends Extension> ListResult<E> list(Class<E> type, Predicate<E> predicate,
Comparator<E> comparator, int page, int size);
<E extends Extension> List<E> listAll(Class<E> type, ListOptions options, Sort sort);
<E extends Extension> ListResult<E> listBy(Class<E> type, ListOptions options,
PageRequest page);
/**
* Fetches Extension by its type and name.
*
@ -82,6 +89,8 @@ public interface ExtensionClient {
*/
<E extends Extension> void delete(E extension);
IndexedQueryEngine indexedQueryEngine();
void watch(Watcher watcher);
}

View File

@ -0,0 +1,14 @@
package run.halo.app.extension;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
public interface ExtensionMatcher {
GroupVersionKind getGvk();
LabelSelector getLabelSelector();
FieldSelector getFieldSelector();
boolean match(Extension extension);
}

View File

@ -0,0 +1,13 @@
package run.halo.app.extension;
import lombok.Data;
import lombok.experimental.Accessors;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
@Data
@Accessors(chain = true)
public class ListOptions {
private LabelSelector labelSelector;
private FieldSelector fieldSelector;
}

View File

@ -0,0 +1,65 @@
package run.halo.app.extension;
import org.springframework.data.domain.Sort;
import org.springframework.util.Assert;
/**
* <p>{@link PageRequest} is an interface for pagination information.</p>
* <p>Page number starts from 1.</p>
* <p>if page size is 0, it means no pagination and all results will be returned.</p>
*
* @author guqing
* @see PageRequestImpl
* @since 2.12.0
*/
public interface PageRequest {
int getPageNumber();
int getPageSize();
PageRequest previous();
PageRequest next();
/**
* Returns the previous {@link PageRequest} or the first {@link PageRequest} if the current one
* already is the first one.
*
* @return a new {@link org.springframework.data.domain.PageRequest} with
* {@link #getPageNumber()} - 1 as {@link #getPageNumber()}
*/
PageRequest previousOrFirst();
/**
* Returns the {@link PageRequest} requesting the first page.
*
* @return a new {@link org.springframework.data.domain.PageRequest} with
* {@link #getPageNumber()} = 1 as {@link #getPageNumber()}
*/
PageRequest first();
/**
* Creates a new {@link PageRequest} with {@code pageNumber} applied.
*
* @param pageNumber 1-based page index.
* @return a new {@link org.springframework.data.domain.PageRequest}
*/
PageRequest withPage(int pageNumber);
PageRequestImpl withSort(Sort sort);
boolean hasPrevious();
Sort getSort();
/**
* Returns the current {@link Sort} or the given one if the current one is unsorted.
*
* @param sort must not be {@literal null}.
* @return the current {@link Sort} or the given one if the current one is unsorted.
*/
default Sort getSortOr(Sort sort) {
Assert.notNull(sort, "Fallback Sort must not be null");
return getSort().isSorted() ? getSort() : sort;
}
}

View File

@ -0,0 +1,86 @@
package run.halo.app.extension;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
import org.springframework.data.domain.Sort;
import org.springframework.util.Assert;
public class PageRequestImpl implements PageRequest {
private final int pageNumber;
private final int pageSize;
private final Sort sort;
public PageRequestImpl(int pageNumber, int pageSize, Sort sort) {
Assert.notNull(sort, "Sort must not be null");
Assert.isTrue(pageNumber >= 0, "Page index must not be less than zero!");
Assert.isTrue(pageSize >= 0, "Page size must not be less than one!");
this.pageNumber = pageNumber;
this.pageSize = pageSize;
this.sort = sort;
}
public static PageRequestImpl of(int pageNumber, int pageSize) {
return of(pageNumber, pageSize, Sort.unsorted());
}
public static PageRequestImpl of(int pageNumber, int pageSize, Sort sort) {
return new PageRequestImpl(pageNumber, pageSize, sort);
}
public static PageRequestImpl ofSize(int pageSize) {
return PageRequestImpl.of(1, pageSize);
}
@Override
public int getPageNumber() {
return pageNumber;
}
@Override
public int getPageSize() {
return pageSize;
}
@Override
public PageRequest previous() {
return getPageNumber() == 0 ? this
: new PageRequestImpl(getPageNumber() - 1, getPageSize(), getSort());
}
@Override
public Sort getSort() {
return sort;
}
@Override
public PageRequest next() {
return new PageRequestImpl(getPageNumber() + 1, getPageSize(), getSort());
}
@Override
public PageRequest previousOrFirst() {
return hasPrevious() ? previous() : first();
}
@Override
public PageRequest first() {
return new PageRequestImpl(1, getPageSize(), getSort());
}
@Override
public PageRequest withPage(int pageNumber) {
return new PageRequestImpl(pageNumber, getPageSize(), getSort());
}
@Override
public PageRequestImpl withSort(Sort sort) {
return new PageRequestImpl(getPageNumber(), getPageSize(),
defaultIfNull(sort, Sort.unsorted()));
}
@Override
public boolean hasPrevious() {
return pageNumber > 1;
}
}

View File

@ -2,8 +2,10 @@ package run.halo.app.extension;
import java.util.Comparator;
import java.util.function.Predicate;
import org.springframework.data.domain.Sort;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import run.halo.app.extension.index.IndexedQueryEngine;
/**
* ExtensionClient is an interface which contains some operations on Extension instead of
@ -39,6 +41,11 @@ public interface ReactiveExtensionClient {
<E extends Extension> Mono<ListResult<E>> list(Class<E> type, Predicate<E> predicate,
Comparator<E> comparator, int page, int size);
<E extends Extension> Flux<E> listAll(Class<E> type, ListOptions options, Sort sort);
<E extends Extension> Mono<ListResult<E>> listBy(Class<E> type, ListOptions options,
PageRequest pageable);
/**
* Fetches Extension by its type and name.
*
@ -80,6 +87,8 @@ public interface ReactiveExtensionClient {
*/
<E extends Extension> Mono<E> delete(E extension);
IndexedQueryEngine indexedQueryEngine();
void watch(Watcher watcher);
}

View File

@ -3,13 +3,17 @@ package run.halo.app.extension;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
import org.springframework.lang.NonNull;
import run.halo.app.extension.exception.SchemeNotFoundException;
import run.halo.app.extension.index.IndexSpecs;
public interface SchemeManager {
void register(@NonNull Scheme scheme);
void register(@NonNull Scheme scheme, Consumer<IndexSpecs> specsConsumer);
/**
* Registers an Extension using its type.
*
@ -20,6 +24,9 @@ public interface SchemeManager {
register(Scheme.buildFromType(type));
}
default <T extends Extension> void register(Class<T> type, Consumer<IndexSpecs> specsConsumer) {
register(Scheme.buildFromType(type), specsConsumer);
}
void unregister(@NonNull Scheme scheme);

View File

@ -3,9 +3,14 @@ package run.halo.app.extension;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import reactor.core.Disposable;
import run.halo.app.extension.controller.Reconciler;
public interface Watcher extends Disposable {
default void onAdd(Reconciler.Request request) {
// Do nothing here, just for sync all on start.
}
default void onAdd(Extension extension) {
// Do nothing here
}

View File

@ -0,0 +1,61 @@
package run.halo.app.extension;
import java.util.Objects;
import lombok.Builder;
import lombok.Getter;
import org.springframework.util.Assert;
public class WatcherExtensionMatchers {
@Getter
private final ExtensionClient client;
private final GroupVersionKind gvk;
private final ExtensionMatcher onAddMatcher;
private final ExtensionMatcher onUpdateMatcher;
private final ExtensionMatcher onDeleteMatcher;
/**
* Constructs a new {@link WatcherExtensionMatchers} with the given
* {@link DefaultExtensionMatcher}.
*/
@Builder(builderMethodName = "internalBuilder")
public WatcherExtensionMatchers(ExtensionClient client,
GroupVersionKind gvk, ExtensionMatcher onAddMatcher,
ExtensionMatcher onUpdateMatcher, ExtensionMatcher onDeleteMatcher) {
Assert.notNull(client, "The client must not be null.");
Assert.notNull(gvk, "The gvk must not be null.");
this.client = client;
this.gvk = gvk;
this.onAddMatcher =
Objects.requireNonNullElse(onAddMatcher, emptyMatcher(client, gvk));
this.onUpdateMatcher =
Objects.requireNonNullElse(onUpdateMatcher, emptyMatcher(client, gvk));
this.onDeleteMatcher =
Objects.requireNonNullElse(onDeleteMatcher, emptyMatcher(client, gvk));
}
public GroupVersionKind getGroupVersionKind() {
return this.gvk;
}
public ExtensionMatcher onAddMatcher() {
return this.onAddMatcher;
}
public ExtensionMatcher onUpdateMatcher() {
return this.onUpdateMatcher;
}
public ExtensionMatcher onDeleteMatcher() {
return this.onDeleteMatcher;
}
public static WatcherExtensionMatchersBuilder builder(ExtensionClient client,
GroupVersionKind gvk) {
return internalBuilder().gvk(gvk).client(client);
}
static ExtensionMatcher emptyMatcher(ExtensionClient client,
GroupVersionKind gvk) {
return DefaultExtensionMatcher.builder(client, gvk).build();
}
}

View File

@ -2,13 +2,12 @@ package run.halo.app.extension.controller;
import java.time.Duration;
import java.time.Instant;
import java.util.function.BiPredicate;
import java.util.function.Predicate;
import java.util.function.Supplier;
import org.springframework.util.Assert;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.WatcherPredicates;
import run.halo.app.extension.ExtensionMatcher;
import run.halo.app.extension.WatcherExtensionMatchers;
import run.halo.app.extension.controller.Reconciler.Request;
public class ControllerBuilder {
@ -19,17 +18,17 @@ public class ControllerBuilder {
private Duration maxDelay;
private Reconciler<Request> reconciler;
private final Reconciler<Request> reconciler;
private Supplier<Instant> nowSupplier;
private Extension extension;
private Predicate<Extension> onAddPredicate;
private ExtensionMatcher onAddMatcher;
private Predicate<Extension> onDeletePredicate;
private ExtensionMatcher onDeleteMatcher;
private BiPredicate<Extension, Extension> onUpdatePredicate;
private ExtensionMatcher onUpdateMatcher;
private final ExtensionClient client;
@ -65,19 +64,18 @@ public class ControllerBuilder {
return this;
}
public ControllerBuilder onAddPredicate(Predicate<Extension> onAddPredicate) {
this.onAddPredicate = onAddPredicate;
public ControllerBuilder onAddMatcher(ExtensionMatcher onAddMatcher) {
this.onAddMatcher = onAddMatcher;
return this;
}
public ControllerBuilder onDeletePredicate(Predicate<Extension> onDeletePredicate) {
this.onDeletePredicate = onDeletePredicate;
public ControllerBuilder onDeleteMatcher(ExtensionMatcher onDeleteMatcher) {
this.onDeleteMatcher = onDeleteMatcher;
return this;
}
public ControllerBuilder onUpdatePredicate(
BiPredicate<Extension, Extension> onUpdatePredicate) {
this.onUpdatePredicate = onUpdatePredicate;
public ControllerBuilder onUpdateMatcher(ExtensionMatcher extensionMatcher) {
this.onUpdateMatcher = extensionMatcher;
return this;
}
@ -107,18 +105,18 @@ public class ControllerBuilder {
Assert.notNull(reconciler, "Reconciler must not be null");
var queue = new DefaultQueue<Request>(nowSupplier, minDelay);
var predicates = new WatcherPredicates.Builder()
.withGroupVersionKind(extension.groupVersionKind())
.onAddPredicate(onAddPredicate)
.onUpdatePredicate(onUpdatePredicate)
.onDeletePredicate(onDeletePredicate)
var extensionMatchers = WatcherExtensionMatchers.builder(client,
extension.groupVersionKind())
.onAddMatcher(onAddMatcher)
.onUpdateMatcher(onUpdateMatcher)
.onDeleteMatcher(onDeleteMatcher)
.build();
var watcher = new ExtensionWatcher(queue, predicates);
var watcher = new ExtensionWatcher(queue, extensionMatchers);
var synchronizer = new RequestSynchronizer(syncAllOnStart,
client,
extension,
watcher,
predicates.onAddPredicate());
extensionMatchers.onAddMatcher());
return new DefaultController<>(name, reconciler, queue, synchronizer, minDelay, maxDelay,
workerCount);
}

View File

@ -2,7 +2,7 @@ package run.halo.app.extension.controller;
import run.halo.app.extension.Extension;
import run.halo.app.extension.Watcher;
import run.halo.app.extension.WatcherPredicates;
import run.halo.app.extension.WatcherExtensionMatchers;
import run.halo.app.extension.controller.Reconciler.Request;
public class ExtensionWatcher implements Watcher {
@ -12,16 +12,25 @@ public class ExtensionWatcher implements Watcher {
private volatile boolean disposed = false;
private Runnable disposeHook;
private final WatcherPredicates predicates;
public ExtensionWatcher(RequestQueue<Request> queue, WatcherPredicates predicates) {
private final WatcherExtensionMatchers matchers;
public ExtensionWatcher(RequestQueue<Request> queue, WatcherExtensionMatchers matchers) {
this.queue = queue;
this.predicates = predicates;
this.matchers = matchers;
}
@Override
public void onAdd(Request request) {
if (isDisposed()) {
return;
}
queue.addImmediately(request);
}
@Override
public void onAdd(Extension extension) {
if (isDisposed() || !predicates.onAddPredicate().test(extension)) {
if (isDisposed() || !matchers.onAddMatcher().match(extension)) {
return;
}
// TODO filter the event
@ -30,7 +39,7 @@ public class ExtensionWatcher implements Watcher {
@Override
public void onUpdate(Extension oldExtension, Extension newExtension) {
if (isDisposed() || !predicates.onUpdatePredicate().test(oldExtension, newExtension)) {
if (isDisposed() || !matchers.onUpdateMatcher().match(newExtension)) {
return;
}
// TODO filter the event
@ -39,7 +48,7 @@ public class ExtensionWatcher implements Watcher {
@Override
public void onDelete(Extension extension) {
if (isDisposed() || !predicates.onDeletePredicate().test(extension)) {
if (isDisposed() || !matchers.onDeleteMatcher().match(extension)) {
return;
}
// TODO filter the event

View File

@ -1,42 +1,47 @@
package run.halo.app.extension.controller;
import java.util.function.Predicate;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionMatcher;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.Watcher;
import run.halo.app.extension.controller.Reconciler.Request;
import run.halo.app.extension.index.IndexedQueryEngine;
@Slf4j
public class RequestSynchronizer implements Synchronizer<Request> {
private final ExtensionClient client;
private final Class<? extends Extension> type;
private final GroupVersionKind type;
private final boolean syncAllOnStart;
private volatile boolean disposed = false;
private volatile boolean started = false;
private final IndexedQueryEngine indexedQueryEngine;
private final Watcher watcher;
private final Predicate<Extension> listPredicate;
private final ExtensionMatcher listMatcher;
@Getter
private volatile boolean started = false;
public RequestSynchronizer(boolean syncAllOnStart,
ExtensionClient client,
Extension extension,
Watcher watcher,
Predicate<Extension> listPredicate) {
ExtensionMatcher listMatcher) {
this.syncAllOnStart = syncAllOnStart;
this.client = client;
this.type = extension.getClass();
this.type = extension.groupVersionKind();
this.watcher = watcher;
if (listPredicate == null) {
listPredicate = e -> true;
}
this.listPredicate = listPredicate;
this.indexedQueryEngine = client.indexedQueryEngine();
this.listMatcher = listMatcher;
}
@Override
@ -48,17 +53,18 @@ public class RequestSynchronizer implements Synchronizer<Request> {
started = true;
if (syncAllOnStart) {
client.list(type, listPredicate::test, null)
.forEach(watcher::onAdd);
var listOptions = new ListOptions();
if (listMatcher != null) {
listOptions.setFieldSelector(listMatcher.getFieldSelector());
listOptions.setLabelSelector(listMatcher.getLabelSelector());
}
indexedQueryEngine.retrieveAll(type, listOptions)
.forEach(name -> watcher.onAdd(new Request(name)));
}
client.watch(this.watcher);
log.info("Started request({}) synchronizer.", type);
}
public boolean isStarted() {
return started;
}
@Override
public void dispose() {
disposed = true;

View File

@ -0,0 +1,33 @@
package run.halo.app.extension.index;
import lombok.EqualsAndHashCode;
import org.springframework.util.Assert;
import run.halo.app.extension.Extension;
import run.halo.app.extension.GVK;
@EqualsAndHashCode
public abstract class AbstractIndexAttribute<E extends Extension> implements IndexAttribute {
private final Class<E> objectType;
/**
* Creates a new {@link AbstractIndexAttribute} for the given object type.
*
* @param objectType must not be {@literal null}.
*/
public AbstractIndexAttribute(Class<E> objectType) {
Assert.notNull(objectType, "Object type must not be null");
Assert.state(isValidExtension(objectType),
"Invalid extension type, make sure you have annotated it with @" + GVK.class
.getSimpleName());
this.objectType = objectType;
}
@Override
public Class<E> getObjectType() {
return this.objectType;
}
boolean isValidExtension(Class<? extends Extension> type) {
return type.getAnnotation(GVK.class) != null;
}
}

View File

@ -0,0 +1,54 @@
package run.halo.app.extension.index;
import java.util.Set;
import java.util.function.Function;
import lombok.EqualsAndHashCode;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
import run.halo.app.extension.Extension;
import run.halo.app.extension.Unstructured;
@EqualsAndHashCode(callSuper = true)
public class FunctionalIndexAttribute<E extends Extension>
extends AbstractIndexAttribute<E> {
@EqualsAndHashCode.Exclude
private final Function<E, String> valueFunc;
/**
* Creates a new {@link FunctionalIndexAttribute} for the given object type and value function.
*
* @param objectType must not be {@literal null}.
* @param valueFunc value function must not be {@literal null}.
*/
public FunctionalIndexAttribute(Class<E> objectType,
Function<E, String> valueFunc) {
super(objectType);
Assert.notNull(valueFunc, "Value function must not be null");
this.valueFunc = valueFunc;
}
@Override
public Set<String> getValues(Extension object) {
var value = getValue(object);
return value == null ? Set.of() : Set.of(value);
}
/**
* Gets the value for the given object.
*
* @param object the object to get the value for.
* @return returns the value for the given object.
*/
@Nullable
public String getValue(Extension object) {
if (object instanceof Unstructured unstructured) {
var ext = Unstructured.OBJECT_MAPPER.convertValue(unstructured, getObjectType());
return valueFunc.apply(ext);
}
if (getObjectType().isInstance(object)) {
return valueFunc.apply(getObjectType().cast(object));
}
throw new IllegalArgumentException("Object type does not match");
}
}

View File

@ -0,0 +1,46 @@
package run.halo.app.extension.index;
import java.util.Set;
import java.util.function.Function;
import lombok.EqualsAndHashCode;
import org.springframework.util.Assert;
import run.halo.app.extension.Extension;
import run.halo.app.extension.Unstructured;
@EqualsAndHashCode(callSuper = true)
public class FunctionalMultiValueIndexAttribute<E extends Extension>
extends AbstractIndexAttribute<E> {
@EqualsAndHashCode.Exclude
private final Function<E, Set<String>> valueFunc;
/**
* Creates a new {@link FunctionalIndexAttribute} for the given object type and value function.
*
* @param objectType object type must not be {@literal null}.
* @param valueFunc value function must not be {@literal null}.
*/
public FunctionalMultiValueIndexAttribute(Class<E> objectType,
Function<E, Set<String>> valueFunc) {
super(objectType);
Assert.notNull(valueFunc, "Value function must not be null");
this.valueFunc = valueFunc;
}
@Override
public Set<String> getValues(Extension object) {
if (object instanceof Unstructured unstructured) {
var ext = Unstructured.OBJECT_MAPPER.convertValue(unstructured, getObjectType());
return getNonNullValues(ext);
}
if (getObjectType().isInstance(object)) {
return getNonNullValues(getObjectType().cast(object));
}
throw new IllegalArgumentException("Object type does not match");
}
private Set<String> getNonNullValues(E object) {
var values = valueFunc.apply(object);
return values == null ? Set.of() : values;
}
}

View File

@ -0,0 +1,23 @@
package run.halo.app.extension.index;
import java.util.Set;
import run.halo.app.extension.Extension;
public interface IndexAttribute {
/**
* Specify this class is belonged to which extension.
*
* @return the extension class.
*/
Class<? extends Extension> getObjectType();
/**
* Get the value of the attribute.
*
* @param object the object to get value from.
* @param <E> the type of the object.
* @return the value of the attribute must not be null.
*/
<E extends Extension> Set<String> getValues(E object);
}

View File

@ -0,0 +1,20 @@
package run.halo.app.extension.index;
import java.util.Set;
import java.util.function.Function;
import lombok.experimental.UtilityClass;
import run.halo.app.extension.Extension;
@UtilityClass
public class IndexAttributeFactory {
public static <E extends Extension> IndexAttribute simpleAttribute(Class<E> type,
Function<E, String> valueFunc) {
return new FunctionalIndexAttribute<>(type, valueFunc);
}
public static <E extends Extension> IndexAttribute multiValueAttribute(Class<E> type,
Function<E, Set<String>> valueFunc) {
return new FunctionalMultiValueIndexAttribute<>(type, valueFunc);
}
}

View File

@ -0,0 +1,39 @@
package run.halo.app.extension.index;
import com.google.common.base.Objects;
import lombok.Data;
import lombok.experimental.Accessors;
@Data
@Accessors(chain = true)
public class IndexSpec {
private String name;
private IndexAttribute indexFunc;
private OrderType order;
private boolean unique;
public enum OrderType {
ASC,
DESC
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
IndexSpec indexSpec = (IndexSpec) o;
return Objects.equal(name, indexSpec.name);
}
@Override
public int hashCode() {
return Objects.hashCode(name);
}
}

View File

@ -0,0 +1,54 @@
package run.halo.app.extension.index;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.Scheme;
/**
* <p>{@link IndexSpecRegistry} is a registry for {@link IndexSpecs} to manage {@link IndexSpecs}
* for different {@link Extension}.</p>
*
* @author guqing
* @since 2.12.0
*/
public interface IndexSpecRegistry {
/**
* <p>Create a new {@link IndexSpecs} for the given {@link Scheme}.</p>
* <p>The returned {@link IndexSpecs} is always includes some default {@link IndexSpec} that
* does not need to be registered again:</p>
* <ul>
* <li>{@link Metadata#getName()} for unique primary index spec named metadata_name</li>
* <li>{@link Metadata#getCreationTimestamp()} for creation_timestamp index spec</li>
* <li>{@link Metadata#getDeletionTimestamp()} for deletion_timestamp index spec</li>
* <li>{@link Metadata#getLabels()} for labels index spec</li>
* </ul>
*
* @param scheme must not be {@literal null}.
* @return the {@link IndexSpecs} for the given {@link Scheme}.
*/
IndexSpecs indexFor(Scheme scheme);
/**
* Get {@link IndexSpecs} for the given {@link Scheme} type registered before.
*
* @param scheme must not be {@literal null}.
* @return the {@link IndexSpecs} for the given {@link Scheme}.
* @throws IllegalArgumentException if no {@link IndexSpecs} found for the given
* {@link Scheme}.
*/
IndexSpecs getIndexSpecs(Scheme scheme);
boolean contains(Scheme scheme);
void removeIndexSpecs(Scheme scheme);
/**
* Get key space for an extension type.
*
* @param scheme is a scheme of an Extension.
* @return key space(never null)
*/
@NonNull
String getKeySpace(Scheme scheme);
}

View File

@ -0,0 +1,54 @@
package run.halo.app.extension.index;
import java.util.List;
import org.springframework.lang.Nullable;
/**
* An interface that defines a collection of {@link IndexSpec}, and provides methods to add,
* remove, and get {@link IndexSpec}.
*
* @author guqing
* @since 2.12.0
*/
public interface IndexSpecs {
/**
* Add a new {@link IndexSpec} to the collection.
*
* @param indexSpec the index spec to add.
* @throws IllegalArgumentException if the index spec with the same name already exists or
* the index spec is invalid
*/
void add(IndexSpec indexSpec);
/**
* Get all {@link IndexSpec} in the collection.
*
* @return all index specs
*/
List<IndexSpec> getIndexSpecs();
/**
* Get the {@link IndexSpec} with the given name.
*
* @param indexName the name of the index spec to get.
* @return the index spec with the given name, or {@code null} if not found.
*/
@Nullable
IndexSpec getIndexSpec(String indexName);
/**
* Check if the collection contains the {@link IndexSpec} with the given name.
*
* @param indexName the name of the index spec to check.
* @return {@code true} if the collection contains the index spec with the given name,
*/
boolean contains(String indexName);
/**
* Remove the {@link IndexSpec} with the given name.
*
* @param name the name of the index spec to remove.
*/
void remove(String name);
}

View File

@ -0,0 +1,42 @@
package run.halo.app.extension.index;
import java.util.List;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.PageRequest;
/**
* <p>An interface for querying indexed object records from the index store.</p>
* <p>It provides a way to retrieve the object records by the given {@link GroupVersionKind} and
* {@link ListOptions}, the final result will be ordered by the index what {@link ListOptions}
* used and specified by the {@link PageRequest#getSort()}.</p>
*
* @author guqing
* @since 2.12.0
*/
public interface IndexedQueryEngine {
/**
* Page retrieve the object records by the given {@link GroupVersionKind} and
* {@link ListOptions}.
*
* @param type the type of the object must exist in
* {@link run.halo.app.extension.SchemeManager}.
* @param options the list options to use for retrieving the object records.
* @param page which page to retrieve and how large the page should be.
* @return a collection of {@link Metadata#getName()} for the given page.
*/
ListResult<String> retrieve(GroupVersionKind type, ListOptions options, PageRequest page);
/**
* Retrieve all the object records by the given {@link GroupVersionKind} and
* {@link ListOptions}.
*
* @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager}
* @param options the list options to use for retrieving the object records
* @return a collection of {@link Metadata#getName()}
*/
List<String> retrieveAll(GroupVersionKind type, ListOptions options);
}

View File

@ -0,0 +1,15 @@
package run.halo.app.extension.index.query;
import java.util.NavigableSet;
public class All extends SimpleQuery {
public All(String fieldName) {
super(fieldName, null);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
return indexView.getAllIdsForField(fieldName);
}
}

View File

@ -0,0 +1,36 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.NavigableSet;
public class And extends LogicalQuery {
/**
* Creates a new And query with the given child queries.
*
* @param childQueries The child queries
*/
public And(Collection<Query> childQueries) {
super(childQueries);
if (this.size < 2) {
throw new IllegalStateException(
"An 'And' query cannot have fewer than 2 child queries, " + childQueries.size()
+ " were supplied");
}
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
NavigableSet<String> resultSet = null;
for (Query query : childQueries) {
NavigableSet<String> currentResult = query.matches(indexView);
if (resultSet == null) {
resultSet = Sets.newTreeSet(currentResult);
} else {
resultSet.retainAll(currentResult);
}
}
return resultSet == null ? Sets.newTreeSet() : resultSet;
}
}

View File

@ -0,0 +1,35 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class Between extends SimpleQuery {
private final String lowerValue;
private final boolean lowerInclusive;
private final String upperValue;
private final boolean upperInclusive;
public Between(String fieldName, String lowerValue, boolean lowerInclusive,
String upperValue, boolean upperInclusive) {
// value and isFieldRef are not used in Between
super(fieldName, null, false);
this.lowerValue = lowerValue;
this.lowerInclusive = lowerInclusive;
this.upperValue = upperValue;
this.upperInclusive = upperInclusive;
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
NavigableSet<String> allValues = indexView.getAllValuesForField(fieldName);
// get all values in the specified range
var subSet = allValues.subSet(lowerValue, lowerInclusive, upperValue, upperInclusive);
var resultSet = Sets.<String>newTreeSet();
for (String val : subSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
}
}

View File

@ -0,0 +1,30 @@
package run.halo.app.extension.index.query;
import java.util.NavigableSet;
public class EqualQuery extends SimpleQuery {
public EqualQuery(String fieldName, String value) {
super(fieldName, value);
}
public EqualQuery(String fieldName, String value, boolean isFieldRef) {
super(fieldName, value, isFieldRef);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
}
return resultSetForExactValue(indexView);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueEqual(fieldName, value);
}
private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) {
return indexView.getIdsForFieldValue(fieldName, value);
}
}

View File

@ -0,0 +1,41 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class GreaterThanQuery extends SimpleQuery {
private final boolean orEqual;
public GreaterThanQuery(String fieldName, String value, boolean orEqual) {
this(fieldName, value, orEqual, false);
}
public GreaterThanQuery(String fieldName, String value, boolean orEqual, boolean isFieldRef) {
super(fieldName, value, isFieldRef);
this.orEqual = orEqual;
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
}
return resultSetForExtractValue(indexView);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueGreaterThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForExtractValue(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
NavigableSet<String> tailSet =
orEqual ? allValues.tailSet(value, true) : allValues.tailSet(value, false);
for (String val : tailSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
}
}

View File

@ -0,0 +1,23 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
import java.util.Set;
public class InQuery extends SimpleQuery {
private final Set<String> values;
public InQuery(String columnName, Set<String> values) {
super(columnName, null);
this.values = values;
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
NavigableSet<String> resultSet = Sets.newTreeSet();
for (String val : values) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
}
}

View File

@ -0,0 +1,41 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class LessThanQuery extends SimpleQuery {
private final boolean orEqual;
public LessThanQuery(String fieldName, String value, boolean orEqual) {
this(fieldName, value, orEqual, false);
}
public LessThanQuery(String fieldName, String value, boolean orEqual, boolean isFieldRef) {
super(fieldName, value, isFieldRef);
this.orEqual = orEqual;
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
}
return resultSetForExactValue(indexView);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueLessThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
var headSet = orEqual ? allValues.headSet(value, true)
: allValues.headSet(value, false);
for (String val : headSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
}
}

View File

@ -0,0 +1,31 @@
package run.halo.app.extension.index.query;
import java.util.Collection;
import java.util.Objects;
public abstract class LogicalQuery implements Query {
protected final Collection<Query> childQueries;
protected final int size;
/**
* Creates a new logical query with the given child queries.
*
* @param childQueries with the given child queries.
*/
public LogicalQuery(Collection<Query> childQueries) {
Objects.requireNonNull(childQueries,
"The child queries supplied to a logical query cannot be null");
for (Query query : childQueries) {
if (!isValid(query)) {
throw new IllegalStateException("Unexpected type of query: " + (query == null ? null
: query + ", " + query.getClass()));
}
}
this.size = childQueries.size();
this.childQueries = childQueries;
}
boolean isValid(Query query) {
return query instanceof LogicalQuery || query instanceof SimpleQuery;
}
}

View File

@ -0,0 +1,31 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class NotEqual extends SimpleQuery {
private final EqualQuery equalQuery;
public NotEqual(String fieldName, String value) {
this(fieldName, value, false);
}
public NotEqual(String fieldName, String value, boolean isFieldRef) {
super(fieldName, value, isFieldRef);
this.equalQuery = new EqualQuery(fieldName, value, isFieldRef);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var names = equalQuery.matches(indexView);
var allNames = indexView.getAllIdsForField(fieldName);
var resultSet = Sets.<String>newTreeSet();
for (String name : allNames) {
if (!names.contains(name)) {
resultSet.add(name);
}
}
return resultSet;
}
}

View File

@ -0,0 +1,21 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.NavigableSet;
public class Or extends LogicalQuery {
public Or(Collection<Query> childQueries) {
super(childQueries);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
for (Query query : childQueries) {
resultSet.addAll(query.matches(indexView));
}
return resultSet;
}
}

View File

@ -0,0 +1,22 @@
package run.halo.app.extension.index.query;
import java.util.NavigableSet;
import run.halo.app.extension.Metadata;
/**
* A {@link Query} is used to match {@link QueryIndexView} objects.
*
* @author guqing
* @since 2.12.0
*/
public interface Query {
/**
* Matches the given {@link QueryIndexView} and returns the matched object names see
* {@link Metadata#getName()}.
*
* @param indexView the {@link QueryIndexView} to match.
* @return the matched object names ordered by natural order.
*/
NavigableSet<String> matches(QueryIndexView indexView);
}

View File

@ -0,0 +1,168 @@
package run.halo.app.extension.index.query;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import lombok.experimental.UtilityClass;
import org.springframework.util.Assert;
@UtilityClass
public class QueryFactory {
public static Query all() {
return new All("metadata.name");
}
public static Query all(String fieldName) {
return new All(fieldName);
}
public static Query notEqual(String fieldName, String attributeValue) {
return new NotEqual(fieldName, attributeValue);
}
public static Query notEqualOtherField(String fieldName, String otherFieldName) {
return new NotEqual(fieldName, otherFieldName, true);
}
public static Query equal(String fieldName, String attributeValue) {
return new EqualQuery(fieldName, attributeValue);
}
public static Query equalOtherField(String fieldName, String otherFieldName) {
return new EqualQuery(fieldName, otherFieldName, true);
}
public static Query lessThanOtherField(String fieldName, String otherFieldName) {
return new LessThanQuery(fieldName, otherFieldName, false, true);
}
public static Query lessThanOrEqualOtherField(String fieldName, String otherFieldName) {
return new LessThanQuery(fieldName, otherFieldName, true, true);
}
public static Query lessThan(String fieldName, String attributeValue) {
return new LessThanQuery(fieldName, attributeValue, false);
}
public static Query lessThanOrEqual(String fieldName, String attributeValue) {
return new LessThanQuery(fieldName, attributeValue, true);
}
public static Query greaterThan(String fieldName, String attributeValue) {
return new GreaterThanQuery(fieldName, attributeValue, false);
}
public static Query greaterThanOrEqual(String fieldName, String attributeValue) {
return new GreaterThanQuery(fieldName, attributeValue, true);
}
public static Query greaterThanOtherField(String fieldName, String otherFieldName) {
return new GreaterThanQuery(fieldName, otherFieldName, false, true);
}
public static Query greaterThanOrEqualOtherField(String fieldName,
String otherFieldName) {
return new GreaterThanQuery(fieldName, otherFieldName, true, true);
}
public static Query in(String fieldName, String... attributeValues) {
return in(fieldName, Set.of(attributeValues));
}
public static Query in(String fieldName, Collection<String> values) {
Assert.notNull(values, "Values must not be null");
if (values.size() == 1) {
String singleValue = values.iterator().next();
return equal(fieldName, singleValue);
}
// Copy the values into a Set if necessary...
var valueSet = (values instanceof Set ? (Set<String>) values
: new HashSet<>(values));
return new InQuery(fieldName, valueSet);
}
public static Query and(Collection<Query> queries) {
Assert.notEmpty(queries, "Queries must not be empty");
if (queries.size() == 1) {
return queries.iterator().next();
}
return new And(queries);
}
public static And and(Query query1, Query query2) {
Collection<Query> queries = Arrays.asList(query1, query2);
return new And(queries);
}
public static Query and(Query query1, Query query2, Query... additionalQueries) {
var queries = new ArrayList<Query>(2 + additionalQueries.length);
queries.add(query1);
queries.add(query2);
Collections.addAll(queries, additionalQueries);
return new And(queries);
}
public static Query and(Query query1, Query query2, Collection<Query> additionalQueries) {
var queries = new ArrayList<Query>(2 + additionalQueries.size());
queries.add(query1);
queries.add(query2);
queries.addAll(additionalQueries);
return new And(queries);
}
public static Query or(Query query1, Query query2) {
Collection<Query> queries = Arrays.asList(query1, query2);
return new Or(queries);
}
public static Query or(Query query1, Query query2, Query... additionalQueries) {
var queries = new ArrayList<Query>(2 + additionalQueries.length);
queries.add(query1);
queries.add(query2);
Collections.addAll(queries, additionalQueries);
return new Or(queries);
}
public static Query or(Query query1, Query query2, Collection<Query> additionalQueries) {
var queries = new ArrayList<Query>(2 + additionalQueries.size());
queries.add(query1);
queries.add(query2);
queries.addAll(additionalQueries);
return new Or(queries);
}
public static Query betweenLowerExclusive(String fieldName, String lowerValue,
String upperValue) {
return new Between(fieldName, lowerValue, false, upperValue, true);
}
public static Query betweenUpperExclusive(String fieldName, String lowerValue,
String upperValue) {
return new Between(fieldName, lowerValue, true, upperValue, false);
}
public static Query betweenExclusive(String fieldName, String lowerValue,
String upperValue) {
return new Between(fieldName, lowerValue, false, upperValue, false);
}
public static Query between(String fieldName, String lowerValue, String upperValue) {
return new Between(fieldName, lowerValue, true, upperValue, true);
}
public static Query startsWith(String fieldName, String value) {
return new StringStartsWith(fieldName, value);
}
public static Query endsWith(String fieldName, String value) {
return new StringEndsWith(fieldName, value);
}
public static Query contains(String fieldName, String value) {
return new StringContains(fieldName, value);
}
}

View File

@ -0,0 +1,57 @@
package run.halo.app.extension.index.query;
import java.util.NavigableSet;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.index.IndexSpec;
/**
* <p>A view of an index entries that can be queried.</p>
* <p>Explanation of naming:</p>
* <ul>
* <li>fieldName: a field of an index, usually {@link IndexSpec#getName()}</li>
* <li>fieldValue: a value of a field, e.g. a value of a field "name" could be "foo"</li>
* <li>id: the id of an object pointing to object position, see {@link Metadata#getName()}</li>
* </ul>
*
* @author guqing
* @since 2.12.0
*/
public interface QueryIndexView {
/**
* Gets all object ids for a given field name and field value.
*
* @param fieldName the field name
* @param fieldValue the field value
* @return all indexed object ids associated with the given field name and field value
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue);
/**
* Gets all field values for a given field name.
*
* @param fieldName the field name
* @return all field values for the given field name
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getAllValuesForField(String fieldName);
/**
* Gets all object ids for a given field name.
*
* @param fieldName the field name
* @return all indexed object ids for the given field name
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getAllIdsForField(String fieldName);
NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2);
NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1, String fieldName2,
boolean orEqual);
NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2,
boolean orEqual);
void removeAllFieldValuesByIdNotIn(NavigableSet<String> ids);
}

View File

@ -0,0 +1,194 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.Multimaps;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.NavigableSet;
import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
/**
* A default implementation for {@link QueryIndexView}.
*
* @author guqing
* @since 2.12.0
*/
public class QueryIndexViewImpl implements QueryIndexView {
private final Lock lock = new ReentrantLock();
private final Map<String, SetMultimap<String, String>> orderedMatches;
/**
* Creates a new {@link QueryIndexViewImpl} for the given {@link Map} of index entries.
*
* @param indexEntries index entries from indexer to create the view for.
*/
public QueryIndexViewImpl(Map<String, Collection<Map.Entry<String, String>>> indexEntries) {
this.orderedMatches = new HashMap<>();
for (var entry : indexEntries.entrySet()) {
// do not use stream collect here as it is slower
this.orderedMatches.put(entry.getKey(), createSetMultiMap(entry.getValue()));
}
}
@Override
public NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
SetMultimap<String, String> fieldMap = orderedMatches.get(fieldName);
return fieldMap != null ? new TreeSet<>(fieldMap.get(fieldValue)) : emptySet();
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> getAllValuesForField(String fieldName) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
SetMultimap<String, String> fieldMap = orderedMatches.get(fieldName);
return fieldMap != null ? new TreeSet<>(fieldMap.keySet()) : emptySet();
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> getAllIdsForField(String fieldName) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
SetMultimap<String, String> fieldMap = orderedMatches.get(fieldName);
return fieldMap != null ? new TreeSet<>(fieldMap.values()) : emptySet();
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2) {
lock.lock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
var index1 = orderedMatches.get(fieldName1);
var index2 = orderedMatches.get(fieldName2);
var idFieldValuesForIndex2Map =
Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build());
var result = Sets.<String>newTreeSet();
for (Map.Entry<String, String> entryForIndex1 : index1.entries()) {
var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue());
for (String item : fieldValues) {
if (entryForIndex1.getKey().equals(item)) {
result.add(entryForIndex1.getValue());
}
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1,
String fieldName2, boolean orEqual) {
lock.lock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
var index1 = orderedMatches.get(fieldName1);
var index2 = orderedMatches.get(fieldName2);
var idFieldValuesForIndex2Map =
Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build());
var result = Sets.<String>newTreeSet();
for (Map.Entry<String, String> entryForIndex1 : index1.entries()) {
var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue());
for (String item : fieldValues) {
int compare = entryForIndex1.getKey().compareTo(item);
if (orEqual ? compare >= 0 : compare > 0) {
result.add(entryForIndex1.getValue());
}
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2,
boolean orEqual) {
lock.lock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
SetMultimap<String, String> index1 = orderedMatches.get(fieldName1);
SetMultimap<String, String> index2 = orderedMatches.get(fieldName2);
var idFieldValuesForIndex2Map =
Multimaps.invertFrom(index2, MultimapBuilder.treeKeys().treeSetValues().build());
var result = Sets.<String>newTreeSet();
for (Map.Entry<String, String> entryForIndex1 : index1.entries()) {
var fieldValues = idFieldValuesForIndex2Map.get(entryForIndex1.getValue());
for (String item : fieldValues) {
int compare = entryForIndex1.getKey().compareTo(item);
if (orEqual ? compare <= 0 : compare < 0) {
result.add(entryForIndex1.getValue());
}
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public void removeAllFieldValuesByIdNotIn(NavigableSet<String> ids) {
lock.lock();
try {
for (var fieldNameValuesEntry : orderedMatches.entrySet()) {
SetMultimap<String, String> indicates = fieldNameValuesEntry.getValue();
indicates.entries().removeIf(entry -> !ids.contains(entry.getValue()));
}
} finally {
lock.unlock();
}
}
private void checkFieldNameIndexed(String fieldName) {
if (!orderedMatches.containsKey(fieldName)) {
throw new IllegalArgumentException("Field name " + fieldName
+ " is not indexed, please ensure it added to the index spec before querying");
}
}
private TreeSet<String> emptySet() {
return new TreeSet<>();
}
private SetMultimap<String, String> createSetMultiMap(
Collection<Map.Entry<String, String>> entries) {
SetMultimap<String, String> multiMap = MultimapBuilder.hashKeys()
.hashSetValues()
.build();
for (Map.Entry<String, String> entry : entries) {
multiMap.put(entry.getKey(), entry.getValue());
}
return multiMap;
}
}

View File

@ -0,0 +1,33 @@
package run.halo.app.extension.index.query;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.Assert;
public abstract class SimpleQuery implements Query {
protected final String fieldName;
protected final String value;
/**
* <p>Whether the value if a field reference.</p>
* For example, {@code fieldName = "salary", value = "cost"} can lead to a query
* <pre>
* salary > cost
* </pre>
* means that we want to find all the records whose salary is greater than cost.
*
* @see EqualQuery
* @see GreaterThanQuery
* @see LessThanQuery
*/
protected final boolean isFieldRef;
protected SimpleQuery(String fieldName, String value) {
this(fieldName, value, false);
}
protected SimpleQuery(String fieldName, String value, boolean isFieldRef) {
Assert.isTrue(StringUtils.isNotBlank(fieldName), "fieldName cannot be blank.");
this.fieldName = fieldName;
this.value = value;
this.isFieldRef = isFieldRef;
}
}

View File

@ -0,0 +1,23 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
import org.apache.commons.lang3.StringUtils;
public class StringContains extends SimpleQuery {
public StringContains(String fieldName, String value) {
super(fieldName, value);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var fieldValues = indexView.getAllValuesForField(fieldName);
for (String val : fieldValues) {
if (StringUtils.containsIgnoreCase(val, value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
}
return resultSet;
}
}

View File

@ -0,0 +1,22 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class StringEndsWith extends SimpleQuery {
public StringEndsWith(String fieldName, String value) {
super(fieldName, value);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var fieldValues = indexView.getAllValuesForField(fieldName);
for (String val : fieldValues) {
if (val.endsWith(value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
}
return resultSet;
}
}

View File

@ -0,0 +1,23 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class StringStartsWith extends SimpleQuery {
public StringStartsWith(String fieldName, String value) {
super(fieldName, value);
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
for (String val : allValues) {
if (val.startsWith(value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
}
return resultSet;
}
}

View File

@ -3,6 +3,7 @@ package run.halo.app.extension.router;
import static run.halo.app.extension.Comparators.compareCreationTimestamp;
import static run.halo.app.extension.Comparators.compareName;
import static run.halo.app.extension.Comparators.nullsComparator;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToListOptions;
import static run.halo.app.extension.router.selector.SelectorUtil.labelAndFieldSelectorToPredicate;
import io.swagger.v3.oas.annotations.media.ArraySchema;
@ -17,6 +18,9 @@ import org.springframework.data.domain.Sort;
import org.springframework.web.server.ServerWebExchange;
import run.halo.app.core.extension.endpoint.SortResolver;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.PageRequestImpl;
public class SortableRequest extends IListRequest.QueryListRequest {
@ -48,6 +52,19 @@ public class SortableRequest extends IListRequest.QueryListRequest {
return labelAndFieldSelectorToPredicate(getLabelSelector(), getFieldSelector());
}
/**
* Build {@link ListOptions} from query params.
*
* @return a list options.
*/
public ListOptions toListOptions() {
return labelAndFieldSelectorToListOptions(getLabelSelector(), getFieldSelector());
}
public PageRequest toPageRequest() {
return PageRequestImpl.of(getPage(), getSize(), getSort());
}
/**
* Build comparator from sort.
*

View File

@ -0,0 +1,73 @@
package run.halo.app.extension.router.selector;
import java.util.function.Function;
import java.util.function.Predicate;
public class EqualityMatcher implements SelectorMatcher {
private final Operator operator;
private final String key;
private final String value;
EqualityMatcher(String key, Operator operator, String value) {
this.key = key;
this.operator = operator;
this.value = value;
}
/**
* The "equal" matcher. Matches a label if the label is present and equal.
*
* @param key the matching label key
* @param value the matching label value
* @return the equality matcher
*/
public static EqualityMatcher equal(String key, String value) {
return new EqualityMatcher(key, Operator.EQUAL, value);
}
/**
* The "not equal" matcher. Matches a label if the label is not present or not equal.
*
* @param key the matching label key
* @param value the matching label value
* @return the equality matcher
*/
public static EqualityMatcher notEqual(String key, String value) {
return new EqualityMatcher(key, Operator.NOT_EQUAL, value);
}
@Override
public String toString() {
return key
+ " "
+ operator.name().toLowerCase()
+ " "
+ value;
}
@Override
public boolean test(String s) {
return operator.with(value).test(s);
}
@Override
public String getKey() {
return key;
}
protected enum Operator {
EQUAL(arg -> arg::equals),
DOUBLE_EQUAL(arg -> arg::equals),
NOT_EQUAL(arg -> v -> !arg.equals(v));
private final Function<String, Predicate<String>> matcherFunc;
Operator(Function<String, Predicate<String>> matcherFunc) {
this.matcherFunc = matcherFunc;
}
Predicate<String> with(String value) {
return matcherFunc.apply(value);
}
}
}

View File

@ -5,6 +5,7 @@ import org.springframework.core.convert.converter.Converter;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
@Deprecated(since = "2.12.0")
public class FieldCriteriaPredicateConverter<E extends Extension>
implements Converter<SelectorCriteria, Predicate<E>> {

View File

@ -0,0 +1,15 @@
package run.halo.app.extension.router.selector;
import java.util.Objects;
import run.halo.app.extension.index.query.Query;
import run.halo.app.extension.index.query.QueryFactory;
public record FieldSelector(Query query) {
public FieldSelector(Query query) {
this.query = Objects.requireNonNullElseGet(query, QueryFactory::all);
}
public static FieldSelector of(Query query) {
return new FieldSelector(query);
}
}

View File

@ -0,0 +1,45 @@
package run.halo.app.extension.router.selector;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
import java.util.Set;
import org.springframework.core.convert.converter.Converter;
import org.springframework.lang.NonNull;
import org.springframework.util.CollectionUtils;
import run.halo.app.extension.index.query.Query;
import run.halo.app.extension.index.query.QueryFactory;
public class FieldSelectorConverter implements Converter<SelectorCriteria, Query> {
@NonNull
@Override
public Query convert(@NonNull SelectorCriteria criteria) {
var key = criteria.key();
// compatible with old field selector
if ("name".equals(key)) {
key = "metadata.name";
}
switch (criteria.operator()) {
case Equals -> {
return QueryFactory.equal(key, getSingleValue(criteria));
}
case NotEquals -> {
return QueryFactory.notEqual(key, getSingleValue(criteria));
}
// compatible with old field selector
case IN -> {
var valueArr = defaultIfNull(criteria.values(), Set.<String>of());
return QueryFactory.in(key, valueArr);
}
default -> throw new IllegalArgumentException(
"Unsupported operator: " + criteria.operator());
}
}
String getSingleValue(SelectorCriteria criteria) {
if (CollectionUtils.isEmpty(criteria.values())) {
return null;
}
return criteria.values().iterator().next();
}
}

View File

@ -5,6 +5,7 @@ import org.springframework.core.convert.converter.Converter;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
@Deprecated(since = "2.12.0")
public class LabelCriteriaPredicateConverter<E extends Extension>
implements Converter<SelectorCriteria, Predicate<E>> {

View File

@ -0,0 +1,73 @@
package run.halo.app.extension.router.selector;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import lombok.Data;
import lombok.experimental.Accessors;
import org.springframework.lang.NonNull;
import org.springframework.util.Assert;
@Data
@Accessors(chain = true)
public class LabelSelector implements Predicate<Map<String, String>> {
private List<SelectorMatcher> matchers;
@Override
public boolean test(@NonNull Map<String, String> labels) {
Assert.notNull(labels, "Labels must not be null");
if (matchers == null || matchers.isEmpty()) {
return true;
}
return matchers.stream()
.allMatch(matcher -> matcher.test(labels.get(matcher.getKey())));
}
public static LabelSelectorBuilder builder() {
return new LabelSelectorBuilder();
}
public static class LabelSelectorBuilder {
private final List<SelectorMatcher> matchers = new ArrayList<>();
public LabelSelectorBuilder eq(String key, String value) {
matchers.add(EqualityMatcher.equal(key, value));
return this;
}
public LabelSelectorBuilder notEq(String key, String value) {
matchers.add(EqualityMatcher.notEqual(key, value));
return this;
}
public LabelSelectorBuilder in(String key, String... values) {
matchers.add(SetMatcher.in(key, values));
return this;
}
public LabelSelectorBuilder notIn(String key, String... values) {
matchers.add(SetMatcher.notIn(key, values));
return this;
}
public LabelSelectorBuilder exists(String key) {
matchers.add(SetMatcher.exists(key));
return this;
}
public LabelSelectorBuilder notExists(String key) {
matchers.add(SetMatcher.notExists(key));
return this;
}
/**
* Build the label selector.
*/
public LabelSelector build() {
var labelSelector = new LabelSelector();
labelSelector.setMatchers(matchers);
return labelSelector;
}
}
}

View File

@ -0,0 +1,44 @@
package run.halo.app.extension.router.selector;
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
import java.util.Set;
import org.springframework.core.convert.converter.Converter;
import org.springframework.lang.NonNull;
import org.springframework.util.CollectionUtils;
public class LabelSelectorConverter implements Converter<SelectorCriteria, SelectorMatcher> {
@NonNull
@Override
public SelectorMatcher convert(@NonNull SelectorCriteria criteria) {
switch (criteria.operator()) {
case Equals -> {
return EqualityMatcher.equal(criteria.key(), getSingleValue(criteria));
}
case NotEquals -> {
return EqualityMatcher.notEqual(criteria.key(), getSingleValue(criteria));
}
case NotExist -> {
return SetMatcher.notExists(criteria.key());
}
case Exist -> {
return SetMatcher.exists(criteria.key());
}
case IN -> {
var valueArr =
defaultIfNull(criteria.values(), Set.<String>of()).toArray(new String[0]);
return SetMatcher.in(criteria.key(), valueArr);
}
default -> throw new IllegalArgumentException(
"Unsupported operator: " + criteria.operator());
}
}
String getSingleValue(SelectorCriteria criteria) {
if (CollectionUtils.isEmpty(criteria.values())) {
return null;
}
return criteria.values().iterator().next();
}
}

View File

@ -0,0 +1,14 @@
package run.halo.app.extension.router.selector;
public interface SelectorMatcher {
String getKey();
/**
* Returns true if a field value matches.
*
* @param s the field value
* @return the boolean
*/
boolean test(String s);
}

View File

@ -1,10 +1,14 @@
package run.halo.app.extension.router.selector;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Predicate;
import org.springframework.data.util.Predicates;
import org.springframework.web.server.ServerWebInputException;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.index.query.QueryFactory;
public final class SelectorUtil {
@ -58,4 +62,41 @@ public final class SelectorUtil {
return SelectorUtil.<E>labelSelectorsToPredicate(labelSelectors)
.and(fieldSelectorToPredicate(fieldSelectors));
}
/**
* Convert label and field selector expressions to {@link ListOptions}.
*
* @param labelSelectorTerms label selector expressions
* @param fieldSelectorTerms field selector expressions
* @return list options(never null)
*/
public static ListOptions labelAndFieldSelectorToListOptions(
List<String> labelSelectorTerms, List<String> fieldSelectorTerms) {
var selectorConverter = new SelectorConverter();
var labelConverter = new LabelSelectorConverter();
var labelMatchers = Optional.ofNullable(labelSelectorTerms)
.map(selectors -> selectors.stream()
.map(selectorConverter::convert)
.filter(Objects::nonNull)
.map(labelConverter::convert)
.toList())
.orElse(List.of());
var fieldConverter = new FieldSelectorConverter();
var fieldQuery = Optional.ofNullable(fieldSelectorTerms)
.map(selectors -> selectors.stream()
.map(selectorConverter::convert)
.filter(Objects::nonNull)
.map(fieldConverter::convert)
.toList()
)
.orElse(List.of());
var listOptions = new ListOptions();
listOptions.setLabelSelector(new LabelSelector().setMatchers(labelMatchers));
if (!fieldQuery.isEmpty()) {
listOptions.setFieldSelector(FieldSelector.of(QueryFactory.and(fieldQuery)));
}
return listOptions;
}
}

View File

@ -0,0 +1,69 @@
package run.halo.app.extension.router.selector;
import java.util.Arrays;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.Predicate;
public class SetMatcher implements SelectorMatcher {
private final SetMatcher.Operator operator;
private final String key;
private final String[] values;
SetMatcher(String key, SetMatcher.Operator operator) {
this(key, operator, new String[] {});
}
SetMatcher(String key, SetMatcher.Operator operator, String[] values) {
this.key = key;
this.operator = operator;
this.values = values;
}
public static SetMatcher in(String key, String... values) {
return new SetMatcher(key, Operator.IN, values);
}
public static SetMatcher notIn(String key, String... values) {
return new SetMatcher(key, Operator.NOT_IN, values);
}
public static SetMatcher exists(String key) {
return new SetMatcher(key, Operator.EXISTS);
}
public static SetMatcher notExists(String key) {
return new SetMatcher(key, Operator.NOT_EXISTS);
}
@Override
public String getKey() {
return key;
}
@Override
public boolean test(String s) {
return operator.with(values).test(s);
}
private enum Operator {
IN(values -> v -> contains(values, v)),
NOT_IN(values -> v -> !contains(values, v)),
EXISTS(values -> Objects::nonNull),
NOT_EXISTS(values -> Objects::isNull);
private final Function<String[], Predicate<String>> matcherFunc;
Operator(Function<String[], Predicate<String>> matcherFunc) {
this.matcherFunc = matcherFunc;
}
private static boolean contains(String[] strArray, String s) {
return Arrays.asList(strArray).contains(s);
}
Predicate<String> with(String... values) {
return matcherFunc.apply(values);
}
}
}

View File

@ -2,16 +2,18 @@ package run.halo.app.extension.controller;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.lenient;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.FakeExtension;
import run.halo.app.extension.index.IndexedQueryEngine;
@ExtendWith(MockitoExtension.class)
class ControllerBuilderTest {
@ -19,6 +21,14 @@ class ControllerBuilderTest {
@Mock
ExtensionClient client;
@Mock
IndexedQueryEngine indexedQueryEngine;
@BeforeEach
void setUp() {
lenient().when(client.indexedQueryEngine()).thenReturn(indexedQueryEngine);
}
@Test
void buildWithNullReconciler() {
assertThrows(IllegalArgumentException.class,
@ -51,9 +61,9 @@ class ControllerBuilderTest {
.syncAllOnStart(true)
.minDelay(Duration.ofMillis(5))
.maxDelay(Duration.ofSeconds(1000))
.onAddPredicate(Objects::nonNull)
.onUpdatePredicate(Objects::equals)
.onDeletePredicate(Objects::nonNull)
.onAddMatcher(null)
.onUpdateMatcher(null)
.onDeleteMatcher(null)
.build()
);
}

View File

@ -13,7 +13,11 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.extension.WatcherPredicates;
import run.halo.app.extension.DefaultExtensionMatcher;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.FakeExtension;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.WatcherExtensionMatchers;
import run.halo.app.extension.controller.Reconciler.Request;
@ExtendWith(MockitoExtension.class)
@ -23,17 +27,26 @@ class ExtensionWatcherTest {
RequestQueue<Request> queue;
@Mock
WatcherPredicates predicates;
ExtensionClient client;
@Mock
WatcherExtensionMatchers matchers;
@InjectMocks
ExtensionWatcher watcher;
private DefaultExtensionMatcher getEmptyMatcher() {
return DefaultExtensionMatcher.builder(client,
GroupVersionKind.fromExtension(FakeExtension.class))
.build();
}
@Test
void shouldAddExtensionWhenAddPredicateAlwaysTrue() {
when(predicates.onAddPredicate()).thenReturn(e -> true);
when(matchers.onAddMatcher()).thenReturn(getEmptyMatcher());
watcher.onAdd(createFake("fake-name"));
verify(predicates, times(1)).onAddPredicate();
verify(matchers, times(1)).onAddMatcher();
verify(queue, times(1)).addImmediately(
argThat(request -> request.name().equals("fake-name")));
verify(queue, times(0)).add(any());
@ -41,10 +54,12 @@ class ExtensionWatcherTest {
@Test
void shouldNotAddExtensionWhenAddPredicateAlwaysFalse() {
when(predicates.onAddPredicate()).thenReturn(e -> false);
var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User");
when(matchers.onAddMatcher()).thenReturn(
DefaultExtensionMatcher.builder(client, type).build());
watcher.onAdd(createFake("fake-name"));
verify(predicates, times(1)).onAddPredicate();
verify(matchers, times(1)).onAddMatcher();
verify(queue, times(0)).add(any());
verify(queue, times(0)).addImmediately(any());
}
@ -54,17 +69,17 @@ class ExtensionWatcherTest {
watcher.dispose();
watcher.onAdd(createFake("fake-name"));
verify(predicates, times(0)).onAddPredicate();
verify(matchers, times(0)).onAddMatcher();
verify(queue, times(0)).addImmediately(any());
verify(queue, times(0)).add(any());
}
@Test
void shouldUpdateExtensionWhenUpdatePredicateAlwaysTrue() {
when(predicates.onUpdatePredicate()).thenReturn((e1, e2) -> true);
when(matchers.onUpdateMatcher()).thenReturn(getEmptyMatcher());
watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name"));
verify(predicates, times(1)).onUpdatePredicate();
verify(matchers, times(1)).onUpdateMatcher();
verify(queue, times(1)).addImmediately(
argThat(request -> request.name().equals("new-fake-name")));
verify(queue, times(0)).add(any());
@ -72,10 +87,12 @@ class ExtensionWatcherTest {
@Test
void shouldUpdateExtensionWhenUpdatePredicateAlwaysFalse() {
when(predicates.onUpdatePredicate()).thenReturn((e1, e2) -> false);
var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User");
when(matchers.onUpdateMatcher()).thenReturn(
DefaultExtensionMatcher.builder(client, type).build());
watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name"));
verify(predicates, times(1)).onUpdatePredicate();
verify(matchers, times(1)).onUpdateMatcher();
verify(queue, times(0)).add(any());
verify(queue, times(0)).addImmediately(any());
}
@ -85,17 +102,17 @@ class ExtensionWatcherTest {
watcher.dispose();
watcher.onUpdate(createFake("old-fake-name"), createFake("new-fake-name"));
verify(predicates, times(0)).onUpdatePredicate();
verify(matchers, times(0)).onUpdateMatcher();
verify(queue, times(0)).add(any());
verify(queue, times(0)).addImmediately(any());
}
@Test
void shouldDeleteExtensionWhenDeletePredicateAlwaysTrue() {
when(predicates.onDeletePredicate()).thenReturn(e -> true);
when(matchers.onDeleteMatcher()).thenReturn(getEmptyMatcher());
watcher.onDelete(createFake("fake-name"));
verify(predicates, times(1)).onDeletePredicate();
verify(matchers, times(1)).onDeleteMatcher();
verify(queue, times(1)).addImmediately(
argThat(request -> request.name().equals("fake-name")));
verify(queue, times(0)).add(any());
@ -103,10 +120,12 @@ class ExtensionWatcherTest {
@Test
void shouldDeleteExtensionWhenDeletePredicateAlwaysFalse() {
when(predicates.onDeletePredicate()).thenReturn(e -> false);
var type = GroupVersionKind.fromAPIVersionAndKind("v1alpha1", "User");
when(matchers.onDeleteMatcher()).thenReturn(
DefaultExtensionMatcher.builder(client, type).build());
watcher.onDelete(createFake("fake-name"));
verify(predicates, times(1)).onDeletePredicate();
verify(matchers, times(1)).onDeleteMatcher();
verify(queue, times(0)).add(any());
verify(queue, times(0)).addImmediately(any());
}
@ -116,7 +135,7 @@ class ExtensionWatcherTest {
watcher.dispose();
watcher.onDelete(createFake("fake-name"));
verify(predicates, times(0)).onDeletePredicate();
verify(matchers, times(0)).onDeleteMatcher();
verify(queue, times(0)).add(any());
verify(queue, times(0)).addImmediately(any());
}

View File

@ -3,22 +3,26 @@ package run.halo.app.extension.controller;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.ArgumentMatchers.same;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.function.Predicate;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionMatcher;
import run.halo.app.extension.FakeExtension;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.Watcher;
import run.halo.app.extension.index.IndexedQueryEngine;
@ExtendWith(MockitoExtension.class)
class RequestSynchronizerTest {
@ -26,41 +30,47 @@ class RequestSynchronizerTest {
@Mock
ExtensionClient client;
@Mock
IndexedQueryEngine indexedQueryEngine;
@Mock
Watcher watcher;
@Mock
Predicate<Extension> listPredicate;
ExtensionMatcher listMatcher;
RequestSynchronizer synchronizer;
@BeforeEach
void setUp() {
when(client.indexedQueryEngine()).thenReturn(indexedQueryEngine);
synchronizer =
new RequestSynchronizer(true, client, new FakeExtension(), watcher, listPredicate);
new RequestSynchronizer(true, client, new FakeExtension(), watcher, listMatcher);
assertFalse(synchronizer.isDisposed());
assertFalse(synchronizer.isStarted());
}
@Test
void shouldStartCorrectlyWhenSyncingAllOnStart() {
when(client.list(same(FakeExtension.class), any(), any())).thenReturn(
List.of(FakeExtension.createFake("fake-01"), FakeExtension.createFake("fake-02")));
var type = GroupVersionKind.fromExtension(FakeExtension.class);
when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class)))
.thenReturn(List.of("fake-01", "fake-02"));
synchronizer.start();
assertTrue(synchronizer.isStarted());
assertFalse(synchronizer.isDisposed());
verify(client, times(1)).list(same(FakeExtension.class), any(), any());
verify(watcher, times(2)).onAdd(any());
verify(indexedQueryEngine, times(1)).retrieveAll(eq(type),
isA(ListOptions.class));
verify(watcher, times(2)).onAdd(isA(Reconciler.Request.class));
verify(client, times(1)).watch(same(watcher));
}
@Test
void shouldStartCorrectlyWhenNotSyncingAllOnStart() {
synchronizer =
new RequestSynchronizer(false, client, new FakeExtension(), watcher, listPredicate);
new RequestSynchronizer(false, client, new FakeExtension(), watcher, listMatcher);
assertFalse(synchronizer.isDisposed());
assertFalse(synchronizer.isStarted());
@ -70,7 +80,7 @@ class RequestSynchronizerTest {
assertFalse(synchronizer.isDisposed());
verify(client, times(0)).list(any(), any(), any());
verify(watcher, times(0)).onAdd(any());
verify(watcher, times(0)).onAdd(isA(Reconciler.Request.class));
verify(client, times(1)).watch(any(Watcher.class));
}
@ -93,7 +103,7 @@ class RequestSynchronizerTest {
synchronizer.start();
verify(client, times(0)).list(any(), any(), any());
verify(watcher, times(0)).onAdd(any());
verify(watcher, times(0)).onAdd(isA(Reconciler.Request.class));
verify(client, times(0)).watch(any());
}

View File

@ -0,0 +1,56 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.Set;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.junit.jupiter.api.Test;
import run.halo.app.extension.AbstractExtension;
import run.halo.app.extension.GVK;
import run.halo.app.extension.Unstructured;
/**
* Tests for {@link FunctionalMultiValueIndexAttribute}.
*
* @author guqing
* @since 2.12.0
*/
class FunctionalMultiValueIndexAttributeTest {
@Test
void create() {
var attribute = new FunctionalMultiValueIndexAttribute<>(FakeExtension.class,
FakeExtension::getCategories);
assertThat(attribute).isNotNull();
}
@Test
void getValues() {
var attribute = new FunctionalMultiValueIndexAttribute<>(FakeExtension.class,
FakeExtension::getCategories);
var fake = new FakeExtension();
fake.setCategories(Set.of("test", "halo"));
assertThat(attribute.getValues(fake)).isEqualTo(fake.getCategories());
var unstructured = Unstructured.OBJECT_MAPPER.convertValue(fake, Unstructured.class);
assertThat(attribute.getValues(unstructured)).isEqualTo(fake.getCategories());
var demoExt = new DemoExtension();
assertThatThrownBy(() -> attribute.getValues(demoExt))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Object type does not match");
}
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test.halo.run", version = "v1", kind = "FakeExtension", plural = "fakes",
singular = "fake")
static class FakeExtension extends AbstractExtension {
Set<String> categories;
}
class DemoExtension extends AbstractExtension {
}
}

View File

@ -0,0 +1,41 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Set;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.junit.jupiter.api.Test;
import run.halo.app.extension.AbstractExtension;
import run.halo.app.extension.GVK;
import run.halo.app.extension.Metadata;
/**
* Tests for {@link IndexAttributeFactory}.
*
* @author guqing
* @since 2.12.0
*/
class IndexAttributeFactoryTest {
@Test
void multiValueAttribute() {
var attribute = IndexAttributeFactory.multiValueAttribute(FakeExtension.class,
FakeExtension::getTags);
assertThat(attribute).isNotNull();
assertThat(attribute.getObjectType()).isEqualTo(FakeExtension.class);
var extension = new FakeExtension();
extension.setMetadata(new Metadata());
extension.getMetadata().setName("fake-name-1");
extension.setTags(Set.of("tag1", "tag2"));
assertThat(attribute.getValues(extension)).isEqualTo(Set.of("tag1", "tag2"));
}
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakes",
singular = "fake")
static class FakeExtension extends AbstractExtension {
Set<String> tags;
}
}

View File

@ -0,0 +1,76 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.junit.jupiter.api.Test;
import run.halo.app.extension.AbstractExtension;
import run.halo.app.extension.GVK;
/**
* Tests for {@link IndexSpec}.
*
* @author guqing
* @since 2.12.0
*/
class IndexSpecTest {
@Test
void equalsVerifier() {
var spec1 = new IndexSpec()
.setName("metadata.name")
.setOrder(IndexSpec.OrderType.ASC)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName())
)
.setUnique(true);
var spec2 = new IndexSpec()
.setName("metadata.name")
.setOrder(IndexSpec.OrderType.ASC)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName())
)
.setUnique(true);
assertThat(spec1).isEqualTo(spec2);
assertThat(spec1.equals(spec2)).isTrue();
assertThat(spec1.hashCode()).isEqualTo(spec2.hashCode());
var spec3 = new IndexSpec()
.setName("metadata.name")
.setOrder(IndexSpec.OrderType.DESC)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName())
)
.setUnique(false);
assertThat(spec1).isEqualTo(spec3);
assertThat(spec1.equals(spec3)).isTrue();
assertThat(spec1.hashCode()).isEqualTo(spec3.hashCode());
var spec4 = new IndexSpec()
.setName("slug")
.setOrder(IndexSpec.OrderType.ASC)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName())
)
.setUnique(true);
assertThat(spec1.equals(spec4)).isFalse();
assertThat(spec1).isNotEqualTo(spec4);
}
@Test
void equalAnotherObject() {
var spec3 = new IndexSpec()
.setName("metadata.name");
assertThat(spec3.equals(new Object())).isFalse();
}
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test", version = "v1", kind = "Fake", plural = "fakes", singular = "fake")
static class FakeExtension extends AbstractExtension {
private String slug;
}
}

View File

@ -0,0 +1,93 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.greaterThan;
import static run.halo.app.extension.index.query.QueryFactory.or;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
/**
* Tests for the {@link And} query.
*
* @author guqing
* @since 2.12.0
*/
public class AndTest {
@Test
void testMatches() {
Collection<Map.Entry<String, String>> deptEntry = List.of(Map.entry("A", "guqing"),
Map.entry("A", "halo"),
Map.entry("B", "lisi"),
Map.entry("B", "zhangsan"),
Map.entry("C", "ryanwang"),
Map.entry("C", "johnniang")
);
Collection<Map.Entry<String, String>> ageEntry = List.of(Map.entry("19", "halo"),
Map.entry("19", "guqing"),
Map.entry("18", "zhangsan"),
Map.entry("17", "lisi"),
Map.entry("17", "ryanwang"),
Map.entry("17", "johnniang")
);
var entries = Map.of("dept", deptEntry, "age", ageEntry);
var indexView = new QueryIndexViewImpl(entries);
var query = and(equal("dept", "B"), equal("age", "18"));
var resultSet = query.matches(indexView);
assertThat(resultSet).containsExactly("zhangsan");
query = and(equal("dept", "C"), equal("age", "18"));
resultSet = query.matches(indexView);
assertThat(resultSet).isEmpty();
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "B")),
// guqing, halo, zhangsan
or(equal("age", "19"), equal("age", "18"))
);
resultSet = query.matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan");
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "B")),
// guqing, halo, zhangsan
or(equal("age", "19"), equal("age", "18"))
);
resultSet = query.matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan");
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "C")),
// guqing, halo, zhangsan
and(equal("age", "17"), equal("age", "17"))
);
resultSet = query.matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder("ryanwang", "johnniang");
}
@Test
void andMatch2() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var query = and(equal("lastName", "Fay"),
and(
equal("hireDate", "17"),
and(greaterThan("salary", "1000"),
and(equal("managerId", "101"),
equal("departmentId", "50")
)
)
)
);
var resultSet = query.matches(indexView);
assertThat(resultSet).containsExactly("100");
}
}

View File

@ -0,0 +1,98 @@
package run.halo.app.extension.index.query;
import java.util.Collection;
import java.util.List;
import java.util.Map;
public class EmployeeDataSet {
/**
* Create a {@link QueryIndexView} for employee to test.
*
* @return a {@link QueryIndexView} for employee to test
*/
public static QueryIndexView createEmployeeIndexView() {
/*
* id firstName lastName email hireDate salary managerId departmentId
* 100 Pat Fay p 17 2600 101 50
* 101 Lee Day l 17 2400 102 40
* 102 William Jay w 19 2200 102 50
* 103 Mary Day p 17 2000 103 50
* 104 John Fay j 17 1800 103 50
* 105 Gon Fay p 18 1900 101 40
*/
Collection<Map.Entry<String, String>> idEntry = List.of(
Map.entry("100", "100"),
Map.entry("101", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("104", "104"),
Map.entry("105", "105")
);
Collection<Map.Entry<String, String>> firstNameEntry = List.of(
Map.entry("Pat", "100"),
Map.entry("Lee", "101"),
Map.entry("William", "102"),
Map.entry("Mary", "103"),
Map.entry("John", "104"),
Map.entry("Gon", "105")
);
Collection<Map.Entry<String, String>> lastNameEntry = List.of(
Map.entry("Fay", "100"),
Map.entry("Day", "101"),
Map.entry("Jay", "102"),
Map.entry("Day", "103"),
Map.entry("Fay", "104"),
Map.entry("Fay", "105")
);
Collection<Map.Entry<String, String>> emailEntry = List.of(
Map.entry("p", "100"),
Map.entry("l", "101"),
Map.entry("w", "102"),
Map.entry("p", "103"),
Map.entry("j", "104"),
Map.entry("p", "105")
);
Collection<Map.Entry<String, String>> hireDateEntry = List.of(
Map.entry("17", "100"),
Map.entry("17", "101"),
Map.entry("19", "102"),
Map.entry("17", "103"),
Map.entry("17", "104"),
Map.entry("18", "105")
);
Collection<Map.Entry<String, String>> salaryEntry = List.of(
Map.entry("2600", "100"),
Map.entry("2400", "101"),
Map.entry("2200", "102"),
Map.entry("2000", "103"),
Map.entry("1800", "104"),
Map.entry("1900", "105")
);
Collection<Map.Entry<String, String>> managerIdEntry = List.of(
Map.entry("101", "100"),
Map.entry("102", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("103", "104"),
Map.entry("101", "105")
);
Collection<Map.Entry<String, String>> departmentIdEntry = List.of(
Map.entry("50", "100"),
Map.entry("40", "101"),
Map.entry("50", "102"),
Map.entry("50", "103"),
Map.entry("50", "104"),
Map.entry("40", "105")
);
var entries = Map.of("id", idEntry,
"firstName", firstNameEntry,
"lastName", lastNameEntry,
"email", emailEntry,
"hireDate", hireDateEntry,
"salary", salaryEntry,
"managerId", managerIdEntry,
"departmentId", departmentIdEntry);
return new QueryIndexViewImpl(entries);
}
}

View File

@ -0,0 +1,234 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static run.halo.app.extension.index.query.QueryFactory.all;
import static run.halo.app.extension.index.query.QueryFactory.between;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.equalOtherField;
import static run.halo.app.extension.index.query.QueryFactory.notEqual;
import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link QueryFactory}.
*
* @author guqing
* @since 2.12.0
*/
class QueryFactoryTest {
@Test
void allTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = all("firstName").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103", "104", "105"
);
}
@Test
void equalTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = equal("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "104", "105"
);
}
@Test
void equalOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = equalOtherField("managerId", "id").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
}
@Test
void notEqualTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = notEqual("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"101", "102", "103"
);
}
@Test
void notEqualOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = notEqualOtherField("managerId", "id").matches(indexView);
// 103 102 is equal
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "104", "105"
);
}
@Test
void lessThanTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThan("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102"
);
}
@Test
void lessThanOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
}
@Test
void lessThanOrEqualTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void lessThanOrEqualOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void greaterThanTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
}
@Test
void greaterThanOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
}
@Test
void greaterThanOrEqualTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104", "105"
);
}
@Test
void greaterThanOrEqualOtherFieldTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103", "104", "105"
);
}
@Test
void inTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.in("id", "103", "104").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104"
);
}
@Test
void inTest2() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "104", "105"
);
}
@Test
void betweenTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = between("id", "103", "105").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104", "105"
);
indexView = EmployeeDataSet.createEmployeeIndexView();
resultSet = between("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"101", "102", "103"
);
}
@Test
void betweenLowerExclusive() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.betweenLowerExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"101", "102"
);
}
@Test
void betweenUpperExclusive() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.betweenUpperExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
}
@Test
void betweenExclusive() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.betweenExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
);
}
@Test
void startsWithTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
);
}
@Test
void endsWithTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103"
);
}
@Test
void containsTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.contains("firstName", "i").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
);
resultSet = QueryFactory.contains("firstName", "N").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
}
}

View File

@ -0,0 +1,83 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link QueryIndexViewImpl}.
*
* @author guqing
* @since 2.12.0
*/
class QueryIndexViewImplTest {
@Test
void findIdsForFieldValueEqualTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueEqual("managerId", "id");
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
}
@Test
void findIdsForFieldValueGreaterThanTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = EmployeeDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
@Test
void findIdsForFieldValueGreaterThanTest2() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = EmployeeDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = EmployeeDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest2() {
var indexView = EmployeeDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = EmployeeDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
}

View File

@ -12,6 +12,7 @@ import org.springframework.context.ApplicationListener;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import run.halo.app.core.extension.content.Post;
import run.halo.app.extension.DefaultExtensionMatcher;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.GroupVersionKind;
@ -53,11 +54,13 @@ public class PostIndexInformer implements ApplicationListener<ApplicationStarted
postIndexer.addIndexFunc(LABEL_INDEXER_NAME, labelIndexFunc());
this.postWatcher = new PostWatcher();
var emptyPost = new Post();
this.synchronizer = new RequestSynchronizer(true,
client,
new Post(),
emptyPost,
postWatcher,
this::checkExtension);
DefaultExtensionMatcher.builder(client, emptyPost.groupVersionKind()).build()
);
}
private DefaultIndexer.IndexFunc<Post> labelIndexFunc() {

View File

@ -4,19 +4,26 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import org.springframework.lang.NonNull;
import org.springframework.lang.Nullable;
import run.halo.app.extension.SchemeWatcherManager.SchemeRegistered;
import run.halo.app.extension.SchemeWatcherManager.SchemeUnregistered;
import run.halo.app.extension.index.IndexSpecRegistry;
import run.halo.app.extension.index.IndexSpecs;
public class DefaultSchemeManager implements SchemeManager {
private final List<Scheme> schemes;
private final IndexSpecRegistry indexSpecRegistry;
@Nullable
private final SchemeWatcherManager watcherManager;
public DefaultSchemeManager(@Nullable SchemeWatcherManager watcherManager) {
public DefaultSchemeManager(IndexSpecRegistry indexSpecRegistry,
@Nullable SchemeWatcherManager watcherManager) {
this.indexSpecRegistry = indexSpecRegistry;
this.watcherManager = watcherManager;
// we have to use CopyOnWriteArrayList at here to prevent concurrent modification between
// registering and listing.
@ -26,14 +33,27 @@ public class DefaultSchemeManager implements SchemeManager {
@Override
public void register(@NonNull Scheme scheme) {
if (!schemes.contains(scheme)) {
indexSpecRegistry.indexFor(scheme);
schemes.add(scheme);
getWatchers().forEach(watcher -> watcher.onChange(new SchemeRegistered(scheme)));
}
}
@Override
public void register(@NonNull Scheme scheme, Consumer<IndexSpecs> specsConsumer) {
if (schemes.contains(scheme)) {
return;
}
var indexSpecs = indexSpecRegistry.indexFor(scheme);
specsConsumer.accept(indexSpecs);
schemes.add(scheme);
getWatchers().forEach(watcher -> watcher.onChange(new SchemeRegistered(scheme)));
}
@Override
public void unregister(@NonNull Scheme scheme) {
if (schemes.contains(scheme)) {
indexSpecRegistry.removeIndexSpecs(scheme);
schemes.remove(scheme);
getWatchers().forEach(watcher -> watcher.onChange(new SchemeUnregistered(scheme)));
}

View File

@ -4,7 +4,9 @@ import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;
import run.halo.app.extension.index.IndexedQueryEngine;
/**
* DelegateExtensionClient fully delegates ReactiveExtensionClient.
@ -32,6 +34,17 @@ public class DelegateExtensionClient implements ExtensionClient {
return client.list(type, predicate, comparator, page, size).block();
}
@Override
public <E extends Extension> List<E> listAll(Class<E> type, ListOptions options, Sort sort) {
return client.listAll(type, options, sort).collectList().block();
}
@Override
public <E extends Extension> ListResult<E> listBy(Class<E> type, ListOptions options,
PageRequest page) {
return client.listBy(type, options, page).block();
}
@Override
public <E extends Extension> Optional<E> fetch(Class<E> type, String name) {
return client.fetch(type, name).blockOptional();
@ -57,6 +70,11 @@ public class DelegateExtensionClient implements ExtensionClient {
client.delete(extension).block();
}
@Override
public IndexedQueryEngine indexedQueryEngine() {
return client.indexedQueryEngine();
}
@Override
public void watch(Watcher watcher) {
client.watch(watcher);

View File

@ -11,17 +11,36 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.util.Predicates;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.retry.Retry;
import run.halo.app.extension.exception.ExtensionNotFoundException;
import run.halo.app.extension.index.DefaultExtensionIterator;
import run.halo.app.extension.index.ExtensionIterator;
import run.halo.app.extension.index.ExtensionPaginatedLister;
import run.halo.app.extension.index.IndexedQueryEngine;
import run.halo.app.extension.index.IndexerFactory;
import run.halo.app.extension.store.ReactiveExtensionStoreClient;
@Slf4j
@Component
@RequiredArgsConstructor
public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
private final ReactiveExtensionStoreClient client;
@ -30,18 +49,16 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
private final SchemeManager schemeManager;
private final Watcher.WatcherComposite watchers;
private final Watcher.WatcherComposite watchers = new Watcher.WatcherComposite();
private final ObjectMapper objectMapper;
public ReactiveExtensionClientImpl(ReactiveExtensionStoreClient client,
ExtensionConverter converter, SchemeManager schemeManager, ObjectMapper objectMapper) {
this.client = client;
this.converter = converter;
this.schemeManager = schemeManager;
this.objectMapper = objectMapper;
this.watchers = new Watcher.WatcherComposite();
}
private final IndexerFactory indexerFactory;
private final IndexedQueryEngine indexedQueryEngine;
private final ConcurrentMap<GroupKind, AtomicBoolean> indexBuildingState =
new ConcurrentHashMap<>();
@Override
public <E extends Extension> Flux<E> list(Class<E> type, Predicate<E> predicate,
@ -74,6 +91,37 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
});
}
@Override
public <E extends Extension> Flux<E> listAll(Class<E> type, ListOptions options, Sort sort) {
return listBy(type, options, PageRequestImpl.ofSize(0).withSort(sort))
.flatMapIterable(ListResult::getItems);
}
@Override
public <E extends Extension> Mono<ListResult<E>> listBy(Class<E> type, ListOptions options,
PageRequest page) {
var scheme = schemeManager.get(type);
return Mono.fromSupplier(
() -> indexedQueryEngine.retrieve(scheme.groupVersionKind(), options, page)
)
.flatMap(objectKeys -> {
var storeNames = objectKeys.get()
.map(objectKey -> ExtensionStoreUtil.buildStoreName(scheme, objectKey))
.toList();
final long startTimeMs = System.currentTimeMillis();
return client.listByNames(storeNames)
.map(extensionStore -> converter.convertFrom(type, extensionStore))
.doFinally(s -> {
log.debug("Successfully retrieved by names from db for {} in {}ms",
scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs);
})
.collectList()
.map(result -> new ListResult<>(page.getPageNumber(), page.getPageSize(),
objectKeys.getTotal(), result));
})
.defaultIfEmpty(ListResult.emptyResult());
}
@Override
public <E extends Extension> Mono<E> fetch(Class<E> type, String name) {
var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(type), name);
@ -103,8 +151,9 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
}
@Override
@SuppressWarnings("unchecked")
@Transactional
public <E extends Extension> Mono<E> create(E extension) {
checkClientWritable(extension);
return Mono.just(extension)
.doOnNext(ext -> {
var metadata = extension.getMetadata();
@ -117,7 +166,7 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
if (!hasText(metadata.getGenerateName())) {
throw new IllegalArgumentException(
"The metadata.generateName must not be blank when metadata.name is "
+ "blank");
+ "blank");
}
// generate name with random text
metadata.setName(metadata.getGenerateName() + randomAlphabetic(5));
@ -125,18 +174,20 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
extension.setMetadata(metadata);
})
.map(converter::convertTo)
.flatMap(extStore -> client.create(extStore.getName(), extStore.getData())
.map(created -> converter.convertFrom((Class<E>) extension.getClass(), created))
.doOnNext(watchers::onAdd))
.flatMap(extStore -> doCreate(extension, extStore.getName(), extStore.getData())
.doOnNext(watchers::onAdd)
)
.retryWhen(Retry.backoff(3, Duration.ofMillis(100))
// retry when generateName is set
.filter(t -> t instanceof DataIntegrityViolationException
&& hasText(extension.getMetadata().getGenerateName())));
&& hasText(extension.getMetadata().getGenerateName()))
);
}
@Override
@SuppressWarnings("unchecked")
@Transactional
public <E extends Extension> Mono<E> update(E extension) {
checkClientWritable(extension);
// Refactor the atomic reference if we have a better solution.
return getLatest(extension).flatMap(old -> {
var oldJsonExt = new JsonExtension(objectMapper, old);
@ -156,8 +207,7 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
isOnlyStatusChanged(oldJsonExt.getInternal(), newJsonExt.getInternal());
var store = this.converter.convertTo(newJsonExt);
var updated = client.update(store.getName(), store.getVersion(), store.getData())
.map(ext -> converter.convertFrom((Class<E>) extension.getClass(), ext));
var updated = doUpdate(extension, store.getName(), store.getVersion(), store.getData());
if (!onlyStatusChanged) {
updated = updated.doOnNext(ext -> watchers.onUpdate(old, ext));
}
@ -173,15 +223,60 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
}
@Override
@SuppressWarnings("unchecked")
@Transactional
public <E extends Extension> Mono<E> delete(E extension) {
checkClientWritable(extension);
// set deletionTimestamp
extension.getMetadata().setDeletionTimestamp(Instant.now());
var extensionStore = converter.convertTo(extension);
return client.update(extensionStore.getName(), extensionStore.getVersion(),
extensionStore.getData())
.map(deleted -> converter.convertFrom((Class<E>) extension.getClass(), deleted))
.doOnNext(watchers::onDelete);
return doUpdate(extension, extensionStore.getName(),
extensionStore.getVersion(), extensionStore.getData()
).doOnNext(watchers::onDelete);
}
@Override
public IndexedQueryEngine indexedQueryEngine() {
return this.indexedQueryEngine;
}
@SuppressWarnings("unchecked")
<E extends Extension> Mono<E> doCreate(E oldExtension, String name, byte[] data) {
return Mono.defer(() -> {
var gvk = oldExtension.groupVersionKind();
var type = (Class<E>) oldExtension.getClass();
var indexer = indexerFactory.getIndexer(gvk);
return client.create(name, data)
.map(created -> converter.convertFrom(type, created))
.doOnNext(indexer::indexRecord);
});
}
@SuppressWarnings("unchecked")
<E extends Extension> Mono<E> doUpdate(E oldExtension, String name, Long version, byte[] data) {
return Mono.defer(() -> {
var type = (Class<E>) oldExtension.getClass();
var indexer = indexerFactory.getIndexer(oldExtension.groupVersionKind());
return client.update(name, version, data)
.map(updated -> converter.convertFrom(type, updated))
.doOnNext(indexer::updateRecord);
});
}
/**
* If the extension is being updated, we should the index is not building index for the
* extension, otherwise the {@link IllegalStateException} will be thrown.
*/
private <E extends Extension> void checkClientWritable(E extension) {
var buildingState = indexBuildingState.get(extension.groupVersionKind().groupKind());
if (buildingState != null && buildingState.get()) {
throw new IllegalStateException("Index is building for " + extension.groupVersionKind()
+ ", please wait for a moment and try again.");
}
}
void setIndexBuildingStateFor(GroupKind groupKind, boolean building) {
indexBuildingState.computeIfAbsent(groupKind, k -> new AtomicBoolean(building))
.set(building);
}
@Override
@ -211,4 +306,59 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
}
return true;
}
@Component
@RequiredArgsConstructor
class IndexBuildsManager {
private final SchemeManager schemeManager;
private final IndexerFactory indexerFactory;
private final ExtensionConverter converter;
private final ReactiveExtensionStoreClient client;
private final SchemeWatcherManager schemeWatcherManager;
@NonNull
private ExtensionIterator<Extension> createExtensionIterator(Scheme scheme) {
var type = scheme.type();
var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme);
var lister = new ExtensionPaginatedLister() {
@Override
@SuppressWarnings("unchecked")
public <E extends Extension> Page<E> list(Pageable pageable) {
return client.listByNamePrefix(prefix, pageable)
.map(page -> page.map(
store -> (E) converter.convertFrom(type, store))
)
.block();
}
};
return new DefaultExtensionIterator<>(lister);
}
@EventListener(ContextRefreshedEvent.class)
public void startBuildingIndex() {
final long startTimeMs = System.currentTimeMillis();
log.info("Start building index for all extensions, please wait...");
schemeManager.schemes()
.forEach(this::createIndexerFor);
schemeWatcherManager.register(event -> {
if (event instanceof SchemeWatcherManager.SchemeRegistered schemeRegistered) {
createIndexerFor(schemeRegistered.getNewScheme());
return;
}
if (event instanceof SchemeWatcherManager.SchemeUnregistered schemeUnregistered) {
var scheme = schemeUnregistered.getDeletedScheme();
indexerFactory.removeIndexer(scheme);
}
});
log.info("Successfully built index in {}ms, Preparing to lunch application...",
System.currentTimeMillis() - startTimeMs);
}
private void createIndexerFor(Scheme scheme) {
setIndexBuildingStateFor(scheme.groupVersionKind().groupKind(), true);
indexerFactory.createIndexerFor(scheme.type(), createExtensionIterator(scheme));
setIndexBuildingStateFor(scheme.groupVersionKind().groupKind(), false);
}
}
}

View File

@ -16,6 +16,7 @@ import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.DefaultController;
import run.halo.app.extension.controller.DefaultQueue;
import run.halo.app.extension.controller.Reconciler;
import run.halo.app.extension.index.IndexerFactory;
import run.halo.app.extension.store.ExtensionStoreClient;
@Slf4j
@ -30,21 +31,23 @@ class GcReconciler implements Reconciler<GcRequest> {
private final SchemeManager schemeManager;
private final IndexerFactory indexerFactory;
private final SchemeWatcherManager schemeWatcherManager;
GcReconciler(ExtensionClient client,
ExtensionStoreClient storeClient,
ExtensionConverter converter,
SchemeManager schemeManager,
SchemeManager schemeManager, IndexerFactory indexerFactory,
SchemeWatcherManager schemeWatcherManager) {
this.client = client;
this.storeClient = storeClient;
this.converter = converter;
this.schemeManager = schemeManager;
this.indexerFactory = indexerFactory;
this.schemeWatcherManager = schemeWatcherManager;
}
@Override
public Result reconcile(GcRequest request) {
log.debug("Extension {} is being deleted", request);
@ -54,6 +57,9 @@ class GcReconciler implements Reconciler<GcRequest> {
.ifPresent(extension -> {
var extensionStore = converter.convertTo(extension);
storeClient.delete(extensionStore.getName(), extensionStore.getVersion());
// drop index for this extension
var indexer = indexerFactory.getIndexer(extension.groupVersionKind());
indexer.unIndexRecord(request.name());
log.debug("Extension {} was deleted", request);
});

View File

@ -2,9 +2,12 @@ package run.halo.app.extension.gc;
import static run.halo.app.extension.Comparators.compareCreationTimestamp;
import java.util.List;
import java.util.function.Predicate;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.Scheme;
import run.halo.app.extension.SchemeManager;
import run.halo.app.extension.SchemeWatcherManager;
@ -12,6 +15,8 @@ import run.halo.app.extension.SchemeWatcherManager.SchemeRegistered;
import run.halo.app.extension.Watcher;
import run.halo.app.extension.controller.RequestQueue;
import run.halo.app.extension.controller.Synchronizer;
import run.halo.app.extension.index.query.QueryFactory;
import run.halo.app.extension.router.selector.FieldSelector;
class GcSynchronizer implements Synchronizer<GcRequest> {
@ -58,6 +63,7 @@ class GcSynchronizer implements Synchronizer<GcRequest> {
this.schemeWatcherManager.register(event -> {
if (event instanceof SchemeRegistered registeredEvent) {
var newScheme = registeredEvent.getNewScheme();
listDeleted(newScheme.type()).forEach(watcher::onDelete);
client.list(newScheme.type(), deleted(), compareCreationTimestamp(true))
.forEach(watcher::onDelete);
}
@ -65,8 +71,18 @@ class GcSynchronizer implements Synchronizer<GcRequest> {
client.watch(watcher);
schemeManager.schemes().stream()
.map(Scheme::type)
.forEach(type -> client.list(type, deleted(), compareCreationTimestamp(true))
.forEach(watcher::onDelete));
.forEach(type -> listDeleted(type).forEach(watcher::onDelete));
}
<E extends Extension> List<E> listDeleted(Class<E> type) {
var options = new ListOptions()
.setFieldSelector(
FieldSelector.of(QueryFactory.all("metadata.deletionTimestamp"))
);
return client.listAll(type, options, Sort.by("metadata.creationTimestamp"))
.stream()
.sorted(compareCreationTimestamp(true))
.toList();
}
private <E extends Extension> Predicate<E> deleted() {

View File

@ -0,0 +1,62 @@
package run.halo.app.extension.index;
import java.util.List;
import java.util.NoSuchElementException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.Extension;
/**
* Default implementation of {@link ExtensionIterator}.
*
* @param <E> the type of the extension.
* @author guqing
* @since 2.12.0
*/
public class DefaultExtensionIterator<E extends Extension> implements ExtensionIterator<E> {
static final int DEFAULT_PAGE_SIZE = 500;
private final ExtensionPaginatedLister lister;
private Pageable currentPageable;
private List<E> currentData;
private int currentIndex;
/**
* Constructs a new DefaultExtensionIterator with the given lister.
*
* @param lister the lister to use to load data.
*/
public DefaultExtensionIterator(ExtensionPaginatedLister lister) {
this.lister = lister;
this.currentPageable = PageRequest.of(0, DEFAULT_PAGE_SIZE, Sort.by("name"));
this.currentData = loadData();
}
private List<E> loadData() {
Page<E> page = lister.list(currentPageable);
currentPageable = page.hasNext() ? page.nextPageable() : null;
return page.getContent();
}
@Override
public boolean hasNext() {
if (currentIndex < currentData.size()) {
return true;
}
if (currentPageable == null) {
return false;
}
currentData = loadData();
currentIndex = 0;
return !currentData.isEmpty();
}
@Override
public E next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
return currentData.get(currentIndex++);
}
}

View File

@ -0,0 +1,64 @@
package run.halo.app.extension.index;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.lang3.StringUtils;
/**
* Default implementation of {@link IndexSpecs}.
*
* @author guqing
* @since 2.12.0
*/
public class DefaultIndexSpecs implements IndexSpecs {
private final ConcurrentMap<String, IndexSpec> indexSpecs;
public DefaultIndexSpecs() {
this.indexSpecs = new ConcurrentHashMap<>();
}
@Override
public void add(IndexSpec indexSpec) {
checkIndexSpec(indexSpec);
var indexName = indexSpec.getName();
var existingSpec = indexSpecs.putIfAbsent(indexName, indexSpec);
if (existingSpec != null) {
throw new IllegalArgumentException(
"IndexSpec with name " + indexName + " already exists");
}
}
@Override
public List<IndexSpec> getIndexSpecs() {
return List.copyOf(this.indexSpecs.values());
}
@Override
public IndexSpec getIndexSpec(String indexName) {
return this.indexSpecs.get(indexName);
}
@Override
public boolean contains(String indexName) {
return this.indexSpecs.containsKey(indexName);
}
@Override
public void remove(String name) {
this.indexSpecs.remove(name);
}
private void checkIndexSpec(IndexSpec indexSpec) {
var order = indexSpec.getOrder();
if (order == null) {
indexSpec.setOrder(IndexSpec.OrderType.ASC);
}
if (StringUtils.isBlank(indexSpec.getName())) {
throw new IllegalArgumentException("IndexSpec name must not be blank");
}
if (indexSpec.getIndexFunc() == null) {
throw new IllegalArgumentException("IndexSpec indexFunc must not be null");
}
}
}

View File

@ -0,0 +1,200 @@
package run.halo.app.extension.index;
import static run.halo.app.extension.index.IndexerTransaction.ChangeRecord;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import org.apache.commons.lang3.BooleanUtils;
import run.halo.app.extension.Extension;
/**
* <p>A default implementation of {@link Indexer}.</p>
* <p>It uses the {@link IndexEntryContainer} to store the index entries for the specified
* {@link IndexDescriptor}s.</p>
*
* @author guqing
* @since 2.12.0
*/
public class DefaultIndexer implements Indexer {
private final ReadWriteLock rwl = new ReentrantReadWriteLock();
private final Lock readLock = rwl.readLock();
private final Lock writeLock = rwl.writeLock();
private final List<IndexDescriptor> indexDescriptors;
private final IndexEntryContainer indexEntries;
/**
* Constructs a new {@link DefaultIndexer} with the given {@link IndexDescriptor}s and
* {@link IndexEntryContainer}.
*
* @param indexDescriptors the index descriptors.
* @param oldIndexEntries must have the same size with the given descriptors
*/
public DefaultIndexer(List<IndexDescriptor> indexDescriptors,
IndexEntryContainer oldIndexEntries) {
this.indexDescriptors = new ArrayList<>(indexDescriptors);
this.indexEntries = new IndexEntryContainer();
for (IndexEntry entry : oldIndexEntries) {
this.indexEntries.add(entry);
}
for (IndexDescriptor indexDescriptor : indexDescriptors) {
if (!indexDescriptor.isReady()) {
throw new IllegalArgumentException(
"Index descriptor is not ready for: " + indexDescriptor.getSpec().getName());
}
if (!this.indexEntries.contains(indexDescriptor)) {
throw new IllegalArgumentException(
"Index entry not found for: " + indexDescriptor.getSpec().getName());
}
}
}
static String getObjectKey(Extension extension) {
return PrimaryKeySpecUtils.getObjectPrimaryKey(extension);
}
@Override
public <E extends Extension> void indexRecord(E extension) {
writeLock.lock();
var transaction = new IndexerTransactionImpl();
try {
transaction.begin();
doIndexRecord(extension).forEach(transaction::add);
transaction.commit();
} catch (Throwable e) {
transaction.rollback();
throw e;
} finally {
writeLock.unlock();
}
}
@Override
public <E extends Extension> void updateRecord(E extension) {
writeLock.lock();
var transaction = new IndexerTransactionImpl();
try {
transaction.begin();
unIndexRecord(getObjectKey(extension));
indexRecord(extension);
transaction.commit();
} catch (Throwable e) {
transaction.rollback();
throw e;
} finally {
writeLock.unlock();
}
}
@Override
public void unIndexRecord(String extensionName) {
writeLock.lock();
var transaction = new IndexerTransactionImpl();
try {
transaction.begin();
doUnIndexRecord(extensionName).forEach(transaction::add);
transaction.commit();
} catch (Throwable e) {
transaction.rollback();
throw e;
} finally {
writeLock.unlock();
}
}
private List<ChangeRecord> doUnIndexRecord(String extensionName) {
List<ChangeRecord> changeRecords = new ArrayList<>();
for (IndexEntry indexEntry : indexEntries) {
indexEntry.entries().forEach(records -> {
var indexKey = records.getKey();
var objectKey = records.getValue();
if (objectKey.equals(extensionName)) {
changeRecords.add(ChangeRecord.onRemove(indexEntry, indexKey, objectKey));
}
});
}
return changeRecords;
}
private <E extends Extension> List<ChangeRecord> doIndexRecord(E extension) {
List<ChangeRecord> changeRecords = new ArrayList<>();
for (IndexDescriptor indexDescriptor : indexDescriptors) {
var indexEntry = indexEntries.get(indexDescriptor);
var indexFunc = indexDescriptor.getSpec().getIndexFunc();
Set<String> indexKeys = indexFunc.getValues(extension);
var objectKey = PrimaryKeySpecUtils.getObjectPrimaryKey(extension);
for (String indexKey : indexKeys) {
changeRecords.add(ChangeRecord.onAdd(indexEntry, indexKey, objectKey));
}
}
return changeRecords;
}
@Override
public IndexDescriptor findIndexByName(String name) {
readLock.lock();
try {
return indexDescriptors.stream()
.filter(descriptor -> descriptor.getSpec().getName().equals(name))
.findFirst()
.orElse(null);
} finally {
readLock.unlock();
}
}
@Override
public IndexEntry createIndexEntry(IndexDescriptor descriptor) {
return new IndexEntryImpl(descriptor);
}
@Override
public void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn) {
writeLock.lock();
try {
var iterator = indexEntries.iterator();
while (iterator.hasNext()) {
var entry = iterator.next();
if (BooleanUtils.isTrue(matchFn.apply(entry.getIndexDescriptor()))) {
iterator.remove();
entry.clear();
indexEntries.add(createIndexEntry(entry.getIndexDescriptor()));
}
}
} finally {
writeLock.unlock();
}
}
@Override
public Iterator<IndexEntry> readyIndexesIterator() {
readLock.lock();
try {
var readyIndexes = new ArrayList<IndexEntry>();
for (IndexEntry entry : indexEntries) {
if (entry.getIndexDescriptor().isReady()) {
readyIndexes.add(entry);
}
}
return readyIndexes.iterator();
} finally {
readLock.unlock();
}
}
@Override
public Iterator<IndexEntry> allIndexesIterator() {
readLock.lock();
try {
return indexEntries.iterator();
} finally {
readLock.unlock();
}
}
}

View File

@ -0,0 +1,17 @@
package run.halo.app.extension.index;
import java.util.Iterator;
import run.halo.app.extension.Extension;
/**
* An iterator over a collection of extensions, it is used to iterate extensions in a paginated
* way to avoid loading all extensions into memory at once.
*
* @param <E> the type of the extension.
* @author guqing
* @see DefaultExtensionIterator
* @since 2.12.0
*/
public interface ExtensionIterator<E extends Extension> extends Iterator<E> {
}

View File

@ -0,0 +1,23 @@
package run.halo.app.extension.index;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import run.halo.app.extension.Extension;
/**
* List extensions with pagination, used for {@link ExtensionIterator}.
*
* @author guqing
* @since 2.12.0
*/
public interface ExtensionPaginatedLister {
/**
* List extensions with pagination.
*
* @param pageable pageable
* @param <E> extension type
* @return page of extensions
*/
<E extends Extension> Page<E> list(Pageable pageable);
}

View File

@ -0,0 +1,26 @@
package run.halo.app.extension.index;
import org.springframework.lang.NonNull;
/**
* {@link IndexBuilder} is used to build index for a specific
* {@link run.halo.app.extension.Extension} type on startup.
*
* @author guqing
* @since 2.12.0
*/
public interface IndexBuilder {
/**
* Start building index for a specific {@link run.halo.app.extension.Extension} type.
*/
void startBuildingIndex();
/**
* Gets final index entries after building index.
*
* @return index entries must not be null.
* @throws IllegalStateException if any index entries are not ready yet.
*/
@NonNull
IndexEntryContainer getIndexEntries();
}

View File

@ -0,0 +1,65 @@
package run.halo.app.extension.index;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
public class IndexBuilderImpl implements IndexBuilder {
private final List<IndexDescriptor> indexDescriptors;
private final ExtensionIterator<? extends Extension> extensionIterator;
private final IndexEntryContainer indexEntries = new IndexEntryContainer();
public static IndexBuilder of(List<IndexDescriptor> indexDescriptors,
ExtensionIterator<? extends Extension> extensionIterator) {
return new IndexBuilderImpl(indexDescriptors, extensionIterator);
}
IndexBuilderImpl(List<IndexDescriptor> indexDescriptors,
ExtensionIterator<? extends Extension> extensionIterator) {
this.indexDescriptors = indexDescriptors;
this.extensionIterator = extensionIterator;
indexDescriptors.forEach(indexDescriptor -> {
var indexEntry = new IndexEntryImpl(indexDescriptor);
indexEntries.add(indexEntry);
});
}
@Override
public void startBuildingIndex() {
while (extensionIterator.hasNext()) {
var extensionRecord = extensionIterator.next();
indexRecords(extensionRecord);
}
for (IndexDescriptor indexDescriptor : indexDescriptors) {
indexDescriptor.setReady(true);
}
}
@Override
@NonNull
public IndexEntryContainer getIndexEntries() {
for (IndexEntry indexEntry : indexEntries) {
if (!indexEntry.getIndexDescriptor().isReady()) {
throw new IllegalStateException(
"IndexEntry are not ready yet for index named "
+ indexEntry.getIndexDescriptor().getSpec().getName());
}
}
return indexEntries;
}
private <E extends Extension> void indexRecords(E extension) {
for (IndexDescriptor indexDescriptor : indexDescriptors) {
var indexEntry = indexEntries.get(indexDescriptor);
var indexFunc = indexDescriptor.getSpec().getIndexFunc();
Set<String> indexKeys = indexFunc.getValues(extension);
indexEntry.addEntry(new LinkedList<>(indexKeys),
PrimaryKeySpecUtils.getObjectPrimaryKey(extension));
}
}
}

View File

@ -0,0 +1,20 @@
package run.halo.app.extension.index;
import lombok.Data;
import lombok.ToString;
@Data
@ToString(callSuper = true)
public class IndexDescriptor {
private final IndexSpec spec;
/**
* Record whether the index is ready, managed by {@link IndexBuilder}.
*/
private boolean ready;
public IndexDescriptor(IndexSpec spec) {
this.spec = spec;
}
}

View File

@ -0,0 +1,117 @@
package run.halo.app.extension.index;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import run.halo.app.extension.Metadata;
/**
* <p>{@link IndexEntry} used to store the mapping between index key and
* {@link Metadata#getName()}.</p>
* <p>For example, if we have a {@link Metadata} with name {@code foo} and labels {@code bar=1}
* and {@code baz=2}, then the index entry will be:</p>
* <pre>
* bar=1 -> foo
* baz=2 -> foo
* </pre>
* <p>And if we have another {@link Metadata} with name {@code bar} and labels {@code bar=1}
* and {@code baz=3}, then the index entry will be:</p>
* <pre>
* bar=1 -> foo, bar
* baz=2 -> foo
* baz=3 -> bar
* </pre>
* <p>{@link #getIndexDescriptor()} describes the owner of this index entry.</p>
* <p>Index entries is ordered by key, and the order is determined by
* {@link IndexSpec#getOrder()}.</p>
* <p>Do not modify the returned result for all methods of this class.</p>
* <p>This class is thread-safe.</p>
*
* @author guqing
* @since 2.12.0
*/
public interface IndexEntry {
/**
* Acquires the read lock for reading such as {@link #getByIndexKey(String)},
* {@link #entries()}, {@link #indexedKeys()}, because the returned result set of these
* methods is not immutable.
*/
void acquireReadLock();
/**
* Releases the read lock.
*/
void releaseReadLock();
/**
* <p>Adds a new entry to this index entry.</p>
* <p>For example, if we have a {@link Metadata} with name {@code foo} and labels {@code bar=1}
* and {@code baz=2} and index order is {@link IndexSpec.OrderType#ASC}, then the index entry
* will be:</p>
* <pre>
* bar=1 -> foo
* baz=2 -> foo
* </pre>
*
* @param indexKeys index keys
* @param objectKey object key (usually is {@link Metadata#getName()}).
*/
void addEntry(List<String> indexKeys, String objectKey);
/**
* Removes the entry with the given {@code indexedKey} and {@code objectKey}.
*
* @param indexedKey indexed key
* @param objectKey object key (usually is {@link Metadata#getName()}).
*/
void removeEntry(String indexedKey, String objectKey);
/**
* Removes all entries with the given {@code objectKey}.
*
* @param objectKey object key(usually is {@link Metadata#getName()}).
*/
void remove(String objectKey);
/**
* Returns the {@link IndexDescriptor} of this entry.
*
* @return the {@link IndexDescriptor} of this entry.
*/
IndexDescriptor getIndexDescriptor();
/**
* Returns the indexed keys of this entry in order.
*
* @return distinct indexed keys of this entry.
*/
Set<String> indexedKeys();
/**
* <p>Returns the entries of this entry in order.</p>
* <p>Note That: Any modification to the returned result will affect the original data
* directly.</p>
*
* @return entries of this entry.
*/
Collection<Map.Entry<String, String>> entries();
/**
* Returns the immutable entries of this entry in order, it is safe to modify the returned
* result, but extra cost is made.
*
* @return immutable entries of this entry.
*/
Collection<Map.Entry<String, String>> immutableEntries();
/**
* Returns the object names of this entry in order.
*
* @return object names of this entry.
*/
List<String> getByIndexKey(String indexKey);
void clear();
}

View File

@ -0,0 +1,71 @@
package run.halo.app.extension.index;
import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer;
import org.springframework.lang.NonNull;
/**
* <p>A container for {@link IndexEntry}s, it is used to store all {@link IndexEntry}s according
* to the {@link IndexDescriptor}.</p>
* <p>This class is thread-safe.</p>
*
* @author guqing
* @see DefaultIndexer
* @since 2.12.0
*/
public class IndexEntryContainer implements Iterable<IndexEntry> {
private final ConcurrentMap<IndexDescriptor, IndexEntry> indexEntryMap;
public IndexEntryContainer() {
this.indexEntryMap = new ConcurrentHashMap<>();
}
/**
* Add an {@link IndexEntry} to this container.
*
* @param entry the entry to add
* @throws IllegalArgumentException if the entry already exists
*/
public void add(IndexEntry entry) {
IndexEntry existing = indexEntryMap.putIfAbsent(entry.getIndexDescriptor(), entry);
if (existing != null) {
throw new IllegalArgumentException(
"Index entry already exists for " + entry.getIndexDescriptor());
}
}
/**
* Get the {@link IndexEntry} for the given {@link IndexDescriptor}.
*
* @param indexDescriptor the index descriptor
* @return the index entry
*/
public IndexEntry get(IndexDescriptor indexDescriptor) {
return indexEntryMap.get(indexDescriptor);
}
public boolean contains(IndexDescriptor indexDescriptor) {
return indexEntryMap.containsKey(indexDescriptor);
}
public void remove(IndexDescriptor indexDescriptor) {
indexEntryMap.remove(indexDescriptor);
}
public int size() {
return indexEntryMap.size();
}
@Override
@NonNull
public Iterator<IndexEntry> iterator() {
return indexEntryMap.values().iterator();
}
@Override
public void forEach(Consumer<? super IndexEntry> action) {
indexEntryMap.values().forEach(action);
}
}

View File

@ -0,0 +1,179 @@
package run.halo.app.extension.index;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.MultimapBuilder;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import lombok.Data;
import run.halo.app.infra.exception.DuplicateNameException;
@Data
public class IndexEntryImpl implements IndexEntry {
private final ReadWriteLock rwl = new ReentrantReadWriteLock();
private final Lock readLock = rwl.readLock();
private final Lock writeLock = rwl.writeLock();
private final IndexDescriptor indexDescriptor;
private final ListMultimap<String, String> indexKeyObjectNamesMap;
/**
* Creates a new {@link IndexEntryImpl} for the given {@link IndexDescriptor}.
*
* @param indexDescriptor for which the {@link IndexEntryImpl} is created.
*/
public IndexEntryImpl(IndexDescriptor indexDescriptor) {
this.indexDescriptor = indexDescriptor;
this.indexKeyObjectNamesMap = MultimapBuilder.treeKeys(keyComparator())
.linkedListValues().build();
}
Comparator<String> keyComparator() {
var order = indexDescriptor.getSpec().getOrder();
if (IndexSpec.OrderType.ASC.equals(order)) {
return KeyComparator.INSTANCE;
}
return KeyComparator.INSTANCE.reversed();
}
static class KeyComparator implements Comparator<String> {
public static final KeyComparator INSTANCE = new KeyComparator();
@Override
public int compare(String a, String b) {
int i = 0;
int j = 0;
while (i < a.length() && j < b.length()) {
if (Character.isDigit(a.charAt(i)) && Character.isDigit(b.charAt(j))) {
// handle number part
int num1 = 0;
int num2 = 0;
while (i < a.length() && Character.isDigit(a.charAt(i))) {
num1 = num1 * 10 + (a.charAt(i++) - '0');
}
while (j < b.length() && Character.isDigit(b.charAt(j))) {
num2 = num2 * 10 + (b.charAt(j++) - '0');
}
if (num1 != num2) {
return num1 - num2;
}
} else if (a.charAt(i) != b.charAt(j)) {
// handle non-number part
return a.charAt(i) - b.charAt(j);
} else {
i++;
j++;
}
}
return a.length() - b.length();
}
}
@Override
public void acquireReadLock() {
this.rwl.readLock().lock();
}
@Override
public void releaseReadLock() {
this.rwl.readLock().unlock();
}
@Override
public void addEntry(List<String> keys, String objectName) {
var isUnique = indexDescriptor.getSpec().isUnique();
for (String key : keys) {
writeLock.lock();
try {
if (isUnique && indexKeyObjectNamesMap.containsKey(key)) {
throw new DuplicateNameException(
"The value [%s] is already exists for unique index [%s].".formatted(
key,
indexDescriptor.getSpec().getName()),
null,
"problemDetail.index.duplicateKey",
new Object[] {key, indexDescriptor.getSpec().getName()});
}
this.indexKeyObjectNamesMap.put(key, objectName);
} finally {
writeLock.unlock();
}
}
}
@Override
public void removeEntry(String indexedKey, String objectKey) {
writeLock.lock();
try {
indexKeyObjectNamesMap.remove(indexedKey, objectKey);
} finally {
writeLock.unlock();
}
}
@Override
public void remove(String objectName) {
writeLock.lock();
try {
indexKeyObjectNamesMap.values().removeIf(objectName::equals);
} finally {
writeLock.unlock();
}
}
@Override
public Set<String> indexedKeys() {
readLock.lock();
try {
return indexKeyObjectNamesMap.keySet();
} finally {
readLock.unlock();
}
}
@Override
public Collection<Map.Entry<String, String>> entries() {
readLock.lock();
try {
return indexKeyObjectNamesMap.entries();
} finally {
readLock.unlock();
}
}
@Override
public Collection<Map.Entry<String, String>> immutableEntries() {
readLock.lock();
try {
return ImmutableListMultimap.copyOf(indexKeyObjectNamesMap).entries();
} finally {
readLock.unlock();
}
}
@Override
public List<String> getByIndexKey(String indexKey) {
readLock.lock();
try {
return indexKeyObjectNamesMap.get(indexKey);
} finally {
readLock.unlock();
}
}
@Override
public void clear() {
writeLock.lock();
try {
indexKeyObjectNamesMap.clear();
} finally {
writeLock.unlock();
}
}
}

View File

@ -0,0 +1,85 @@
package run.halo.app.extension.index;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionStoreUtil;
import run.halo.app.extension.Scheme;
/**
* <p>A default implementation of {@link IndexSpecRegistry}.</p>
*
* @author guqing
* @since 2.12.0
*/
public class IndexSpecRegistryImpl implements IndexSpecRegistry {
private final ConcurrentMap<String, IndexSpecs> extensionIndexSpecs = new ConcurrentHashMap<>();
@Override
public IndexSpecs indexFor(Scheme scheme) {
var keySpace = getKeySpace(scheme);
var indexSpecs = new DefaultIndexSpecs();
useDefaultIndexSpec(scheme.type(), indexSpecs);
extensionIndexSpecs.put(keySpace, indexSpecs);
return indexSpecs;
}
@Override
public IndexSpecs getIndexSpecs(Scheme scheme) {
var keySpace = getKeySpace(scheme);
var result = extensionIndexSpecs.get(keySpace);
if (result == null) {
throw new IllegalArgumentException(
"No index specs found for extension type: " + scheme.groupVersionKind()
+ ", make sure you have called indexFor() before calling getIndexSpecs()");
}
return result;
}
@Override
public boolean contains(Scheme scheme) {
var keySpace = getKeySpace(scheme);
return extensionIndexSpecs.containsKey(keySpace);
}
@Override
public void removeIndexSpecs(Scheme scheme) {
var keySpace = getKeySpace(scheme);
extensionIndexSpecs.remove(keySpace);
}
@Override
@NonNull
public String getKeySpace(Scheme scheme) {
return ExtensionStoreUtil.buildStoreNamePrefix(scheme);
}
<E extends Extension> void useDefaultIndexSpec(Class<E> extensionType,
IndexSpecs indexSpecs) {
var nameIndexSpec = PrimaryKeySpecUtils.primaryKeyIndexSpec(extensionType);
indexSpecs.add(nameIndexSpec);
var creationTimestampIndexSpec = new IndexSpec()
.setName("metadata.creationTimestamp")
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(false)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(extensionType,
e -> e.getMetadata().getCreationTimestamp().toString())
);
indexSpecs.add(creationTimestampIndexSpec);
var deletionTimestampIndexSpec = new IndexSpec()
.setName("metadata.deletionTimestamp")
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(false)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(extensionType,
e -> Objects.toString(e.getMetadata().getDeletionTimestamp(), null))
);
indexSpecs.add(deletionTimestampIndexSpec);
indexSpecs.add(LabelIndexSpecUtils.labelIndexSpec(extensionType));
}
}

View File

@ -0,0 +1,296 @@
package run.halo.app.extension.index;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StopWatch;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.index.query.Query;
import run.halo.app.extension.index.query.QueryIndexViewImpl;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
import run.halo.app.extension.router.selector.SelectorMatcher;
/**
* A default implementation of {@link IndexedQueryEngine}.
*
* @author guqing
* @since 2.12.0
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class IndexedQueryEngineImpl implements IndexedQueryEngine {
private final IndexerFactory indexerFactory;
private static Map<String, IndexEntry> fieldPathIndexEntryMap(Indexer indexer) {
// O(n) time complexity
Map<String, IndexEntry> indexEntryMap = new HashMap<>();
var iterator = indexer.readyIndexesIterator();
while (iterator.hasNext()) {
var indexEntry = iterator.next();
var descriptor = indexEntry.getIndexDescriptor();
var indexedFieldPath = descriptor.getSpec().getName();
indexEntryMap.put(indexedFieldPath, indexEntry);
}
return indexEntryMap;
}
static IndexEntry getIndexEntry(String fieldPath, Map<String, IndexEntry> fieldPathEntryMap) {
if (!fieldPathEntryMap.containsKey(fieldPath)) {
throwNotIndexedException(fieldPath);
}
return fieldPathEntryMap.get(fieldPath);
}
@Override
public ListResult<String> retrieve(GroupVersionKind type, ListOptions options,
PageRequest page) {
var indexer = indexerFactory.getIndexer(type);
var allMatchedResult = doRetrieve(indexer, options, page.getSort());
var list = ListResult.subList(allMatchedResult, page.getPageNumber(), page.getPageSize());
return new ListResult<>(page.getPageNumber(), page.getPageSize(),
allMatchedResult.size(), list);
}
@Override
public List<String> retrieveAll(GroupVersionKind type, ListOptions options) {
var indexer = indexerFactory.getIndexer(type);
return doRetrieve(indexer, options, Sort.unsorted());
}
static <T> List<T> intersection(List<T> list1, List<T> list2) {
Set<T> set = new LinkedHashSet<>(list1);
List<T> intersection = new ArrayList<>();
for (T item : list2) {
if (set.contains(item) && !intersection.contains(item)) {
intersection.add(item);
}
}
return intersection;
}
static void throwNotIndexedException(String fieldPath) {
throw new IllegalArgumentException(
"No index found for fieldPath: " + fieldPath
+ ", make sure you have created an index for this field.");
}
List<String> retrieveForLabelMatchers(List<SelectorMatcher> labelMatchers,
Map<String, IndexEntry> fieldPathEntryMap, List<String> allMetadataNames) {
var indexEntry = getIndexEntry(LabelIndexSpecUtils.LABEL_PATH, fieldPathEntryMap);
// O(m) time complexity, m is the number of labelMatchers
var labelKeysToQuery = labelMatchers.stream()
.sorted(Comparator.comparing(SelectorMatcher::getKey))
.map(SelectorMatcher::getKey)
.collect(Collectors.toSet());
Map<String, Map<String, String>> objectNameLabelsMap = new HashMap<>();
indexEntry.acquireReadLock();
try {
indexEntry.entries().forEach(entry -> {
// key is labelKey=labelValue, value is objectName
var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey());
if (!labelKeysToQuery.contains(labelPair.getFirst())) {
return;
}
objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>())
.put(labelPair.getFirst(), labelPair.getSecond());
});
} finally {
indexEntry.releaseReadLock();
}
// O(p * m) time complexity, p is the number of allMetadataNames
return allMetadataNames.stream()
.filter(objectName -> {
var labels = objectNameLabelsMap.getOrDefault(objectName, Map.of());
// object match all labels will be returned
return labelMatchers.stream()
.allMatch(matcher -> matcher.test(labels.get(matcher.getKey())));
})
.toList();
}
List<String> doRetrieve(Indexer indexer, ListOptions options, Sort sort) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("build index entry map");
var fieldPathEntryMap = fieldPathIndexEntryMap(indexer);
stopWatch.stop();
var primaryEntry = getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, fieldPathEntryMap);
// O(n) time complexity
stopWatch.start("retrieve all metadata names");
var allMetadataNames = new ArrayList<String>();
primaryEntry.acquireReadLock();
try {
allMetadataNames.addAll(primaryEntry.indexedKeys());
} finally {
primaryEntry.releaseReadLock();
}
stopWatch.stop();
stopWatch.start("retrieve matched metadata names");
var hasLabelSelector = hasLabelSelector(options.getLabelSelector());
final List<String> matchedByLabels = hasLabelSelector
? retrieveForLabelMatchers(options.getLabelSelector().getMatchers(), fieldPathEntryMap,
allMetadataNames)
: allMetadataNames;
stopWatch.stop();
stopWatch.start("retrieve matched metadata names by fields");
final var hasFieldSelector = hasFieldSelector(options.getFieldSelector());
var matchedByFields = hasFieldSelector
? retrieveForFieldSelector(options.getFieldSelector().query(), fieldPathEntryMap)
: allMetadataNames;
stopWatch.stop();
stopWatch.start("merge result");
List<String> foundObjectKeys;
if (!hasLabelSelector && !hasFieldSelector) {
foundObjectKeys = allMetadataNames;
} else if (!hasLabelSelector) {
foundObjectKeys = matchedByFields;
} else {
foundObjectKeys = intersection(matchedByFields, matchedByLabels);
}
stopWatch.stop();
stopWatch.start("sort result");
ResultSorter resultSorter = new ResultSorter(fieldPathEntryMap, foundObjectKeys);
var result = resultSorter.sortBy(sort);
stopWatch.stop();
if (log.isTraceEnabled()) {
log.trace("Retrieve result from indexer, {}", stopWatch.prettyPrint());
}
return result;
}
boolean hasLabelSelector(LabelSelector labelSelector) {
return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers());
}
boolean hasFieldSelector(FieldSelector fieldSelector) {
return fieldSelector != null && fieldSelector.query() != null;
}
List<String> retrieveForFieldSelector(Query query, Map<String, IndexEntry> fieldPathEntryMap) {
Map<String, Collection<Map.Entry<String, String>>> indexView = new HashMap<>();
for (Map.Entry<String, IndexEntry> entry : fieldPathEntryMap.entrySet()) {
indexView.put(entry.getKey(), entry.getValue().immutableEntries());
}
// TODO optimize build indexView time
var queryIndexView = new QueryIndexViewImpl(indexView);
var resultSet = query.matches(queryIndexView);
return new ArrayList<>(resultSet);
}
/**
* Sort the given list by the given {@link Sort}.
*/
static class ResultSorter {
private final Map<String, IndexEntry> fieldPathEntryMap;
private final List<String> list;
public ResultSorter(Map<String, IndexEntry> fieldPathEntryMap, List<String> list) {
this.fieldPathEntryMap = fieldPathEntryMap;
this.list = list;
}
public List<String> sortBy(@NonNull Sort sort) {
if (sort.isUnsorted()) {
return list;
}
var sortedLists = new ArrayList<List<String>>();
for (Sort.Order order : sort) {
var indexEntry = fieldPathEntryMap.get(order.getProperty());
if (indexEntry == null) {
throwNotIndexedException(order.getProperty());
}
var set = new HashSet<>(list);
var objectNames = new ArrayList<String>();
indexEntry.acquireReadLock();
try {
for (var entry : indexEntry.entries()) {
var objectName = entry.getValue();
if (set.contains(objectName)) {
objectNames.add(objectName);
}
}
} finally {
indexEntry.releaseReadLock();
}
var indexOrder = indexEntry.getIndexDescriptor().getSpec().getOrder();
var asc = IndexSpec.OrderType.ASC.equals(indexOrder);
if (asc != order.isAscending()) {
Collections.reverse(objectNames);
}
sortedLists.add(objectNames);
}
return mergeSortedLists(sortedLists);
}
/**
* <p>Merge the given sorted lists into one sorted list.</p>
* <p>The time complexity is O(n * log(m)), n is the number of all elements in the
* sortedLists, m is the number of sortedLists.</p>
*/
private List<String> mergeSortedLists(List<List<String>> sortedLists) {
List<String> result = new ArrayList<>();
// Use a priority queue to store the current element of each list and its index in
// the list
PriorityQueue<Pair> minHeap = new PriorityQueue<>(
Comparator.comparing(pair -> pair.value));
// Initialize the priority queue and add the first element of each list to the queue
for (int i = 0; i < sortedLists.size(); i++) {
if (!sortedLists.get(i).isEmpty()) {
minHeap.add(new Pair(i, 0, sortedLists.get(i).get(0)));
}
}
while (!minHeap.isEmpty()) {
Pair current = minHeap.poll();
result.add(current.value());
// Add the next element of this list to the priority queue
if (current.indexInList() + 1 < sortedLists.get(current.listIndex()).size()) {
var list = sortedLists.get(current.listIndex());
minHeap.add(new Pair(current.listIndex(),
current.indexInList() + 1,
list.get(current.indexInList() + 1))
);
}
}
return result;
}
/**
* <p>A pair of element and its position in the original list.</p>
* <pre>
* listIndex: column index.
* indexInList: element index in the list.
* value: element value.
* </pre>
*/
private record Pair(int listIndex, int indexInList, String value) {
}
}
}

View File

@ -0,0 +1,91 @@
package run.halo.app.extension.index;
import java.util.Iterator;
import java.util.function.Function;
import run.halo.app.extension.Extension;
/**
* <p>The {@link Indexer} is owned by the {@link Extension} and is responsible for the lookup and
* lifetimes of the indexes in a {@link Extension} collection. Every {@link Extension} has
* exactly one instance of this class.</p>
* <p>Callers are expected to have acquired the necessary locks while accessing this interface.</p>
* To inspect the contents of this {@link Indexer}, callers may obtain an iterator from
* getIndexIterator().
*
* @author guqing
* @since 2.12.0
*/
public interface Indexer {
/**
* <p>Index the specified {@link Extension} by {@link IndexDescriptor}s.</p>
* <p>First, the {@link Indexer} will index the {@link Extension} by the
* {@link IndexDescriptor}s and record the index entries to {@link IndexerTransaction} and
* commit the transaction, if any error occurs, the transaction will be rollback to keep the
* {@link Indexer} consistent.</p>
*
* @param extension the {@link Extension} to be indexed
* @param <E> the type of the {@link Extension}
*/
<E extends Extension> void indexRecord(E extension);
/**
* <p>Update indexes for the specified {@link Extension} by {@link IndexDescriptor}s.</p>
* <p>First, the {@link Indexer} will remove the index entries of the {@link Extension} by
* the old {@link IndexDescriptor}s and reindex the {@link Extension} to generate change logs
* to {@link IndexerTransaction} and commit the transaction, if any error occurs, the
* transaction will be rollback to keep the {@link Indexer} consistent.</p>
*
* @param extension the {@link Extension} to be updated
* @param <E> the type of the {@link Extension}
*/
<E extends Extension> void updateRecord(E extension);
/**
* <p>Remove indexes (index entries) for the specified {@link Extension} already indexed by
* {@link IndexDescriptor}s.</p>
*
* @param extensionName the {@link Extension} to be removed
*/
void unIndexRecord(String extensionName);
/**
* <p>Find index by name.</p>
* <p>The index name uniquely identifies an index.</p>
*
* @param name index name
* @return index descriptor if found, null otherwise
*/
IndexDescriptor findIndexByName(String name);
/**
* <p>Create an index entry for the specified {@link IndexDescriptor}.</p>
*
* @param descriptor the {@link IndexDescriptor} to be recorded
* @return the {@link IndexEntry} created
*/
IndexEntry createIndexEntry(IndexDescriptor descriptor);
/**
* <p>Remove all index entries that match the given {@link IndexDescriptor}.</p>
*
* @param matchFn the {@link IndexDescriptor} to be matched
*/
void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn);
/**
* <p>Gets an iterator over all the ready {@link IndexEntry}s, in no particular order.</p>
*
* @return an iterator over all the ready {@link IndexEntry}s
* @link {@link IndexDescriptor#isReady()}
*/
Iterator<IndexEntry> readyIndexesIterator();
/**
* <p>Gets an iterator over all the {@link IndexEntry}s, in no particular order.</p>
*
* @return an iterator over all the {@link IndexEntry}s
* @link {@link IndexDescriptor#isReady()}
*/
Iterator<IndexEntry> allIndexesIterator();
}

View File

@ -0,0 +1,56 @@
package run.halo.app.extension.index;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionStoreUtil;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.Scheme;
import run.halo.app.extension.SchemeManager;
/**
* <p>{@link IndexerFactory} is used to create {@link Indexer} for {@link Extension} type.</p>
* <p>It's stored {@link Indexer} by key space, the key space is generated by {@link Scheme} like
* {@link ExtensionStoreUtil#buildStoreNamePrefix(Scheme)}.</p>
* <p>E.g. create {@link Indexer} for Post extension, the mapping relationship is:</p>
* <pre>
* /registry/content.halo.run/posts -> Indexer
* </pre>
*
* @author guqing
* @since 2.12.0
*/
public interface IndexerFactory {
/**
* Create {@link Indexer} for {@link Extension} type.
*
* @param extensionType the extension type must exist in {@link SchemeManager}.
* @param extensionIterator the extension iterator to iterate all records for the extension type
* @return created {@link Indexer}
*/
@NonNull
Indexer createIndexerFor(Class<? extends Extension> extensionType,
ExtensionIterator<? extends Extension> extensionIterator);
/**
* Get {@link Indexer} for {@link GroupVersionKind}.
*
* @param gvk the group version kind must exist in {@link SchemeManager}
* @return the indexer
* @throws IllegalArgumentException if the {@link GroupVersionKind} represents a special
* {@link Extension} not exists in {@link SchemeManager}
*/
@NonNull
Indexer getIndexer(GroupVersionKind gvk);
boolean contains(GroupVersionKind gvk);
/**
* <p>Remove a specific {@link Indexer} by {@link Scheme} that represents a {@link Extension}
* .</p>
* <p>Usually, the specified {@link Scheme} is not in {@link SchemeManager} at this time.</p>
*
* @param scheme the scheme represents a {@link Extension}
*/
void removeIndexer(Scheme scheme);
}

View File

@ -0,0 +1,84 @@
package run.halo.app.extension.index;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionStoreUtil;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.Scheme;
import run.halo.app.extension.SchemeManager;
/**
* <p>A default implementation of {@link IndexerFactory}.</p>
*
* @author guqing
* @since 2.12.0
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class IndexerFactoryImpl implements IndexerFactory {
private final ConcurrentMap<String, Indexer> keySpaceIndexer = new ConcurrentHashMap<>();
private final IndexSpecRegistry indexSpecRegistry;
private final SchemeManager schemeManager;
@Override
@NonNull
public Indexer createIndexerFor(Class<? extends Extension> extensionType,
ExtensionIterator<? extends Extension> extensionIterator) {
var scheme = schemeManager.get(extensionType);
var keySpace = indexSpecRegistry.getKeySpace(scheme);
if (keySpaceIndexer.containsKey(keySpace)) {
throw new IllegalArgumentException("Indexer already exists for type: " + keySpace);
}
if (!indexSpecRegistry.contains(scheme)) {
indexSpecRegistry.indexFor(scheme);
}
var specs = indexSpecRegistry.getIndexSpecs(scheme);
var indexDescriptors = specs.getIndexSpecs()
.stream()
.map(IndexDescriptor::new)
.toList();
final long startTimeMs = System.currentTimeMillis();
log.info("Start building index for type: {}, please wait...", keySpace);
var indexBuilder = IndexBuilderImpl.of(indexDescriptors, extensionIterator);
indexBuilder.startBuildingIndex();
var indexer =
new DefaultIndexer(indexDescriptors, indexBuilder.getIndexEntries());
keySpaceIndexer.put(keySpace, indexer);
log.info("Index for type: {} built successfully, cost {} ms", keySpace,
System.currentTimeMillis() - startTimeMs);
return indexer;
}
@Override
@NonNull
public Indexer getIndexer(GroupVersionKind gvk) {
var scheme = schemeManager.get(gvk);
var indexer = keySpaceIndexer.get(indexSpecRegistry.getKeySpace(scheme));
if (indexer == null) {
throw new IllegalArgumentException("No indexer found for type: " + gvk);
}
return indexer;
}
@Override
public boolean contains(GroupVersionKind gvk) {
var schemeOpt = schemeManager.fetch(gvk);
return schemeOpt.isPresent()
&& keySpaceIndexer.containsKey(indexSpecRegistry.getKeySpace(schemeOpt.get()));
}
@Override
public void removeIndexer(Scheme scheme) {
var keySpace = ExtensionStoreUtil.buildStoreNamePrefix(scheme);
keySpaceIndexer.remove(keySpace);
indexSpecRegistry.removeIndexSpecs(scheme);
}
}

View File

@ -0,0 +1,40 @@
package run.halo.app.extension.index;
import org.springframework.util.Assert;
/**
* <p>{@link IndexerTransaction} is a transactional interface for {@link Indexer} to ensure
* consistency when {@link Indexer} indexes objects.</p>
* <p>It is not supported to call {@link #begin()} twice without calling {@link #commit()} or
* {@link #rollback()} in between and it is not supported to call one of {@link #commit()} or
* {@link #rollback()} in different thread than {@link #begin()} was called.</p>
*
* @author guqing
* @since 2.12.0
*/
public interface IndexerTransaction {
void begin();
void commit();
void rollback();
void add(ChangeRecord changeRecord);
record ChangeRecord(IndexEntry indexEntry, String key, String value, boolean isAdd) {
public ChangeRecord {
Assert.notNull(indexEntry, "IndexEntry must not be null");
Assert.notNull(key, "Key must not be null");
Assert.notNull(value, "Value must not be null");
}
public static ChangeRecord onAdd(IndexEntry indexEntry, String key, String value) {
return new ChangeRecord(indexEntry, key, value, true);
}
public static ChangeRecord onRemove(IndexEntry indexEntry, String key, String value) {
return new ChangeRecord(indexEntry, key, value, false);
}
}
}

View File

@ -0,0 +1,105 @@
package run.halo.app.extension.index;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
/**
* Implementation of {@link IndexerTransaction}.
*
* @author guqing
* @since 2.12.0
*/
public class IndexerTransactionImpl implements IndexerTransaction {
private Deque<ChangeRecord> changeRecords;
private boolean inTransaction = false;
private Long threadId;
@Override
public synchronized void begin() {
if (inTransaction) {
throw new IllegalStateException("Transaction already active");
}
threadId = Thread.currentThread().getId();
this.changeRecords = new ArrayDeque<>();
inTransaction = true;
}
@Override
public synchronized void commit() {
checkThread();
if (!inTransaction) {
throw new IllegalStateException("Transaction not started");
}
Deque<ChangeRecord> committedRecords = new ArrayDeque<>();
try {
while (!changeRecords.isEmpty()) {
var changeRecord = changeRecords.pop();
applyChange(changeRecord);
committedRecords.push(changeRecord);
}
// Reset threadId after transaction ends
inTransaction = false;
// Reset threadId after transaction ends
threadId = null;
} catch (Exception e) {
// Rollback the changes that were committed before the error occurred
while (!committedRecords.isEmpty()) {
var changeRecord = committedRecords.pop();
revertChange(changeRecord);
}
throw e;
}
}
@Override
public synchronized void rollback() {
checkThread();
if (!inTransaction) {
throw new IllegalStateException("Transaction not started");
}
changeRecords.clear();
inTransaction = false;
// Reset threadId after transaction ends
threadId = null;
}
@Override
public synchronized void add(ChangeRecord changeRecord) {
if (inTransaction) {
changeRecords.push(changeRecord);
} else {
throw new IllegalStateException("No active transaction to add changes");
}
}
private void applyChange(ChangeRecord changeRecord) {
var indexEntry = changeRecord.indexEntry();
var key = changeRecord.key();
var value = changeRecord.value();
var isAdd = changeRecord.isAdd();
if (isAdd) {
indexEntry.addEntry(List.of(key), value);
} else {
indexEntry.removeEntry(key, value);
}
}
private void revertChange(ChangeRecord changeRecord) {
var indexEntry = changeRecord.indexEntry();
var key = changeRecord.key();
var value = changeRecord.value();
var isAdd = changeRecord.isAdd();
if (isAdd) {
indexEntry.removeEntry(key, value);
} else {
indexEntry.addEntry(List.of(key), value);
}
}
private void checkThread() {
if (threadId != null && !threadId.equals(Thread.currentThread().getId())) {
throw new IllegalStateException("Transaction cannot span multiple threads!");
}
}
}

View File

@ -0,0 +1,55 @@
package run.halo.app.extension.index;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.experimental.UtilityClass;
import org.springframework.data.util.Pair;
import run.halo.app.extension.Extension;
@UtilityClass
public class LabelIndexSpecUtils {
public static final String LABEL_PATH = "metadata.labels";
/**
* Creates a label index spec.
*
* @param extensionType extension type
* @param <E> extension type
* @return label index spec
*/
public static <E extends Extension> IndexSpec labelIndexSpec(Class<E> extensionType) {
return new IndexSpec()
.setName(LABEL_PATH)
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(false)
.setIndexFunc(IndexAttributeFactory.multiValueAttribute(extensionType,
LabelIndexSpecUtils::labelIndexValueFunc)
);
}
/**
* Label key-value pair from indexed label key string, e.g. "key=value".
*
* @param indexedLabelKey indexed label key
* @return label key-value pair
*/
public static Pair<String, String> labelKeyValuePair(String indexedLabelKey) {
var idx = indexedLabelKey.indexOf('=');
if (idx != -1) {
return Pair.of(indexedLabelKey.substring(0, idx), indexedLabelKey.substring(idx + 1));
}
throw new IllegalArgumentException("Invalid label key-value pair: " + indexedLabelKey);
}
static <E extends Extension> Set<String> labelIndexValueFunc(E obj) {
var labels = obj.getMetadata().getLabels();
if (labels == null) {
return Set.of();
}
return labels.entrySet()
.stream()
.map(entry -> entry.getKey() + "=" + entry.getValue())
.collect(Collectors.toSet());
}
}

View File

@ -0,0 +1,30 @@
package run.halo.app.extension.index;
import lombok.experimental.UtilityClass;
import run.halo.app.extension.Extension;
@UtilityClass
public class PrimaryKeySpecUtils {
public static final String PRIMARY_INDEX_NAME = "metadata.name";
/**
* Primary key index spec.
*
* @param type the type
* @param <E> the type parameter of {@link Extension}
* @return the index spec
*/
public static <E extends Extension> IndexSpec primaryKeyIndexSpec(Class<E> type) {
return new IndexSpec()
.setName(PRIMARY_INDEX_NAME)
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(true)
.setIndexFunc(IndexAttributeFactory.simpleAttribute(type,
e -> e.getMetadata().getName())
);
}
public static String getObjectPrimaryKey(Extension obj) {
return obj.getMetadata().getName();
}
}

View File

@ -25,11 +25,10 @@ class ExtensionListHandler implements ListHandler {
@NonNull
public Mono<ServerResponse> handle(@NonNull ServerRequest request) {
var queryParams = new SortableRequest(request.exchange());
return client.list(scheme.type(),
queryParams.toPredicate(),
queryParams.toComparator(),
queryParams.getPage(),
queryParams.getSize())
return client.listBy(scheme.type(),
queryParams.toListOptions(),
queryParams.toPageRequest()
)
.flatMap(listResult -> ServerResponse
.ok()
.contentType(MediaType.APPLICATION_JSON)

View File

@ -2,6 +2,8 @@ package run.halo.app.extension.store;
import java.util.List;
import java.util.Optional;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
/**
* An interface to query and operate ExtensionStore.
@ -18,6 +20,10 @@ public interface ExtensionStoreClient {
*/
List<ExtensionStore> listByNamePrefix(String prefix);
Page<ExtensionStore> listByNamePrefix(String prefix, Pageable pageable);
List<ExtensionStore> listByNames(List<String> names);
/**
* Fetches an ExtensionStore by unique name.
*

View File

@ -1,10 +1,10 @@
package run.halo.app.extension.store;
import jakarta.persistence.EntityNotFoundException;
import java.util.List;
import java.util.Optional;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Mono;
/**
* An implementation of ExtensionStoreClient using JPA.
@ -14,44 +14,44 @@ import reactor.core.publisher.Mono;
@Service
public class ExtensionStoreClientJPAImpl implements ExtensionStoreClient {
private final ExtensionStoreRepository repository;
private final ReactiveExtensionStoreClient storeClient;
public ExtensionStoreClientJPAImpl(ExtensionStoreRepository repository) {
this.repository = repository;
public ExtensionStoreClientJPAImpl(ReactiveExtensionStoreClient storeClient) {
this.storeClient = storeClient;
}
@Override
public List<ExtensionStore> listByNamePrefix(String prefix) {
return repository.findAllByNameStartingWith(prefix).collectList().block();
return storeClient.listByNamePrefix(prefix).collectList().block();
}
@Override
public Page<ExtensionStore> listByNamePrefix(String prefix, Pageable pageable) {
return storeClient.listByNamePrefix(prefix, pageable).block();
}
@Override
public List<ExtensionStore> listByNames(List<String> names) {
return storeClient.listByNames(names).collectList().block();
}
@Override
public Optional<ExtensionStore> fetchByName(String name) {
return repository.findById(name).blockOptional();
return storeClient.fetchByName(name).blockOptional();
}
@Override
public ExtensionStore create(String name, byte[] data) {
var store = new ExtensionStore(name, data);
return repository.save(store).block();
return storeClient.create(name, data).block();
}
@Override
public ExtensionStore update(String name, Long version, byte[] data) {
var store = new ExtensionStore(name, data, version);
return repository.save(store).block();
return storeClient.update(name, version, data).block();
}
@Override
public ExtensionStore delete(String name, Long version) {
return repository.findById(name)
.switchIfEmpty(Mono.error(() -> new EntityNotFoundException(
"Extension store with name " + name + " was not found.")))
.flatMap(deleting -> {
deleting.setVersion(version);
return repository.delete(deleting).thenReturn(deleting);
})
.block();
return storeClient.delete(name, version).block();
}
}

View File

@ -1,8 +1,11 @@
package run.halo.app.extension.store;
import java.util.List;
import org.springframework.data.domain.Pageable;
import org.springframework.data.r2dbc.repository.R2dbcRepository;
import org.springframework.stereotype.Repository;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* This repository contains some basic operations on ExtensionStore entity.
@ -20,4 +23,16 @@ public interface ExtensionStoreRepository extends R2dbcRepository<ExtensionStore
*/
Flux<ExtensionStore> findAllByNameStartingWith(String prefix);
Flux<ExtensionStore> findAllByNameStartingWith(String prefix, Pageable pageable);
Mono<Long> countByNameStartingWith(String prefix);
/**
* <p>Finds all ExtensionStore by name in, the result no guarantee the same order as the given
* names, so if you want this, please order the result by yourself.</p>
*
* @param names names to find
* @return a flux of extension stores
*/
Flux<ExtensionStore> findByNameIn(List<String> names);
}

View File

@ -1,5 +1,8 @@
package run.halo.app.extension.store;
import java.util.List;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
@ -7,6 +10,16 @@ public interface ReactiveExtensionStoreClient {
Flux<ExtensionStore> listByNamePrefix(String prefix);
Mono<Page<ExtensionStore>> listByNamePrefix(String prefix, Pageable pageable);
/**
* List stores by names and return data according to given names order.
*
* @param names store names to list
* @return a flux of extension stores
*/
Flux<ExtensionStore> listByNames(List<String> names);
Mono<ExtensionStore> fetchByName(String name);
Mono<ExtensionStore> create(String name, byte[] data);

View File

@ -1,6 +1,12 @@
package run.halo.app.extension.store;
import java.util.Comparator;
import java.util.List;
import java.util.function.ToIntFunction;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import reactor.core.publisher.Flux;
@ -21,6 +27,22 @@ public class ReactiveExtensionStoreClientImpl implements ReactiveExtensionStoreC
return repository.findAllByNameStartingWith(prefix);
}
@Override
public Mono<Page<ExtensionStore>> listByNamePrefix(String prefix, Pageable pageable) {
return this.repository.findAllByNameStartingWith(prefix, pageable)
.collectList()
.zipWith(this.repository.countByNameStartingWith(prefix))
.map(p -> new PageImpl<>(p.getT1(), pageable, p.getT2()));
}
@Override
public Flux<ExtensionStore> listByNames(List<String> names) {
ToIntFunction<ExtensionStore> comparator =
store -> names.indexOf(store.getName());
return repository.findByNameIn(names)
.sort(Comparator.comparingInt(comparator));
}
@Override
public Mono<ExtensionStore> fetchByName(String name) {
return repository.findById(name);

View File

@ -38,6 +38,7 @@ import run.halo.app.extension.ConfigMap;
import run.halo.app.extension.DefaultSchemeManager;
import run.halo.app.extension.DefaultSchemeWatcherManager;
import run.halo.app.extension.Secret;
import run.halo.app.extension.index.IndexSpecRegistryImpl;
import run.halo.app.migration.Backup;
import run.halo.app.plugin.extensionpoint.ExtensionDefinition;
import run.halo.app.plugin.extensionpoint.ExtensionPointDefinition;
@ -49,12 +50,7 @@ public class SchemeInitializer implements ApplicationListener<ApplicationContext
@Override
public void onApplicationEvent(@NonNull ApplicationContextInitializedEvent event) {
var watcherManager = new DefaultSchemeWatcherManager();
var schemeManager = new DefaultSchemeManager(watcherManager);
var beanFactory = event.getApplicationContext().getBeanFactory();
beanFactory.registerSingleton("schemeWatcherManager", watcherManager);
beanFactory.registerSingleton("schemeManager", schemeManager);
var schemeManager = createSchemeManager(event);
schemeManager.register(Role.class);
@ -106,4 +102,17 @@ public class SchemeInitializer implements ApplicationListener<ApplicationContext
schemeManager.register(NotifierDescriptor.class);
schemeManager.register(Notification.class);
}
private static DefaultSchemeManager createSchemeManager(
ApplicationContextInitializedEvent event) {
var indexSpecRegistry = new IndexSpecRegistryImpl();
var watcherManager = new DefaultSchemeWatcherManager();
var schemeManager = new DefaultSchemeManager(indexSpecRegistry, watcherManager);
var beanFactory = event.getApplicationContext().getBeanFactory();
beanFactory.registerSingleton("indexSpecRegistry", indexSpecRegistry);
beanFactory.registerSingleton("schemeWatcherManager", watcherManager);
beanFactory.registerSingleton("schemeManager", schemeManager);
return schemeManager;
}
}

View File

@ -40,7 +40,7 @@ problemDetail.run.halo.app.infra.exception.DuplicateNameException=Duplicate name
problemDetail.run.halo.app.infra.exception.PluginAlreadyExistsException=Plugin {0} already exists.
problemDetail.run.halo.app.infra.exception.RateLimitExceededException=API rate limit exceeded, please try again later.
problemDetail.run.halo.app.infra.exception.EmailVerificationFailed=Invalid email verification code.
problemDetail.index.duplicateKey=The value of {0} already exists for unique index {1}, please rename it and retry.
problemDetail.user.email.verify.maxAttempts=Too many verification attempts, please try again later.
problemDetail.user.password.unsatisfied=The password does not meet the specifications.
problemDetail.user.username.unsatisfied=The username does not meet the specifications.

View File

@ -17,7 +17,7 @@ problemDetail.run.halo.app.infra.exception.DuplicateNameException=检测到有
problemDetail.run.halo.app.infra.exception.PluginAlreadyExistsException=插件 {0} 已经存。
problemDetail.run.halo.app.infra.exception.RateLimitExceededException=请求过于频繁,请稍候再试。
problemDetail.run.halo.app.infra.exception.EmailVerificationFailed=验证码错误或已失效。
problemDetail.index.duplicateKey=唯一索引 {1} 中的值 {0} 已存在,请更名后重试。
problemDetail.user.email.verify.maxAttempts=尝试次数过多,请稍候再试。
problemDetail.user.password.unsatisfied=密码不符合规范。
problemDetail.user.username.unsatisfied=用户名不符合规范。

View File

@ -22,17 +22,21 @@ import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.MediaType;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.web.reactive.server.WebTestClient;
import reactor.core.publisher.Flux;
import run.halo.app.core.extension.Role;
import run.halo.app.core.extension.service.RoleService;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.FakeExtension;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.Scheme;
import run.halo.app.extension.SchemeManager;
import run.halo.app.extension.index.IndexerFactory;
import run.halo.app.extension.store.ExtensionStoreRepository;
@DirtiesContext
@SpringBootTest
@AutoConfigureWebTestClient
class ExtensionConfigurationTest {
@ -66,8 +70,15 @@ class ExtensionConfigurationTest {
}
@AfterEach
void cleanUp(@Autowired ExtensionStoreRepository repository) {
repository.deleteAll().subscribe();
void cleanUp(@Autowired ExtensionStoreRepository repository,
@Autowired IndexerFactory indexerFactory) {
var gvk = Scheme.buildFromType(FakeExtension.class).groupVersionKind();
if (indexerFactory.contains(gvk)) {
indexerFactory.getIndexer(gvk).removeIndexRecords(descriptor -> true);
}
repository.deleteAll().block();
schemeManager.fetch(GroupVersionKind.fromExtension(FakeExtension.class))
.ifPresent(scheme -> schemeManager.unregister(scheme));
}
@Test
@ -116,13 +127,17 @@ class ExtensionConfigurationTest {
@BeforeEach
void setUp() {
var metadata = new Metadata();
metadata.setName("my-fake");
metadata.setLabels(Map.of("label-key", "label-value"));
var fake = new FakeExtension();
fake.setMetadata(metadata);
webClient.get()
.uri("/apis/fake.halo.run/v1alpha1/fakes/{}", metadata.getName())
.exchange()
.expectStatus().isNotFound();
createdFake = webClient.post()
.uri("/apis/fake.halo.run/v1alpha1/fakes")
.contentType(MediaType.APPLICATION_JSON)

View File

@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@ -21,10 +22,14 @@ import run.halo.app.extension.SchemeWatcherManager.SchemeRegistered;
import run.halo.app.extension.SchemeWatcherManager.SchemeUnregistered;
import run.halo.app.extension.SchemeWatcherManager.SchemeWatcher;
import run.halo.app.extension.exception.SchemeNotFoundException;
import run.halo.app.extension.index.IndexSpecRegistry;
@ExtendWith(MockitoExtension.class)
class DefaultSchemeManagerTest {
@Mock
private IndexSpecRegistry indexSpecRegistry;
@Mock
SchemeWatcherManager watcherManager;
@ -85,7 +90,7 @@ class DefaultSchemeManagerTest {
schemeManager.register(FakeExtension.class);
verify(watcherManager, times(1)).watchers();
verify(watcher, times(1)).onChange(isA(SchemeRegistered.class));
verify(indexSpecRegistry).indexFor(any(Scheme.class));
}
@Test
@ -105,6 +110,7 @@ class DefaultSchemeManagerTest {
schemeManager.unregister(scheme);
verify(watcherManager, times(2)).watchers();
verify(watcher, times(1)).onChange(isA(SchemeUnregistered.class));
verify(indexSpecRegistry).indexFor(any(Scheme.class));
}
@Test

View File

@ -2,6 +2,7 @@ package run.halo.app.extension;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -12,6 +13,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import run.halo.app.extension.exception.ExtensionConvertException;
import run.halo.app.extension.exception.SchemaViolationException;
import run.halo.app.extension.index.IndexSpecRegistry;
import run.halo.app.extension.store.ExtensionStore;
class JsonExtensionConverterTest {
@ -26,8 +28,9 @@ class JsonExtensionConverterTest {
void setUp() {
localeDefault = Locale.getDefault();
Locale.setDefault(Locale.ENGLISH);
var indexSpecRegistry = mock(IndexSpecRegistry.class);
DefaultSchemeManager schemeManager = new DefaultSchemeManager(null);
DefaultSchemeManager schemeManager = new DefaultSchemeManager(indexSpecRegistry, null);
converter = new JSONExtensionConverter(schemeManager);
objectMapper = converter.getObjectMapper();

View File

@ -13,6 +13,7 @@ import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.ArgumentMatchers.same;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -40,6 +41,8 @@ import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import run.halo.app.extension.exception.SchemeNotFoundException;
import run.halo.app.extension.index.Indexer;
import run.halo.app.extension.index.IndexerFactory;
import run.halo.app.extension.store.ExtensionStore;
import run.halo.app.extension.store.ReactiveExtensionStoreClient;
@ -57,6 +60,9 @@ class ReactiveExtensionClientTest {
@Mock
SchemeManager schemeManager;
@Mock
IndexerFactory indexerFactory;
@Spy
ObjectMapper objectMapper = JsonMapper.builder()
.addModule(new JavaTimeModule())
@ -126,7 +132,8 @@ class ReactiveExtensionClientTest {
.thenThrow(SchemeNotFoundException.class);
assertThrows(SchemeNotFoundException.class,
() -> client.list(UnRegisteredExtension.class, null, null));
() -> client.list(UnRegisteredExtension.class, null,
null));
assertThrows(SchemeNotFoundException.class,
() -> client.list(UnRegisteredExtension.class, null, null, 0, 10));
assertThrows(SchemeNotFoundException.class,
@ -172,7 +179,8 @@ class ReactiveExtensionClientTest {
List.of(createExtensionStore("fake-01"), createExtensionStore("fake-02"))));
// without filter and sorter
var fakes = client.list(FakeExtension.class, null, null);
Flux<FakeExtension> fakes =
client.list(FakeExtension.class, null, null);
StepVerifier.create(fakes)
.expectNext(fake1)
.expectNext(fake2)
@ -327,6 +335,9 @@ class ReactiveExtensionClientTest {
Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake")));
when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.create(fake))
.expectNext(fake)
.verifyComplete();
@ -334,6 +345,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(1)).convertTo(eq(fake));
verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any());
assertNotNull(fake.getMetadata().getCreationTimestamp());
verify(indexer).indexRecord(eq(fake));
}
@Test
@ -347,6 +359,9 @@ class ReactiveExtensionClientTest {
Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake")));
when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.create(fake))
.expectNext(fake)
.verifyComplete();
@ -357,6 +372,7 @@ class ReactiveExtensionClientTest {
}));
verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any());
assertNotNull(fake.getMetadata().getCreationTimestamp());
verify(indexer).indexRecord(eq(fake));
}
@Test
@ -393,6 +409,9 @@ class ReactiveExtensionClientTest {
Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake")));
when(converter.convertFrom(same(Unstructured.class), any())).thenReturn(fake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.create(fake))
.expectNext(fake)
.verifyComplete();
@ -400,6 +419,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(1)).convertTo(eq(fake));
verify(storeClient, times(1)).create(eq("/registry/fake.halo.run/fakes/fake"), any());
assertNotNull(fake.getMetadata().getCreationTimestamp());
verify(indexer).indexRecord(eq(fake));
}
@Test
@ -423,6 +443,9 @@ class ReactiveExtensionClientTest {
.thenReturn(oldFake)
.thenReturn(updatedFake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.update(fake))
.expectNext(updatedFake)
.verifyComplete();
@ -432,6 +455,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(2)).convertFrom(same(FakeExtension.class), any());
verify(storeClient)
.update(eq("/registry/fake.halo.run/fakes/fake"), eq(2L), any());
verify(indexer).updateRecord(eq(updatedFake));
}
@Test
@ -475,6 +499,9 @@ class ReactiveExtensionClientTest {
.thenReturn(oldFake)
.thenReturn(updatedFake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.update(fake))
.expectNext(updatedFake)
.verifyComplete();
@ -484,6 +511,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(2)).convertFrom(same(FakeExtension.class), any());
verify(storeClient)
.update(eq("/registry/fake.halo.run/fakes/fake"), eq(2L), any());
verify(indexer).updateRecord(eq(updatedFake));
}
@Test
@ -506,6 +534,9 @@ class ReactiveExtensionClientTest {
.thenReturn(oldFake)
.thenReturn(updatedFake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.update(fake))
.expectNext(updatedFake)
.verifyComplete();
@ -515,6 +546,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(2)).convertFrom(same(Unstructured.class), any());
verify(storeClient)
.update(eq("/registry/fake.halo.run/fakes/fake"), eq(12345L), any());
verify(indexer).updateRecord(eq(updatedFake));
}
@Test
@ -526,6 +558,9 @@ class ReactiveExtensionClientTest {
Mono.just(createExtensionStore("/registry/fake.halo.run/fakes/fake")));
when(converter.convertFrom(same(FakeExtension.class), any())).thenReturn(fake);
var indexer = mock(Indexer.class);
when(indexerFactory.getIndexer(eq(fake.groupVersionKind()))).thenReturn(indexer);
StepVerifier.create(client.delete(fake))
.expectNext(fake)
.verifyComplete();
@ -533,6 +568,7 @@ class ReactiveExtensionClientTest {
verify(converter, times(1)).convertTo(any());
verify(storeClient, times(1)).update(any(), any(), any());
verify(storeClient, never()).delete(any(), any());
verify(indexer).updateRecord(eq(fake));
}
@Nested
@ -549,10 +585,10 @@ class ReactiveExtensionClientTest {
@Test
void shouldWatchOnAddSuccessfully() {
doNothing().when(watcher).onAdd(any());
doNothing().when(watcher).onAdd(isA(Extension.class));
shouldCreateSuccessfully();
verify(watcher, times(1)).onAdd(any());
verify(watcher, times(1)).onAdd(isA(Extension.class));
}
@Test

Some files were not shown because too many files have changed in this diff Show More