github
/
halo
mirror of https://github.com/halo-dev/halo
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: index mechanism to enhance overall performance (#6039) 

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.17.x

#### What this PR does / why we need it:
重构索引机制的查询和排序以提升整体性能

**how to test it?**
使用 postgre 数据库，初始化 Halo ，然后执行以下脚本创建 30w 文章数据进行测试:
<details>
<summary>点击展开查看 SQL</summary>

```sql
DO $$
DECLARE
    i integer;
    postNameIndex integer;
    snapshotName varchar;
    totalRecords integer;
BEGIN
    postNameIndex := 1;
    totalRecords := 300000;

    FOR i IN 1..3 LOOP
      INSERT INTO "public"."extensions" ("name", "data", "version")
      VALUES (
          '/registry/content.halo.run/categories/category-'||i,
          convert_to(
              jsonb_build_object(
                  'spec', jsonb_build_object(
                      'displayName', '分类-'||i,
                      'slug', 'category-'||i,
                      'description', '测试分类',
                      'cover', '',
                      'template', '',
                      'priority', 0,
                      'children', '[]'::jsonb
                  ),
                  'status', jsonb_build_object(
                      'permalink', '/categories/category-'||i,
                      'postCount', totalRecords,
                      'visiblePostCount', totalRecords
                  ),
                  'apiVersion', 'content.halo.run/v1alpha1',
                  'kind', 'Category',
                  'metadata', jsonb_build_object(
                      'finalizers', jsonb_build_array('category-protection'),
                      'name', 'category-' || i,
                      'annotations', jsonb_build_object(
                          'content.halo.run/permalink-pattern', 'categories'
                      ),
                      'version', 0,
                      'creationTimestamp', '2024-06-12T03:56:40.315592Z'
                  )
          )::text, 'UTF8'),
          0
      );
    END LOOP;


    FOR i IN 1..3 LOOP
      INSERT INTO "public"."extensions" ("name", "data", "version")
        VALUES (
            '/registry/content.halo.run/tags/tag-' || i,
            convert_to(
               jsonb_build_object(
               'spec', jsonb_build_object(
                   'displayName', 'Halo tag ' || i,
                   'slug', 'tag-'||i,
                   'color', '#ffffff',
                   'cover', ''
               ),
               'status', jsonb_build_object(
                   'permalink', '/tags/tag-' || i,
                   'visiblePostCount', totalRecords,
                   'postCount', totalRecords,
                   'observedVersion', 0
               ),
               'apiVersion', 'content.halo.run/v1alpha1',
               'kind', 'Tag',
               'metadata', jsonb_build_object(
                   'finalizers', jsonb_build_array('tag-protection'),
                   'name', 'tag-'||i,
                   'annotations', jsonb_build_object(
                       'content.halo.run/permalink-pattern', 'tags'
                   ),
                   'version', 0,
                   'creationTimestamp', '2024-06-12T03:56:40.406407Z'
               )
       )::text, 'UTF8'),
       0);
    END LOOP;

    FOR i IN postNameIndex..totalRecords LOOP
        -- Generate snapshotName
        snapshotName := 'snapshot-' || i;

        -- Insert post data
        INSERT INTO "public"."extensions" ("name", "data", "version")
        VALUES (
            '/registry/content.halo.run/posts/post-' || postNameIndex,
            convert_to(
                jsonb_build_object(
                    'spec', jsonb_build_object(
                        'title', 'title-' || postNameIndex,
                        'slug', 'slug-' || postNameIndex,
                        'releaseSnapshot', snapshotName,
                        'headSnapshot', snapshotName,
                        'baseSnapshot', snapshotName,
                        'owner', 'admin',
                        'template', '',
                        'cover', '',
                        'deleted', false,
                        'publish', true,
                        'pinned', false,
                        'allowComment', true,
                        'visible', 'PUBLIC',
                        'priority', 0,
                        'excerpt', jsonb_build_object(
                            'autoGenerate', true,
                            'raw', ''
                        ),
                        'categories', ARRAY['category-kEvDb', 'category-XcRVk', 'category-adca'],
                        'tags', ARRAY['tag-RtKos', 'tag-vEsTR', 'tag-UBKCc'],
                        'htmlMetas', '[]'::jsonb
                    ),
                    'status', jsonb_build_object(
                        'phase', 'PUBLISHED',
                        'conditions', ARRAY[
                            jsonb_build_object(
                                'type', 'PUBLISHED',
                                'status', 'TRUE',
                                'lastTransitionTime', '2024-06-11T10:16:15.617748Z',
                                'message', 'Post published successfully.',
                                'reason', 'Published'
                            ),
                            jsonb_build_object(
                                'type', 'DRAFT',
                                'status', 'TRUE',
                                'lastTransitionTime', '2024-06-11T10:16:15.457668Z',
                                'message', 'Drafted post successfully.',
                                'reason', 'DraftedSuccessfully'
                            )
                        ],
                        'permalink', '/archives/slug-' || postNameIndex,
                        'excerpt', '如果你看到了这一篇文章，那么证明你已经安装成功了，感谢使用 Halo 进行创作，希望能够使用愉快。',
                        'inProgress', false,
                        'contributors', ARRAY['admin'],
                        'lastModifyTime', '2024-06-11T10:16:15.421467Z',
                        'observedVersion', 0
                    ),
                    'apiVersion', 'content.halo.run/v1alpha1',
                    'kind', 'Post',
                    'metadata', jsonb_build_object(
                        'finalizers', ARRAY['post-protection'],
                        'name', 'post-' || postNameIndex,
                        'labels', jsonb_build_object(
                            'content.halo.run/published', 'true',
                            'content.halo.run/deleted', 'false',
                            'content.halo.run/owner', 'admin',
                            'content.halo.run/visible', 'PUBLIC',
                            'content.halo.run/archive-year', '2024',
                            'content.halo.run/archive-month', '06',
                            'content.halo.run/archive-day', '11'
                        ),
                        'annotations', jsonb_build_object(
                            'content.halo.run/permalink-pattern', '/archives/{slug}',
                            'content.halo.run/last-released-snapshot', snapshotName,
                            'checksum/config', '73e40d4115f5a7d1e74fcc9228861c53d2ef60468e1e606e367b01efef339309'
                        ),
                        'version', 0,
                        'creationTimestamp', '2024-06-11T05:51:46.059292Z'
                    )
                )::text, 'UTF8'),
            1
        );

        -- Insert content data
        INSERT INTO "public"."extensions" ("name", "data", "version")
        VALUES (
            '/registry/content.halo.run/snapshots/' || snapshotName,
            convert_to(
                jsonb_build_object(
                    'spec', jsonb_build_object(
                        'subjectRef', jsonb_build_object(
                            'group', 'content.halo.run',
                            'version', 'v1alpha1',
                            'kind', 'Post',
                            'name', 'post-' || postNameIndex
                        ),
                        'rawType', 'HTML',
                        'rawPatch', '<p style=\"\">测试内容</p>',
                        'contentPatch', '<p style=\"\">测试内容</p>',
                        'lastModifyTime', '2024-06-11T06:01:25.748755Z',
                        'owner', 'admin',
                        'contributors', ARRAY['admin']
                    ),
                    'apiVersion', 'content.halo.run/v1alpha1',
                    'kind', 'Snapshot',
                    'metadata', jsonb_build_object(
                        'name', snapshotName,
                        'annotations', jsonb_build_object(
                            'content.halo.run/keep-raw', 'true'
                        ),
                        'creationTimestamp', '2024-06-11T06:01:25.748925Z'
                    )
                )::text, 'UTF8'),
            1
        );

        postNameIndex := postNameIndex + 1;
    END LOOP;
END $$;
```

</details>

使用以下 API 查询文章
```
curl 'http://localhost:8090/apis/api.console.halo.run/v1alpha1/posts?page=1&size=20&labelSelector=content.halo.run%2Fdeleted%3Dfalse&labelSelector=content.halo.run%2Fpublished%3Dtrue&fieldSelector=spec.categories%3Dcategory-1&fieldSelector=spec.tags%3Dc33ceabb-d8f1-4711-8991-bb8f5c92ad7c&fieldSelector=status.contributors%3Dadmin&fieldSelector=spec.visible%3DPUBLIC' \
--header 'Authorization: Basic YWRtaW46YWRtaW4='
```
Before:

![SCR-20240612-o20](https://github.com/halo-dev/halo/assets/38999863/fc27a265-6571-4361-a707-a683ea040837)
After:

![SCR-20240612-q1c](https://github.com/halo-dev/halo/assets/38999863/c0a241b8-5ed4-4973-8dfc-c260ffccd727)

#### Does this PR introduce a user-facing change?
```release-note
重构索引机制的查询和排序使整体性能提升 50% 以上
```
pull/6120/head
guqing 2024-06-21 16:04:11 +08:00 committed by GitHub
parent 8bdde317e5
commit c10862d6fe
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 1937 additions and 1182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
api/src/main/java/run/halo/app/extension/ListOptions.java
View File

@ -10,4 +10,19 @@ import run.halo.app.extension.router.selector.LabelSelector;
public class ListOptions {
private LabelSelector labelSelector;
private FieldSelector fieldSelector;
}
@Override
public String toString() {
var sb = new StringBuilder();
if (fieldSelector != null) {
sb.append("fieldSelector: ").append(fieldSelector.query());
}
if (labelSelector != null) {
if (!sb.isEmpty()) {
sb.append(", ");
}
sb.append("labelSelector: ").append(labelSelector);
}
return sb.toString();
}
}

3
api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java
View File

@ -2,6 +2,7 @@ package run.halo.app.extension.controller;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.Extension;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionMatcher;
@ -58,7 +59,7 @@ public class RequestSynchronizer implements Synchronizer<Request> {
listOptions.setFieldSelector(listMatcher.getFieldSelector());
listOptions.setLabelSelector(listMatcher.getLabelSelector());
}
indexedQueryEngine.retrieveAll(type, listOptions)
indexedQueryEngine.retrieveAll(type, listOptions, Sort.by("metadata.creationTimestamp"))
.forEach(name -> watcher.onAdd(new Request(name)));
}
client.watch(this.watcher);

2
application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java → api/src/main/java/run/halo/app/extension/index/IndexDescriptor.java
View File

@ -10,7 +10,7 @@ public class IndexDescriptor {
private final IndexSpec spec;
/**
* Record whether the index is ready, managed by {@link IndexBuilder}.
* Record whether the index is ready, managed by {@code IndexBuilder}.
*/
private boolean ready;

33
application/src/main/java/run/halo/app/extension/index/IndexEntry.java → api/src/main/java/run/halo/app/extension/index/IndexEntry.java
View File

@ -3,7 +3,7 @@ package run.halo.app.extension.index;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.NavigableSet;
import run.halo.app.extension.Metadata;
/**
@ -34,7 +34,7 @@ import run.halo.app.extension.Metadata;
public interface IndexEntry {
/**
* Acquires the read lock for reading such as {@link #getByIndexKey(String)},
* Acquires the read lock for reading such as {@link #getObjectNamesBy(String)},
* {@link #entries()}, {@link #indexedKeys()}, because the returned result set of these
* methods is not immutable.
*/
@ -87,7 +87,7 @@ public interface IndexEntry {
*
* @return distinct indexed keys of this entry.
*/
Set<String> indexedKeys();
NavigableSet<String> indexedKeys();
/**
* <p>Returns the entries of this entry in order.</p>
@ -99,19 +99,34 @@ public interface IndexEntry {
Collection<Map.Entry<String, String>> entries();
/**
* Returns the immutable entries of this entry in order, it is safe to modify the returned
* result, but extra cost is made.
*
* @return immutable entries of this entry.
* <p>Returns the position of the object name in the indexed attribute value mapping for
* sorting.</p>
* For example:
* <pre>
* metadata.name | field1
* ------------- | ------
* foo | 1
* bar | 2
* baz | 2
* </pre>
* "field1" is the indexed attribute, and the position of the object name in the indexed
* attribute
* value mapping for sorting is:
* <pre>
* foo -> 0
* bar -> 1
* baz -> 1
* </pre>
* "bar" and "baz" have the same value, so they have the same position.
*/
Collection<Map.Entry<String, String>> immutableEntries();
Map<String, Integer> getIdPositionMap();
/**
* Returns the object names of this entry in order.
*
* @return object names of this entry.
*/
List<String> getByIndexKey(String indexKey);
List<String> getObjectNamesBy(String indexKey);
void clear();
}

55
api/src/main/java/run/halo/app/extension/index/IndexEntryOperator.java Normal file
View File

@ -0,0 +1,55 @@
package run.halo.app.extension.index;
import java.util.Collection;
import java.util.NavigableSet;
import java.util.Set;
public interface IndexEntryOperator {
/**
* Search all values that key less than the target key.
*
* @param key target key
* @param orEqual whether to include the value of the target key
* @return object names that key less than the target key
*/
NavigableSet<String> lessThan(String key, boolean orEqual);
/**
* Search all values that key greater than the target key.
*
* @param key target key
* @param orEqual whether to include the value of the target key
* @return object names that key greater than the target key
*/
NavigableSet<String> greaterThan(String key, boolean orEqual);
/**
* Search all values that key in the range of [start, end].
*
* @param start start key
* @param end end key
* @param startInclusive whether to include the value of the start key
* @param endInclusive whether to include the value of the end key
* @return object names that key in the range of [start, end]
*/
NavigableSet<String> range(String start, String end, boolean startInclusive,
boolean endInclusive);
/**
* Find all values that key equals to the target key.
*
* @param key target key
* @return object names that key equals to the target key
*/
NavigableSet<String> find(String key);
NavigableSet<String> findIn(Collection<String> keys);
/**
* Get all values in the index entry.
*
* @return a set of all object names
*/
Set<String> getValues();
}

112
api/src/main/java/run/halo/app/extension/index/IndexEntryOperatorImpl.java Normal file
View File

@ -0,0 +1,112 @@
package run.halo.app.extension.index;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
import org.springframework.util.Assert;
public class IndexEntryOperatorImpl implements IndexEntryOperator {
private final IndexEntry indexEntry;
public IndexEntryOperatorImpl(IndexEntry indexEntry) {
this.indexEntry = indexEntry;
}
private static NavigableSet<String> createNavigableSet() {
return new TreeSet<>(KeyComparator.INSTANCE);
}
@Override
public NavigableSet<String> lessThan(String key, boolean orEqual) {
Assert.notNull(key, "Key must not be null.");
indexEntry.acquireReadLock();
try {
var navigableIndexedKeys = indexEntry.indexedKeys();
var headSetKeys = navigableIndexedKeys.headSet(key, orEqual);
return findIn(headSetKeys);
} finally {
indexEntry.releaseReadLock();
}
}
@Override
public NavigableSet<String> greaterThan(String key, boolean orEqual) {
Assert.notNull(key, "Key must not be null.");
indexEntry.acquireReadLock();
try {
var navigableIndexedKeys = indexEntry.indexedKeys();
var tailSetKeys = navigableIndexedKeys.tailSet(key, orEqual);
return findIn(tailSetKeys);
} finally {
indexEntry.releaseReadLock();
}
}
@Override
public NavigableSet<String> range(String start, String end, boolean startInclusive,
boolean endInclusive) {
Assert.notNull(start, "The start must not be null.");
Assert.notNull(end, "The end must not be null.");
indexEntry.acquireReadLock();
try {
var navigableIndexedKeys = indexEntry.indexedKeys();
var tailSetKeys = navigableIndexedKeys.subSet(start, startInclusive, end, endInclusive);
return findIn(tailSetKeys);
} finally {
indexEntry.releaseReadLock();
}
}
@Override
public NavigableSet<String> find(String key) {
Assert.notNull(key, "The key must not be null.");
indexEntry.acquireReadLock();
try {
var resultSet = createNavigableSet();
var result = indexEntry.getObjectNamesBy(key);
if (result != null) {
resultSet.addAll(result);
}
return resultSet;
} finally {
indexEntry.releaseReadLock();
}
}
@Override
public NavigableSet<String> findIn(Collection<String> keys) {
if (keys == null || keys.isEmpty()) {
return createNavigableSet();
}
indexEntry.acquireReadLock();
try {
var keysToSearch = new HashSet<>(keys);
var resultSet = createNavigableSet();
for (var entry : indexEntry.entries()) {
if (keysToSearch.contains(entry.getKey())) {
resultSet.add(entry.getValue());
}
}
return resultSet;
} finally {
indexEntry.releaseReadLock();
}
}
@Override
public Set<String> getValues() {
indexEntry.acquireReadLock();
try {
Set<String> uniqueValues = new HashSet<>();
for (Map.Entry<String, String> entry : indexEntry.entries()) {
uniqueValues.add(entry.getValue());
}
return uniqueValues;
} finally {
indexEntry.releaseReadLock();
}
}
}

4
api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java
View File

@ -1,6 +1,7 @@
package run.halo.app.extension.index;
import java.util.List;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
@ -36,7 +37,8 @@ public interface IndexedQueryEngine {
*
* @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager}
* @param options the list options to use for retrieving the object records
* @param sort the sort to use for retrieving the object records
* @return a collection of {@link Metadata#getName()}
*/
List<String> retrieveAll(GroupVersionKind type, ListOptions options);
List<String> retrieveAll(GroupVersionKind type, ListOptions options, Sort sort);
}

23
application/src/main/java/run/halo/app/extension/index/Indexer.java → api/src/main/java/run/halo/app/extension/index/Indexer.java
View File

@ -2,6 +2,7 @@ package run.halo.app.extension.index;
import java.util.Iterator;
import java.util.function.Function;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
/**
@ -20,7 +21,7 @@ public interface Indexer {
/**
* <p>Index the specified {@link Extension} by {@link IndexDescriptor}s.</p>
* <p>First, the {@link Indexer} will index the {@link Extension} by the
* {@link IndexDescriptor}s and record the index entries to {@link IndexerTransaction} and
* {@link IndexDescriptor}s and record the index entries to {@code IndexerTransaction} and
* commit the transaction, if any error occurs, the transaction will be rollback to keep the
* {@link Indexer} consistent.</p>
*
@ -33,7 +34,7 @@ public interface Indexer {
* <p>Update indexes for the specified {@link Extension} by {@link IndexDescriptor}s.</p>
* <p>First, the {@link Indexer} will remove the index entries of the {@link Extension} by
* the old {@link IndexDescriptor}s and reindex the {@link Extension} to generate change logs
* to {@link IndexerTransaction} and commit the transaction, if any error occurs, the
* to {@code IndexerTransaction} and commit the transaction, if any error occurs, the
* transaction will be rollback to keep the {@link Indexer} consistent.</p>
*
* @param extension the {@link Extension} to be updated
@ -73,11 +74,21 @@ public interface Indexer {
*/
void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn);
/**
* <p>Get the {@link IndexEntry} by index name if found and ready.</p>
*
* @param name an index name
* @return the {@link IndexEntry} if found
* @throws IllegalArgumentException if the index name is not found or the index is not ready
*/
@NonNull
IndexEntry getIndexEntry(String name);
/**
* <p>Gets an iterator over all the ready {@link IndexEntry}s, in no particular order.</p>
*
* @return an iterator over all the ready {@link IndexEntry}s
* @link {@link IndexDescriptor#isReady()}
* @see IndexDescriptor#isReady()
*/
Iterator<IndexEntry> readyIndexesIterator();
@ -85,7 +96,11 @@ public interface Indexer {
* <p>Gets an iterator over all the {@link IndexEntry}s, in no particular order.</p>
*
* @return an iterator over all the {@link IndexEntry}s
* @link {@link IndexDescriptor#isReady()}
* @see IndexDescriptor#isReady()
*/
Iterator<IndexEntry> allIndexesIterator();
void acquireReadLock();
void releaseReadLock();
}

7
api/src/main/java/run/halo/app/extension/index/query/All.java
View File

@ -10,6 +10,11 @@ public class All extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
return indexView.getAllIdsForField(fieldName);
return indexView.getIdsForField(fieldName);
}
@Override
public String toString() {
return fieldName + " != null";
}
}

7
api/src/main/java/run/halo/app/extension/index/query/And.java
View File

@ -3,6 +3,7 @@ package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.NavigableSet;
import java.util.stream.Collectors;
public class And extends LogicalQuery {
@ -33,4 +34,10 @@ public class And extends LogicalQuery {
}
return resultSet == null ? Sets.newTreeSet() : resultSet;
}
@Override
public String toString() {
return "(" + childQueries.stream().map(Query::toString)
.collect(Collectors.joining(" AND ")) + ")";
}
}

16
api/src/main/java/run/halo/app/extension/index/query/Between.java
View File

@ -1,6 +1,5 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class Between extends SimpleQuery {
@ -19,17 +18,14 @@ public class Between extends SimpleQuery {
this.upperInclusive = upperInclusive;
}
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
NavigableSet<String> allValues = indexView.getAllValuesForField(fieldName);
// get all values in the specified range
var subSet = allValues.subSet(lowerValue, lowerInclusive, upperValue, upperInclusive);
return indexView.between(fieldName, lowerValue, lowerInclusive, upperValue, upperInclusive);
}
var resultSet = Sets.<String>newTreeSet();
for (String val : subSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
@Override
public String toString() {
return fieldName + " BETWEEN " + (lowerInclusive ? "[" : "(") + lowerValue + ", "
+ upperValue + (upperInclusive ? "]" : ")");
}
}

16
api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java
View File

@ -17,16 +17,16 @@ public class EqualQuery extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
return indexView.findMatchingIdsWithEqualValues(fieldName, value);
}
return resultSetForExactValue(indexView);
return indexView.findIds(fieldName, value);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueEqual(fieldName, value);
}
private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) {
return indexView.getIdsForFieldValue(fieldName, value);
@Override
public String toString() {
if (isFieldRef) {
return fieldName + " = " + value;
}
return fieldName + " = '" + value + "'";
}
}

24
api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java
View File

@ -1,6 +1,5 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class GreaterThanQuery extends SimpleQuery {
@ -18,24 +17,15 @@ public class GreaterThanQuery extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
return indexView.findMatchingIdsWithGreaterValues(fieldName, value, orEqual);
}
return resultSetForExtractValue(indexView);
return indexView.findIdsGreaterThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueGreaterThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForExtractValue(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
NavigableSet<String> tailSet =
orEqual ? allValues.tailSet(value, true) : allValues.tailSet(value, false);
for (String val : tailSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
@Override
public String toString() {
return fieldName
+ (orEqual ? " >= " : " > ")
+ (isFieldRef ? value : "'" + value + "'");
}
}

18
api/src/main/java/run/halo/app/extension/index/query/InQuery.java
View File

@ -1,8 +1,9 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
import java.util.Set;
import java.util.stream.Collectors;
import run.halo.app.extension.index.IndexEntryOperatorImpl;
public class InQuery extends SimpleQuery {
private final Set<String> values;
@ -14,10 +15,15 @@ public class InQuery extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
NavigableSet<String> resultSet = Sets.newTreeSet();
for (String val : values) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
var indexEntry = indexView.getIndexEntry(fieldName);
var operator = new IndexEntryOperatorImpl(indexEntry);
return operator.findIn(values);
}
@Override
public String toString() {
return fieldName + " IN (" + values.stream()
.map(value -> "'" + value + "'")
.collect(Collectors.joining(", ")) + ")";
}
}

7
api/src/main/java/run/halo/app/extension/index/query/IsNotNull.java
View File

@ -10,6 +10,11 @@ public class IsNotNull extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
return indexView.getAllIdsForField(fieldName);
return indexView.getIdsForField(fieldName);
}
@Override
public String toString() {
return fieldName + " IS NOT NULL";
}
}

18
api/src/main/java/run/halo/app/extension/index/query/IsNull.java
View File

@ -10,9 +10,19 @@ public class IsNull extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var allIds = indexView.getAllIds();
var idsForField = indexView.getAllIdsForField(fieldName);
allIds.removeAll(idsForField);
return allIds;
indexView.acquireReadLock();
try {
var allIds = indexView.getAllIds();
var idsForNonNullValue = indexView.getIdsForField(fieldName);
allIds.removeAll(idsForNonNullValue);
return allIds;
} finally {
indexView.releaseReadLock();
}
}
@Override
public String toString() {
return fieldName + " IS NULL";
}
}

24
api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java
View File

@ -1,6 +1,5 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
public class LessThanQuery extends SimpleQuery {
@ -18,24 +17,15 @@ public class LessThanQuery extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
if (isFieldRef) {
return resultSetForRefValue(indexView);
return indexView.findMatchingIdsWithSmallerValues(fieldName, value, orEqual);
}
return resultSetForExactValue(indexView);
return indexView.findIdsLessThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) {
return indexView.findIdsForFieldValueLessThan(fieldName, value, orEqual);
}
private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
var headSet = orEqual ? allValues.headSet(value, true)
: allValues.headSet(value, false);
for (String val : headSet) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
}
return resultSet;
@Override
public String toString() {
return fieldName
+ (orEqual ? " <= " : " < ")
+ (isFieldRef ? value : "'" + value + "'");
}
}

5
api/src/main/java/run/halo/app/extension/index/query/Not.java
View File

@ -24,4 +24,9 @@ public class Not extends LogicalQuery {
allIds.removeAll(negatedResult);
return allIds;
}
@Override
public String toString() {
return "NOT (" + negatedQuery + ")";
}
}

23
api/src/main/java/run/halo/app/extension/index/query/NotEqual.java
View File

@ -1,6 +1,5 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.NavigableSet;
import org.springframework.util.Assert;
@ -19,15 +18,19 @@ public class NotEqual extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var names = equalQuery.matches(indexView);
var allNames = indexView.getAllIdsForField(fieldName);
var resultSet = Sets.<String>newTreeSet();
for (String name : allNames) {
if (!names.contains(name)) {
resultSet.add(name);
}
indexView.acquireReadLock();
try {
NavigableSet<String> equalNames = equalQuery.matches(indexView);
NavigableSet<String> allNames = indexView.getIdsForField(fieldName);
allNames.removeAll(equalNames);
return allNames;
} finally {
indexView.releaseReadLock();
}
return resultSet;
}
@Override
public String toString() {
return fieldName + " != " + (isFieldRef ? value : "'" + value + "'");
}
}

7
api/src/main/java/run/halo/app/extension/index/query/Or.java
View File

@ -3,6 +3,7 @@ package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.NavigableSet;
import java.util.stream.Collectors;
public class Or extends LogicalQuery {
@ -18,4 +19,10 @@ public class Or extends LogicalQuery {
}
return resultSet;
}
@Override
public String toString() {
return "(" + childQueries.stream().map(Query::toString)
.collect(Collectors.joining(" OR ")) + ")";
}
}

107
api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import java.util.NavigableSet;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.index.IndexEntry;
import run.halo.app.extension.index.IndexSpec;
/**
@ -19,6 +20,7 @@ import run.halo.app.extension.index.IndexSpec;
* @since 2.12.0
*/
public interface QueryIndexView {
/**
* Gets all object ids for a given field name and field value.
*
@ -27,16 +29,7 @@ public interface QueryIndexView {
* @return all indexed object ids associated with the given field name and field value
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue);
/**
* Gets all field values for a given field name.
*
* @param fieldName the field name
* @return all field values for the given field name
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getAllValuesForField(String fieldName);
NavigableSet<String> findIds(String fieldName, String fieldValue);
/**
* Gets all object ids for a given field name without null cells.
@ -45,7 +38,7 @@ public interface QueryIndexView {
* @return all indexed object ids for the given field name
* @throws IllegalArgumentException if the field name is not indexed
*/
NavigableSet<String> getAllIdsForField(String fieldName);
NavigableSet<String> getIdsForField(String fieldName);
/**
* Gets all object ids in this view.
@ -54,15 +47,97 @@ public interface QueryIndexView {
*/
NavigableSet<String> getAllIds();
NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2);
/**
* <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have
* matching values across two fields and where the values are equal.</p>
* For example:
* <pre>
* metadata.name | field1 | field2
* ------------- | ------ | ------
* foo | 1 | 1
* bar | 2 | 3
* baz | 3 | 3
* </pre>
* <code>findMatchingIdsWithEqualValues("field1", "field2")</code> would return ["foo","baz"]
*
* @see #findMatchingIdsWithGreaterValues(String, String, boolean)
* @see #findMatchingIdsWithSmallerValues(String, String, boolean)
*/
NavigableSet<String> findMatchingIdsWithEqualValues(String fieldName1, String fieldName2);
NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1, String fieldName2,
/**
* <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have
* matching values across two fields, but where the value associated with fieldName1 is
* greater than the value associated with fieldName2.</p>
* For example:
* <pre>
* metadata.name | field1 | field2
* ------------- | ------ | ------
* foo | 1 | 1
* bar | 2 | 3
* baz | 3 | 3
* qux | 4 | 2
* </pre>
* <p><code>findMatchingIdsWithGreaterValues("field1", "field2")</code>would return ["qux"]</p>
* <p><code>findMatchingIdsWithGreaterValues("field2", "field1")</code>would return ["bar"]</p>
* <p><code>findMatchingIdsWithGreaterValues("field1", "field2", true)</code>would return
* ["foo","baz","qux"]</p>
*
* @param fieldName1 The field name whose values are compared as the larger values.
* @param fieldName2 The field name whose values are compared as the smaller values.
* @param orEqual whether to include equal values
* @return A result set of ids where the entries in fieldName1 have greater values than those
* in fieldName2 for entries that have the same id across both fields
*/
NavigableSet<String> findMatchingIdsWithGreaterValues(String fieldName1, String fieldName2,
boolean orEqual);
NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2,
NavigableSet<String> findIdsGreaterThan(String fieldName, String fieldValue, boolean orEqual);
/**
* <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have
* matching values across two fields, but where the value associated with fieldName1 is
* less than the value associated with fieldName2.</p>
* For example:
* <pre>
* metadata.name | field1 | field2
* ------------- | ------ | ------
* foo | 1 | 1
* bar | 2 | 3
* baz | 3 | 3
* qux | 4 | 2
* </pre>
* <p><code>findMatchingIdsWithSmallerValues("field1", "field2")</code> would return ["bar"]</p>
* <p><code>findMatchingIdsWithSmallerValues("field2", "field1")</code> would return ["qux"]</p>
* <p><code>findMatchingIdsWithSmallerValues("field1", "field2", true)</code> would return
* ["foo","bar","baz"]</p>
*
* @param fieldName1 The field name whose values are compared as the smaller values.
* @param fieldName2 The field name whose values are compared as the larger values.
* @param orEqual whether to include equal values
* @return A result set of ids where the entries in fieldName1 have smaller values than those
* in fieldName2 for entries that have the same id across both fields
*/
NavigableSet<String> findMatchingIdsWithSmallerValues(String fieldName1, String fieldName2,
boolean orEqual);
void removeByIdNotIn(NavigableSet<String> ids);
NavigableSet<String> findIdsLessThan(String fieldName, String fieldValue, boolean orEqual);
List<String> sortBy(Sort sort);
NavigableSet<String> between(String fieldName, String lowerValue, boolean lowerInclusive,
String upperValue, boolean upperInclusive);
List<String> sortBy(NavigableSet<String> resultSet, Sort sort);
IndexEntry getIndexEntry(String fieldName);
/**
* Acquire a read lock on the indexer.
* if you need to operate on more than one {@code IndexEntry} at the same time, you need to
* lock first.
*
* @see #getIndexEntry(String)
*/
void acquireReadLock();
void releaseReadLock();
}

486
api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java
View File

@ -1,352 +1,230 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.BiPredicate;
import org.springframework.data.domain.Sort;
import org.springframework.lang.Nullable;
import org.springframework.util.CollectionUtils;
import run.halo.app.extension.index.IndexEntry;
import run.halo.app.extension.index.IndexEntryOperator;
import run.halo.app.extension.index.IndexEntryOperatorImpl;
import run.halo.app.extension.index.Indexer;
import run.halo.app.extension.index.KeyComparator;
/**
* A default implementation for {@link run.halo.app.extension.index.query.QueryIndexView}.
*
* @author guqing
* @since 2.12.0
* @since 2.17.0
*/
public class QueryIndexViewImpl implements QueryIndexView {
private final Lock lock = new ReentrantLock();
private final Set<String> fieldNames;
private final Table<String, String, NavigableSet<String>> orderedMatches;
public static final String PRIMARY_INDEX_NAME = "metadata.name";
private final Indexer indexer;
/**
* Creates a new {@link QueryIndexViewImpl} for the given {@link Map} of index entries.
* Construct a new {@link QueryIndexViewImpl} with the given {@link Indexer}.
*
* @param indexEntries index entries from indexer to create the view for.
* @throws IllegalArgumentException if the primary index does not exist
*/
public QueryIndexViewImpl(Map<String, Collection<Map.Entry<String, String>>> indexEntries) {
this.fieldNames = new HashSet<>();
this.orderedMatches = HashBasedTable.create();
for (var entry : indexEntries.entrySet()) {
String fieldName = entry.getKey();
this.fieldNames.add(fieldName);
for (var fieldEntry : entry.getValue()) {
var id = fieldEntry.getValue();
var fieldValue = fieldEntry.getKey();
var columnValue = this.orderedMatches.get(id, fieldName);
if (columnValue == null) {
columnValue = Sets.newTreeSet();
this.orderedMatches.put(id, fieldName, columnValue);
}
columnValue.add(fieldValue);
}
}
public QueryIndexViewImpl(Indexer indexer) {
// check if primary index exists
indexer.getIndexEntry(PRIMARY_INDEX_NAME);
this.indexer = indexer;
}
@Override
public NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
var result = new TreeSet<String>();
for (var cell : orderedMatches.cellSet()) {
if (cell.getColumnKey().equals(fieldName) && cell.getValue().contains(fieldValue)) {
result.add(cell.getRowKey());
}
}
return result;
} finally {
lock.unlock();
}
public NavigableSet<String> findIds(String fieldName, String fieldValue) {
var operator = getEntryOperator(fieldName);
return operator.find(fieldValue);
}
@Override
public NavigableSet<String> getAllValuesForField(String fieldName) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
var result = Sets.<String>newTreeSet();
for (var cell : orderedMatches.cellSet()) {
if (cell.getColumnKey().equals(fieldName)) {
result.addAll(cell.getValue());
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> getAllIdsForField(String fieldName) {
lock.lock();
try {
checkFieldNameIndexed(fieldName);
NavigableSet<String> ids = new TreeSet<>();
// iterate over the table and collect all IDs associated with the given field name
for (var cell : orderedMatches.cellSet()) {
if (cell.getColumnKey().equals(fieldName)) {
ids.add(cell.getRowKey());
}
}
return ids;
} finally {
lock.unlock();
}
public NavigableSet<String> getIdsForField(String fieldName) {
var operator = getEntryOperator(fieldName);
return new TreeSet<>(operator.getValues());
}
@Override
public NavigableSet<String> getAllIds() {
lock.lock();
try {
return new TreeSet<>(orderedMatches.rowKeySet());
} finally {
lock.unlock();
}
return new TreeSet<>(allIds());
}
@Override
public NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2) {
lock.lock();
public NavigableSet<String> findMatchingIdsWithEqualValues(String fieldName1,
String fieldName2) {
indexer.acquireReadLock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
NavigableSet<String> result = new TreeSet<>();
// obtain all values for both fields and their corresponding IDs
var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1);
var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2);
// iterate over each value of the first field
for (Map.Entry<String, NavigableSet<String>> entry : field1ValuesToIds.entrySet()) {
String fieldValue = entry.getKey();
NavigableSet<String> idsForFieldValue = entry.getValue();
// if the second field contains the same value, add all matching IDs
if (field2ValuesToIds.containsKey(fieldValue)) {
NavigableSet<String> matchingIds = field2ValuesToIds.get(fieldValue);
for (String id : idsForFieldValue) {
if (matchingIds.contains(id)) {
result.add(id);
}
}
}
}
return result;
} finally {
lock.unlock();
}
}
private Map<String, NavigableSet<String>> getColumnValuesToIdsMap(String fieldName) {
var valuesToIdsMap = new HashMap<String, NavigableSet<String>>();
for (var cell : orderedMatches.cellSet()) {
if (cell.getColumnKey().equals(fieldName)) {
var celValues = cell.getValue();
if (CollectionUtils.isEmpty(celValues)) {
continue;
}
if (celValues.size() != 1) {
throw new IllegalArgumentException(
"Unsupported multi cell values to join with other field for: " + fieldName
+ " with values: " + celValues);
}
String fieldValue = cell.getValue().first();
if (!valuesToIdsMap.containsKey(fieldValue)) {
valuesToIdsMap.put(fieldValue, new TreeSet<>());
}
valuesToIdsMap.get(fieldValue).add(cell.getRowKey());
}
}
return valuesToIdsMap;
}
@Override
public NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1,
String fieldName2, boolean orEqual) {
lock.lock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
NavigableSet<String> result = new TreeSet<>();
// obtain all values for both fields and their corresponding IDs
var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1);
var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2);
// iterate over each value of the first field
for (var entryField1 : field1ValuesToIds.entrySet()) {
String fieldValue1 = entryField1.getKey();
// iterate over each value of the second field
for (var entryField2 : field2ValuesToIds.entrySet()) {
String fieldValue2 = entryField2.getKey();
int comparison = fieldValue1.compareTo(fieldValue2);
if (orEqual ? comparison >= 0 : comparison > 0) {
// if the second field contains the same value, add all matching IDs
for (String id : entryField1.getValue()) {
if (field2ValuesToIds.get(fieldValue2).contains(id)) {
result.add(id);
}
}
}
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2,
boolean orEqual) {
lock.lock();
try {
checkFieldNameIndexed(fieldName1);
checkFieldNameIndexed(fieldName2);
NavigableSet<String> result = new TreeSet<>();
// obtain all values for both fields and their corresponding IDs
var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1);
var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2);
// iterate over each value of the first field
for (var entryField1 : field1ValuesToIds.entrySet()) {
String fieldValue1 = entryField1.getKey();
// iterate over each value of the second field
for (var entryField2 : field2ValuesToIds.entrySet()) {
String fieldValue2 = entryField2.getKey();
int comparison = fieldValue1.compareTo(fieldValue2);
if (orEqual ? comparison <= 0 : comparison < 0) {
// if the second field contains the same value, add all matching IDs
for (String id : entryField1.getValue()) {
if (field2ValuesToIds.get(fieldValue2).contains(id)) {
result.add(id);
}
}
}
}
}
return result;
} finally {
lock.unlock();
}
}
@Override
public void removeByIdNotIn(NavigableSet<String> ids) {
lock.lock();
try {
Set<String> idsToRemove = new HashSet<>();
// check each row key if it is not in the given ids set
for (String rowKey : orderedMatches.rowKeySet()) {
if (!ids.contains(rowKey)) {
idsToRemove.add(rowKey);
}
}
// remove all rows that are not in the given ids set
for (String idToRemove : idsToRemove) {
orderedMatches.row(idToRemove).clear();
}
} finally {
lock.unlock();
}
}
@Override
public List<String> sortBy(Sort sort) {
lock.lock();
try {
for (Sort.Order order : sort) {
String fieldName = order.getProperty();
checkFieldNameIndexed(fieldName);
}
// obtain all row keys (IDs)
Set<String> allRowKeys = orderedMatches.rowKeySet();
// convert row keys to list for sorting
List<String> sortedRowKeys = new ArrayList<>(allRowKeys);
if (sort.isUnsorted()) {
return sortedRowKeys;
}
// sort row keys according to sort criteria in a Sort object
sortedRowKeys.sort((id1, id2) -> {
for (Sort.Order order : sort) {
String fieldName = order.getProperty();
// compare the values of the two rows on the field
int comparison = compareRowValue(id1, id2, fieldName, order.isAscending());
if (comparison != 0) {
return comparison;
}
}
// if all sort criteria are equal, return 0
return 0;
return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> {
var compare = KeyComparator.INSTANCE.compare(k1, k2);
return compare == 0;
});
return sortedRowKeys;
} finally {
lock.unlock();
indexer.releaseReadLock();
}
}
private int compareRowValue(String id1, String id2, String fieldName, boolean isAscending) {
var value1 = getSingleFieldValueForSort(id1, fieldName);
var value2 = getSingleFieldValueForSort(id2, fieldName);
// nulls are less than everything whatever the sort order is
// do not simply the following code for null check,it's different from KeyComparator
if (value1 == null && value2 == null) {
return 0;
} else if (value1 == null) {
return 1;
} else if (value2 == null) {
return -1;
@Override
public NavigableSet<String> findMatchingIdsWithGreaterValues(String fieldName1,
String fieldName2, boolean orEqual) {
indexer.acquireReadLock();
try {
return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> {
var compare = KeyComparator.INSTANCE.compare(k1, k2);
return orEqual ? compare <= 0 : compare < 0;
});
} finally {
indexer.releaseReadLock();
}
return isAscending ? KeyComparator.INSTANCE.compare(value1, value2)
: KeyComparator.INSTANCE.compare(value2, value1);
}
@Nullable
String getSingleFieldValueForSort(String rowKey, String fieldName) {
return Optional.ofNullable(orderedMatches.get(rowKey, fieldName))
.filter(values -> !CollectionUtils.isEmpty(values))
.map(values -> {
if (values.size() != 1) {
throw new IllegalArgumentException(
"Unsupported multi field values to sort for: " + fieldName
+ " with values: " + values);
@Override
public NavigableSet<String> findIdsGreaterThan(String fieldName, String fieldValue,
boolean orEqual) {
var operator = getEntryOperator(fieldName);
return operator.greaterThan(fieldValue, orEqual);
}
@Override
public NavigableSet<String> findMatchingIdsWithSmallerValues(String fieldName1,
String fieldName2, boolean orEqual) {
indexer.acquireReadLock();
try {
return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> {
var compare = KeyComparator.INSTANCE.compare(k1, k2);
return orEqual ? compare >= 0 : compare > 0;
});
} finally {
indexer.releaseReadLock();
}
}
@Override
public NavigableSet<String> findIdsLessThan(String fieldName, String fieldValue,
boolean orEqual) {
var operator = getEntryOperator(fieldName);
return operator.lessThan(fieldValue, orEqual);
}
@Override
public NavigableSet<String> between(String fieldName, String lowerValue, boolean lowerInclusive,
String upperValue, boolean upperInclusive) {
var operator = getEntryOperator(fieldName);
return operator.range(lowerValue, upperValue, lowerInclusive, upperInclusive);
}
@Override
public List<String> sortBy(NavigableSet<String> ids, Sort sort) {
if (sort.isUnsorted()) {
return new ArrayList<>(ids);
}
indexer.acquireReadLock();
try {
var combinedComparator = sort.stream()
.map(this::comparatorFrom)
.reduce(Comparator::thenComparing)
.orElseThrow();
return ids.stream()
.sorted(combinedComparator)
.toList();
} finally {
indexer.releaseReadLock();
}
}
Comparator<String> comparatorFrom(Sort.Order order) {
var indexEntry = getIndexEntry(order.getProperty());
var idPositionMap = indexEntry.getIdPositionMap();
var isDesc = order.isDescending();
// This sort algorithm works leveraging on that the idPositionMap is a map of id -> position
// if the id is not in the map, it means that it is not indexed, and it will be placed at
// the end
return (a, b) -> {
var indexOfA = idPositionMap.get(a);
var indexOfB = idPositionMap.get(b);
var isAIndexed = indexOfA != null;
var isBIndexed = indexOfB != null;
if (!isAIndexed && !isBIndexed) {
return 0;
}
// un-indexed item are always at the end
if (!isAIndexed) {
return isDesc ? -1 : 1;
}
if (!isBIndexed) {
return isDesc ? 1 : -1;
}
return isDesc ? Integer.compare(indexOfB, indexOfA)
: Integer.compare(indexOfA, indexOfB);
};
}
@Override
public IndexEntry getIndexEntry(String fieldName) {
return indexer.getIndexEntry(fieldName);
}
@Override
public void acquireReadLock() {
indexer.acquireReadLock();
}
@Override
public void releaseReadLock() {
indexer.releaseReadLock();
}
private IndexEntryOperator getEntryOperator(String fieldName) {
var indexEntry = getIndexEntry(fieldName);
return createIndexEntryOperator(indexEntry);
}
private IndexEntryOperator createIndexEntryOperator(IndexEntry entry) {
return new IndexEntryOperatorImpl(entry);
}
private Set<String> allIds() {
var indexEntry = getIndexEntry(PRIMARY_INDEX_NAME);
return createIndexEntryOperator(indexEntry).getValues();
}
/**
* Must lock the indexer before calling this method.
*/
private NavigableSet<String> findIdsWithKeyComparator(String fieldName1, String fieldName2,
BiPredicate<String, String> keyComparator) {
// get entries from indexer for fieldName1
var entriesA = getIndexEntry(fieldName1).entries();
Map<String, List<String>> keyMap = new HashMap<>();
for (Map.Entry<String, String> entry : entriesA) {
keyMap.computeIfAbsent(entry.getValue(), v -> new ArrayList<>()).add(entry.getKey());
}
NavigableSet<String> result = new TreeSet<>();
// get entries from indexer for fieldName2
var entriesB = getIndexEntry(fieldName2).entries();
for (Map.Entry<String, String> entry : entriesB) {
List<String> matchedKeys = keyMap.get(entry.getValue());
if (matchedKeys != null) {
for (String key : matchedKeys) {
if (keyComparator.test(entry.getKey(), key)) {
result.add(entry.getValue());
// found one match, no need to continue
break;
}
}
return values.first();
})
.orElse(null);
}
private void checkFieldNameIndexed(String fieldName) {
if (!fieldNames.contains(fieldName)) {
throw new IllegalArgumentException("Field name " + fieldName
+ " is not indexed, please ensure it added to the index spec before querying");
}
}
return result;
}
}

23
api/src/main/java/run/halo/app/extension/index/query/StringContains.java
View File

@ -1,6 +1,7 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Map;
import java.util.NavigableSet;
import org.apache.commons.lang3.StringUtils;
@ -12,12 +13,24 @@ public class StringContains extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var fieldValues = indexView.getAllValuesForField(fieldName);
for (String val : fieldValues) {
if (StringUtils.containsIgnoreCase(val, value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
var indexEntry = indexView.getIndexEntry(fieldName);
indexEntry.acquireReadLock();
try {
for (Map.Entry<String, String> entry : indexEntry.entries()) {
var fieldValue = entry.getKey();
if (StringUtils.containsIgnoreCase(fieldValue, value)) {
resultSet.add(entry.getValue());
}
}
return resultSet;
} finally {
indexEntry.releaseReadLock();
}
return resultSet;
}
@Override
public String toString() {
return "contains(" + fieldName + ", '" + value + "')";
}
}

24
api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java
View File

@ -1,7 +1,9 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Map;
import java.util.NavigableSet;
import org.apache.commons.lang3.StringUtils;
public class StringEndsWith extends SimpleQuery {
public StringEndsWith(String fieldName, String value) {
@ -11,12 +13,24 @@ public class StringEndsWith extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var fieldValues = indexView.getAllValuesForField(fieldName);
for (String val : fieldValues) {
if (val.endsWith(value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
var indexEntry = indexView.getIndexEntry(fieldName);
indexEntry.acquireReadLock();
try {
for (Map.Entry<String, String> entry : indexEntry.entries()) {
var fieldValue = entry.getKey();
if (StringUtils.endsWith(fieldValue, value)) {
resultSet.add(entry.getValue());
}
}
return resultSet;
} finally {
indexEntry.releaseReadLock();
}
return resultSet;
}
@Override
public String toString() {
return "endsWith(" + fieldName + ", '" + value + "')";
}
}

23
api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java
View File

@ -1,7 +1,9 @@
package run.halo.app.extension.index.query;
import com.google.common.collect.Sets;
import java.util.Map;
import java.util.NavigableSet;
import org.apache.commons.lang3.StringUtils;
public class StringStartsWith extends SimpleQuery {
public StringStartsWith(String fieldName, String value) {
@ -11,13 +13,24 @@ public class StringStartsWith extends SimpleQuery {
@Override
public NavigableSet<String> matches(QueryIndexView indexView) {
var resultSet = Sets.<String>newTreeSet();
var allValues = indexView.getAllValuesForField(fieldName);
var indexEntry = indexView.getIndexEntry(fieldName);
for (String val : allValues) {
if (val.startsWith(value)) {
resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val));
indexEntry.acquireReadLock();
try {
for (Map.Entry<String, String> entry : indexEntry.entries()) {
var fieldValue = entry.getKey();
if (StringUtils.startsWith(fieldValue, value)) {
resultSet.add(entry.getValue());
}
}
return resultSet;
} finally {
indexEntry.releaseReadLock();
}
return resultSet;
}
@Override
public String toString() {
return "startsWith(" + fieldName + ", '" + value + "')";
}
}

5
api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java
View File

@ -16,6 +16,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionMatcher;
import run.halo.app.extension.FakeExtension;
@ -53,7 +54,7 @@ class RequestSynchronizerTest {
@Test
void shouldStartCorrectlyWhenSyncingAllOnStart() {
var type = GroupVersionKind.fromExtension(FakeExtension.class);
when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class)))
when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class), any(Sort.class)))
.thenReturn(List.of("fake-01", "fake-02"));
synchronizer.start();
@ -62,7 +63,7 @@ class RequestSynchronizerTest {
assertFalse(synchronizer.isDisposed());
verify(indexedQueryEngine, times(1)).retrieveAll(eq(type),
isA(ListOptions.class));
isA(ListOptions.class), isA(Sort.class));
verify(watcher, times(2)).onAdd(isA(Reconciler.Request.class));
verify(client, times(1)).watch(same(watcher));
}

24
api/src/test/java/run/halo/app/extension/index/query/InQueryTest.java Normal file
View File

@ -0,0 +1,24 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.LinkedHashSet;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link InQuery}.
*
* @author guqing
* @since 2.17.0
*/
class InQueryTest {
@Test
void testToString() {
var values = new LinkedHashSet<String>();
values.add("Alice");
values.add("Bob");
var inQuery = new InQuery("name", values);
assertThat(inQuery.toString()).isEqualTo("name IN ('Alice', 'Bob')");
}
}

176
api/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java
View File

@ -1,176 +0,0 @@
package run.halo.app.extension.index.query;
import java.util.Collection;
import java.util.List;
import java.util.Map;
public class IndexViewDataSet {
/**
* Create a {@link QueryIndexView} for employee to test.
*
* @return a {@link QueryIndexView} for employee to test
*/
public static QueryIndexView createEmployeeIndexView() {
/*
* id firstName lastName email hireDate salary managerId departmentId
* 100 Pat Fay p 17 2600 101 50
* 101 Lee Day l 17 2400 102 40
* 102 William Jay w 19 2200 102 50
* 103 Mary Day p 17 2000 103 50
* 104 John Fay j 17 1800 103 50
* 105 Gon Fay p 18 1900 101 40
*/
Collection<Map.Entry<String, String>> idEntry = List.of(
Map.entry("100", "100"),
Map.entry("101", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("104", "104"),
Map.entry("105", "105")
);
Collection<Map.Entry<String, String>> firstNameEntry = List.of(
Map.entry("Pat", "100"),
Map.entry("Lee", "101"),
Map.entry("William", "102"),
Map.entry("Mary", "103"),
Map.entry("John", "104"),
Map.entry("Gon", "105")
);
Collection<Map.Entry<String, String>> lastNameEntry = List.of(
Map.entry("Fay", "100"),
Map.entry("Day", "101"),
Map.entry("Jay", "102"),
Map.entry("Day", "103"),
Map.entry("Fay", "104"),
Map.entry("Fay", "105")
);
Collection<Map.Entry<String, String>> emailEntry = List.of(
Map.entry("p", "100"),
Map.entry("l", "101"),
Map.entry("w", "102"),
Map.entry("p", "103"),
Map.entry("j", "104"),
Map.entry("p", "105")
);
Collection<Map.Entry<String, String>> hireDateEntry = List.of(
Map.entry("17", "100"),
Map.entry("17", "101"),
Map.entry("19", "102"),
Map.entry("17", "103"),
Map.entry("17", "104"),
Map.entry("18", "105")
);
Collection<Map.Entry<String, String>> salaryEntry = List.of(
Map.entry("2600", "100"),
Map.entry("2400", "101"),
Map.entry("2200", "102"),
Map.entry("2000", "103"),
Map.entry("1800", "104"),
Map.entry("1900", "105")
);
Collection<Map.Entry<String, String>> managerIdEntry = List.of(
Map.entry("101", "100"),
Map.entry("102", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("103", "104"),
Map.entry("101", "105")
);
Collection<Map.Entry<String, String>> departmentIdEntry = List.of(
Map.entry("50", "100"),
Map.entry("40", "101"),
Map.entry("50", "102"),
Map.entry("50", "103"),
Map.entry("50", "104"),
Map.entry("40", "105")
);
var entries = Map.of("id", idEntry,
"firstName", firstNameEntry,
"lastName", lastNameEntry,
"email", emailEntry,
"hireDate", hireDateEntry,
"salary", salaryEntry,
"managerId", managerIdEntry,
"departmentId", departmentIdEntry);
return new QueryIndexViewImpl(entries);
}
/**
* Create a {@link QueryIndexView} for post to test.
*
* @return a {@link QueryIndexView} for post to test
*/
public static QueryIndexView createPostIndexViewWithNullCell() {
/*
* id title published publishTime owner
* 100 title1 true 2024-01-01T00:00:00 jack
* 101 title2 true 2024-01-02T00:00:00 rose
* 102 title3 false null smith
* 103 title4 false null peter
* 104 title5 false null john
* 105 title6 true 2024-01-05 00:00:00 tom
* 106 title7 true 2024-01-05 13:00:00 jerry
* 107 title8 true 2024-01-05 12:00:00 jerry
* 108 title9 false null jerry
*/
Collection<Map.Entry<String, String>> idEntry = List.of(
Map.entry("100", "100"),
Map.entry("101", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("104", "104"),
Map.entry("105", "105"),
Map.entry("106", "106"),
Map.entry("107", "107"),
Map.entry("108", "108")
);
Collection<Map.Entry<String, String>> titleEntry = List.of(
Map.entry("title1", "100"),
Map.entry("title2", "101"),
Map.entry("title3", "102"),
Map.entry("title4", "103"),
Map.entry("title5", "104"),
Map.entry("title6", "105"),
Map.entry("title7", "106"),
Map.entry("title8", "107"),
Map.entry("title9", "108")
);
Collection<Map.Entry<String, String>> publishedEntry = List.of(
Map.entry("true", "100"),
Map.entry("true", "101"),
Map.entry("false", "102"),
Map.entry("false", "103"),
Map.entry("false", "104"),
Map.entry("true", "105"),
Map.entry("true", "106"),
Map.entry("true", "107"),
Map.entry("false", "108")
);
Collection<Map.Entry<String, String>> publishTimeEntry = List.of(
Map.entry("2024-01-01T00:00:00", "100"),
Map.entry("2024-01-02T00:00:00", "101"),
Map.entry("2024-01-05 00:00:00", "105"),
Map.entry("2024-01-05 13:00:00", "106"),
Map.entry("2024-01-05 12:00:00", "107")
);
Collection<Map.Entry<String, String>> ownerEntry = List.of(
Map.entry("jack", "100"),
Map.entry("rose", "101"),
Map.entry("smith", "102"),
Map.entry("peter", "103"),
Map.entry("john", "104"),
Map.entry("tom", "105"),
Map.entry("jerry", "106"),
Map.entry("jerry", "107"),
Map.entry("jerry", "108")
);
var entries = Map.of("id", idEntry,
"title", titleEntry,
"published", publishedEntry,
"publishTime", publishTimeEntry,
"owner", ownerEntry);
return new QueryIndexViewImpl(entries);
}
}

20
api/src/test/java/run/halo/app/extension/index/query/IsNotNullTest.java Normal file
View File

@ -0,0 +1,20 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link IsNotNull}.
*
* @author guqing
* @since 2.17.0
*/
class IsNotNullTest {
@Test
void testToString() {
var isNotNull = new IsNotNull("name");
assertThat(isNotNull.toString()).isEqualTo("name IS NOT NULL");
}
}

218
api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java
View File

@ -1,218 +0,0 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.springframework.data.domain.Sort;
/**
* Tests for {@link QueryIndexViewImpl}.
*
* @author guqing
* @since 2.12.0
*/
class QueryIndexViewImplTest {
@Test
void getAllIdsForFieldTest() {
var indexView = IndexViewDataSet.createPostIndexViewWithNullCell();
var resultSet = indexView.getAllIdsForField("title");
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103", "104", "105", "106", "107", "108"
);
resultSet = indexView.getAllIdsForField("publishTime");
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "105", "106", "107"
);
}
@Test
void findIdsForFieldValueEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueEqual("managerId", "id");
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
}
@Test
void findIdsForFieldValueGreaterThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = IndexViewDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
@Test
void findIdsForFieldValueGreaterThanTest2() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = IndexViewDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = IndexViewDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest2() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = IndexViewDataSet.createEmployeeIndexView();
resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
@Nested
class SortTest {
@Test
void testSortByUnsorted() {
Collection<Map.Entry<String, String>> entries = List.of(
Map.entry("Item1", "Item1"),
Map.entry("Item2", "Item2")
);
var indexView = new QueryIndexViewImpl(Map.of("field1", entries));
var sort = Sort.unsorted();
List<String> sortedList = indexView.sortBy(sort);
assertThat(sortedList).isEqualTo(List.of("Item1", "Item2"));
}
@Test
void testSortBySortedAscending() {
var indexEntries = new HashMap<String, Collection<Map.Entry<String, String>>>();
indexEntries.put("field1",
List.of(Map.entry("key2", "Item2"), Map.entry("key1", "Item1")));
var indexView = new QueryIndexViewImpl(indexEntries);
var sort = Sort.by(Sort.Order.asc("field1"));
List<String> sortedList = indexView.sortBy(sort);
assertThat(sortedList).containsExactly("Item1", "Item2");
}
@Test
void testSortBySortedDescending() {
var indexEntries = new HashMap<String, Collection<Map.Entry<String, String>>>();
indexEntries.put("field1",
List.of(Map.entry("key1", "Item1"), Map.entry("key2", "Item2")));
var indexView = new QueryIndexViewImpl(indexEntries);
var sort = Sort.by(Sort.Order.desc("field1"));
List<String> sortedList = indexView.sortBy(sort);
assertThat(sortedList).containsExactly("Item2", "Item1");
}
@Test
void testSortByMultipleFields() {
var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>();
indexEntries.put("field1", List.of(Map.entry("k3", "Item3"), Map.entry("k2", "Item2")));
indexEntries.put("field2", List.of(Map.entry("k1", "Item1"), Map.entry("k3", "Item3")));
var indexView = new QueryIndexViewImpl(indexEntries);
var sort = Sort.by(Sort.Order.asc("field1"), Sort.Order.desc("field2"));
List<String> sortedList = indexView.sortBy(sort);
assertThat(sortedList).containsExactly("Item2", "Item3", "Item1");
}
@Test
void testSortByWithMissingFieldInMap() {
var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>();
var indexView = new QueryIndexViewImpl(indexEntries);
var sort = Sort.by(Sort.Order.asc("missingField"));
assertThatThrownBy(() -> indexView.sortBy(sort))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("Field name missingField is not indexed");
}
@Test
void testSortByMultipleFields2() {
var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>();
var entry1 = List.of(Map.entry("John", "John"),
Map.entry("Bob", "Bob"),
Map.entry("Alice", "Alice")
);
var entry2 = List.of(Map.entry("David", "David"),
Map.entry("Eva", "Eva"),
Map.entry("Frank", "Frank")
);
var entry3 = List.of(Map.entry("George", "George"),
Map.entry("Helen", "Helen"),
Map.entry("Ivy", "Ivy")
);
indexEntries.put("field1", entry1);
indexEntries.put("field2", entry2);
indexEntries.put("field3", entry3);
/*
* <pre>
* Row Key | field1 | field2 | field3
* -------|-------|-------|-------
* John | John | |
* Bob | Bob | |
* Alice | Alice | |
* David | | David |
* Eva | | Eva |
* Frank | | Frank |
* George | | | George
* Helen | | | Helen
* Ivy | | | Ivy
* </pre>
*/
var indexView = new QueryIndexViewImpl(indexEntries);
// "John", "Bob", "Alice", "David", "Eva", "Frank", "George", "Helen", "Ivy"
var sort = Sort.by(Sort.Order.desc("field1"), Sort.Order.asc("field2"),
Sort.Order.asc("field3"));
List<String> sortedList = indexView.sortBy(sort);
assertThat(sortedList).containsSequence("John", "Bob", "Alice", "David", "Eva", "Frank",
"George", "Helen", "Ivy");
}
}
}

20
api/src/test/java/run/halo/app/extension/index/query/StringContainsTest.java Normal file
View File

@ -0,0 +1,20 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link StringContains}.
*
* @author guqing
* @since 2.17.0
*/
class StringContainsTest {
@Test
void testToString() {
var stringContains = new StringContains("name", "Alice");
assertThat(stringContains.toString()).isEqualTo("contains(name, 'Alice')");
}
}

28
application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java
View File

@ -112,8 +112,30 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
@Override
public <E extends Extension> Flux<E> listAll(Class<E> type, ListOptions options, Sort sort) {
return listBy(type, options, PageRequestImpl.ofSize(0).withSort(sort))
.flatMapIterable(ListResult::getItems);
var scheme = schemeManager.get(type);
return Mono.fromSupplier(
() -> indexedQueryEngine.retrieveAll(scheme.groupVersionKind(), options, sort))
.doOnSuccess(objectKeys -> {
if (log.isDebugEnabled()) {
if (objectKeys.size() > 500) {
log.warn("The number of objects retrieved by listAll is too large ({}) "
+ "and it is recommended to use paging query.",
objectKeys.size());
}
}
})
.flatMapMany(objectKeys -> {
var storeNames = objectKeys.stream()
.map(objectKey -> ExtensionStoreUtil.buildStoreName(scheme, objectKey))
.toList();
final long startTimeMs = System.currentTimeMillis();
return client.listByNames(storeNames)
.map(extensionStore -> converter.convertFrom(type, extensionStore))
.doOnNext(s -> {
log.debug("Successfully retrieved all by names from db for {} in {}ms",
scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs);
});
});
}
@Override
@ -130,7 +152,7 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient {
final long startTimeMs = System.currentTimeMillis();
return client.listByNames(storeNames)
.map(extensionStore -> converter.convertFrom(type, extensionStore))
.doFinally(s -> {
.doOnNext(s -> {
log.debug("Successfully retrieved by names from db for {} in {}ms",
scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs);
})

33
application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java
View File

@ -11,6 +11,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import org.apache.commons.lang3.BooleanUtils;
import org.springframework.lang.NonNull;
import run.halo.app.extension.Extension;
/**
@ -172,6 +173,28 @@ public class DefaultIndexer implements Indexer {
}
}
@Override
@NonNull
public IndexEntry getIndexEntry(String name) {
readLock.lock();
try {
var indexDescriptor = findIndexByName(name);
if (indexDescriptor == null) {
throw new IllegalArgumentException(
"No index found for fieldPath [" + name + "], "
+ "make sure you have created an index for this field.");
}
if (!indexDescriptor.isReady()) {
throw new IllegalStateException(
"Index [" + name + "] is not ready, "
+ "Please wait for more time or check the index status.");
}
return indexEntries.get(indexDescriptor);
} finally {
readLock.unlock();
}
}
@Override
public Iterator<IndexEntry> readyIndexesIterator() {
readLock.lock();
@ -197,4 +220,14 @@ public class DefaultIndexer implements Indexer {
readLock.unlock();
}
}
@Override
public void acquireReadLock() {
readLock.lock();
}
@Override
public void releaseReadLock() {
readLock.unlock();
}
}

35
application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java
View File

@ -4,9 +4,11 @@ import com.google.common.collect.ListMultimap;
import com.google.common.collect.MultimapBuilder;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.NavigableSet;
import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -94,10 +96,13 @@ public class IndexEntryImpl implements IndexEntry {
}
@Override
public Set<String> indexedKeys() {
public NavigableSet<String> indexedKeys() {
readLock.lock();
try {
return indexKeyObjectNamesMap.keySet();
var keys = indexKeyObjectNamesMap.keySet();
var resultSet = new TreeSet<>(keyComparator());
resultSet.addAll(keys);
return resultSet;
} finally {
readLock.unlock();
}
@ -114,20 +119,32 @@ public class IndexEntryImpl implements IndexEntry {
}
@Override
public Collection<Map.Entry<String, String>> immutableEntries() {
public Map<String, Integer> getIdPositionMap() {
readLock.lock();
try {
// Copy to a new list to avoid ConcurrentModificationException
return indexKeyObjectNamesMap.entries().stream()
.map(entry -> Map.entry(entry.getKey(), entry.getValue()))
.toList();
// asMap is sorted by key
var keyObjectMap = getKeyObjectMap();
int i = 0;
var idPositionMap = new HashMap<String, Integer>();
for (var valueIdsEntry : keyObjectMap.entrySet()) {
var ids = valueIdsEntry.getValue();
for (String id : ids) {
idPositionMap.put(id, i);
}
i++;
}
return idPositionMap;
} finally {
readLock.unlock();
}
}
protected Map<String, Collection<String>> getKeyObjectMap() {
return indexKeyObjectNamesMap.asMap();
}
@Override
public List<String> getByIndexKey(String indexKey) {
public List<String> getObjectNamesBy(String indexKey) {
readLock.lock();
try {
return indexKeyObjectNamesMap.get(indexKey);

243
application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java
View File

@ -1,15 +1,14 @@
package run.halo.app.extension.index;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
@ -23,6 +22,7 @@ import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.index.query.QueryFactory;
import run.halo.app.extension.index.query.QueryIndexView;
import run.halo.app.extension.index.query.QueryIndexViewImpl;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
@ -41,155 +41,108 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
private final IndexerFactory indexerFactory;
private static Map<String, IndexEntry> fieldPathIndexEntryMap(Indexer indexer) {
// O(n) time complexity
Map<String, IndexEntry> indexEntryMap = new HashMap<>();
var iterator = indexer.readyIndexesIterator();
while (iterator.hasNext()) {
var indexEntry = iterator.next();
var descriptor = indexEntry.getIndexDescriptor();
var indexedFieldPath = descriptor.getSpec().getName();
indexEntryMap.put(indexedFieldPath, indexEntry);
}
return indexEntryMap;
}
static IndexEntry getIndexEntry(String fieldPath, Map<String, IndexEntry> fieldPathEntryMap) {
if (!fieldPathEntryMap.containsKey(fieldPath)) {
throwNotIndexedException(fieldPath);
}
return fieldPathEntryMap.get(fieldPath);
}
@Override
public ListResult<String> retrieve(GroupVersionKind type, ListOptions options,
PageRequest page) {
var indexer = indexerFactory.getIndexer(type);
var allMatchedResult = doRetrieve(indexer, options, page.getSort());
var allMatchedResult = doRetrieve(type, options, page.getSort());
var list = ListResult.subList(allMatchedResult, page.getPageNumber(), page.getPageSize());
return new ListResult<>(page.getPageNumber(), page.getPageSize(),
allMatchedResult.size(), list);
}
@Override
public List<String> retrieveAll(GroupVersionKind type, ListOptions options) {
var indexer = indexerFactory.getIndexer(type);
return doRetrieve(indexer, options, Sort.unsorted());
public List<String> retrieveAll(GroupVersionKind type, ListOptions options, Sort sort) {
return doRetrieve(type, options, sort);
}
static <T> List<T> intersection(List<T> list1, List<T> list2) {
Set<T> set = new LinkedHashSet<>(list1);
List<T> intersection = new ArrayList<>();
for (T item : list2) {
if (set.contains(item) && !intersection.contains(item)) {
intersection.add(item);
}
}
return intersection;
}
static void throwNotIndexedException(String fieldPath) {
throw new IllegalArgumentException(
"No index found for fieldPath: " + fieldPath
+ ", make sure you have created an index for this field.");
}
List<String> retrieveForLabelMatchers(List<SelectorMatcher> labelMatchers,
Map<String, IndexEntry> fieldPathEntryMap, List<String> allMetadataNames) {
var indexEntry = getIndexEntry(LabelIndexSpecUtils.LABEL_PATH, fieldPathEntryMap);
// O(m) time complexity, m is the number of labelMatchers
var labelKeysToQuery = labelMatchers.stream()
.sorted(Comparator.comparing(SelectorMatcher::getKey))
.map(SelectorMatcher::getKey)
.collect(Collectors.toSet());
Map<String, Map<String, String>> objectNameLabelsMap = new HashMap<>();
indexEntry.acquireReadLock();
try {
indexEntry.entries().forEach(entry -> {
// key is labelKey=labelValue, value is objectName
var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey());
if (!labelKeysToQuery.contains(labelPair.getFirst())) {
return;
}
objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>())
.put(labelPair.getFirst(), labelPair.getSecond());
});
} finally {
indexEntry.releaseReadLock();
}
// O(p * m) time complexity, p is the number of allMetadataNames
return allMetadataNames.stream()
.filter(objectName -> {
var labels = objectNameLabelsMap.getOrDefault(objectName, Map.of());
NavigableSet<String> retrieveForLabelMatchers(Indexer indexer,
List<SelectorMatcher> labelMatchers) {
var objectLabelMap = ObjectLabelMap.buildFrom(indexer, labelMatchers);
// O(k×m) time complexity, k is the number of keys, m is the number of labelMatchers
return objectLabelMap.objectIdLabelsMap()
.entrySet()
.stream()
.filter(entry -> {
var labels = entry.getValue();
// object match all labels will be returned
return labelMatchers.stream()
.allMatch(matcher -> matcher.test(labels.get(matcher.getKey())));
})
.toList();
.map(Map.Entry::getKey)
.collect(Collectors.toCollection(TreeSet::new));
}
List<String> doRetrieve(Indexer indexer, ListOptions options, Sort sort) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("build index entry map");
var fieldPathEntryMap = fieldPathIndexEntryMap(indexer);
var primaryEntry = getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, fieldPathEntryMap);
stopWatch.stop();
// O(n) time complexity
stopWatch.start("retrieve all metadata names");
var allMetadataNames = new ArrayList<String>();
primaryEntry.acquireReadLock();
try {
allMetadataNames.addAll(primaryEntry.indexedKeys());
} finally {
primaryEntry.releaseReadLock();
}
stopWatch.stop();
stopWatch.start("build index view");
var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort);
var indexViewMap = new HashMap<String, Collection<Map.Entry<String, String>>>();
for (Map.Entry<String, IndexEntry> entry : fieldPathEntryMap.entrySet()) {
if (!fieldNamesUsedInQuery.contains(entry.getKey())) {
continue;
}
indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries());
}
// TODO optimize build indexView time
var indexView = new QueryIndexViewImpl(indexViewMap);
stopWatch.stop();
stopWatch.start("retrieve matched metadata names");
if (hasLabelSelector(options.getLabelSelector())) {
var matchedByLabels = retrieveForLabelMatchers(options.getLabelSelector().getMatchers(),
fieldPathEntryMap, allMetadataNames);
if (allMetadataNames.size() != matchedByLabels.size()) {
indexView.removeByIdNotIn(new TreeSet<>(matchedByLabels));
}
}
stopWatch.stop();
stopWatch.start("retrieve matched metadata names by fields");
NavigableSet<String> evaluateSelectorsForIndex(Indexer indexer, QueryIndexView indexView,
ListOptions options) {
final var hasLabelSelector = hasLabelSelector(options.getLabelSelector());
final var hasFieldSelector = hasFieldSelector(options.getFieldSelector());
if (hasFieldSelector) {
var fieldSelector = options.getFieldSelector();
var query = fieldSelector.query();
var resultSet = query.matches(indexView);
indexView.removeByIdNotIn(resultSet);
if (!hasLabelSelector && !hasFieldSelector) {
return QueryFactory.all().matches(indexView);
}
// only label selector
if (hasLabelSelector && !hasFieldSelector) {
return retrieveForLabelMatchers(indexer, options.getLabelSelector().getMatchers());
}
// only field selector
if (!hasLabelSelector) {
var fieldSelector = options.getFieldSelector();
return fieldSelector.query().matches(indexView);
}
// both label and field selector
var fieldSelector = options.getFieldSelector();
var forField = fieldSelector.query().matches(indexView);
var forLabel =
retrieveForLabelMatchers(indexer, options.getLabelSelector().getMatchers());
// determine the optimal retainAll direction based on the size of the collection
var resultSet = (forField.size() <= forLabel.size()) ? forField : forLabel;
resultSet.retainAll((resultSet == forField) ? forLabel : forField);
return resultSet;
}
List<String> doRetrieve(GroupVersionKind type, ListOptions options, Sort sort) {
var indexer = indexerFactory.getIndexer(type);
StopWatch stopWatch = new StopWatch(type.toString());
stopWatch.start("Check index status to ensure all indexes are ready");
var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort);
checkIndexForNames(indexer, fieldNamesUsedInQuery);
stopWatch.stop();
stopWatch.start("sort result");
var result = indexView.sortBy(sort);
var indexView = new QueryIndexViewImpl(indexer);
stopWatch.start("Evaluate selectors for index");
var resultSet = evaluateSelectorsForIndex(indexer, indexView, options);
stopWatch.stop();
stopWatch.start("Sort result set by sort order");
var result = indexView.sortBy(resultSet, sort);
stopWatch.stop();
if (log.isTraceEnabled()) {
log.trace("Retrieve result from indexer, {}", stopWatch.prettyPrint());
log.trace("Retrieve result from indexer by query [{}],\n {}", options,
stopWatch.prettyPrint(TimeUnit.MILLISECONDS));
}
return result;
}
void checkIndexForNames(Indexer indexer, Set<String> indexNames) {
indexer.acquireReadLock();
try {
for (String indexName : indexNames) {
// get index entry will throw exception if index not found
indexer.getIndexEntry(indexName);
}
} finally {
indexer.releaseReadLock();
}
}
@NonNull
private Set<String> getFieldNamesUsedInListOptions(ListOptions options, Sort sort) {
var fieldNamesUsedInQuery = new HashSet<String>();
@ -213,4 +166,46 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
boolean hasFieldSelector(FieldSelector fieldSelector) {
return fieldSelector != null;
}
record ObjectLabelMap(Map<String, Map<String, String>> objectIdLabelsMap) {
public static ObjectLabelMap buildFrom(Indexer indexer,
List<SelectorMatcher> labelMatchers) {
indexer.acquireReadLock();
try {
final var objectNameLabelsMap = new HashMap<String, Map<String, String>>();
final var labelIndexEntry = indexer.getIndexEntry(LabelIndexSpecUtils.LABEL_PATH);
// O(m) time complexity, m is the number of labelMatchers
final var labelKeysToQuery = labelMatchers.stream()
.sorted(Comparator.comparing(SelectorMatcher::getKey))
.map(SelectorMatcher::getKey)
.collect(Collectors.toSet());
labelIndexEntry.entries().forEach(entry -> {
// key is labelKey=labelValue, value is objectName
var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey());
if (!labelKeysToQuery.contains(labelPair.getFirst())) {
return;
}
objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>())
.put(labelPair.getFirst(), labelPair.getSecond());
});
var nameIndexOperator = new IndexEntryOperatorImpl(
indexer.getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)
);
var allIndexedObjectNames = nameIndexOperator.getValues();
// remove all object names that exist labels,O(n) time complexity
allIndexedObjectNames.removeAll(objectNameLabelsMap.keySet());
// add absent object names to object labels map
for (String name : allIndexedObjectNames) {
objectNameLabelsMap.put(name, new HashMap<>());
}
return new ObjectLabelMap(objectNameLabelsMap);
} finally {
indexer.releaseReadLock();
}
}
}
}

52
application/src/test/java/run/halo/app/content/PostQueryTest.java
View File

@ -1,52 +0,0 @@
package run.halo.app.content;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.mock.web.reactive.function.server.MockServerRequest;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.server.ServerWebExchange;
import run.halo.app.extension.index.query.QueryIndexViewImpl;
/**
* Tests for {@link PostQuery}.
*
* @author guqing
* @since 2.6.0
*/
@ExtendWith(MockitoExtension.class)
class PostQueryTest {
@Test
void userScopedQueryTest() {
MultiValueMap<String, String> multiValueMap = new LinkedMultiValueMap<>();
MockServerRequest request = MockServerRequest.builder()
.queryParams(multiValueMap)
.exchange(mock(ServerWebExchange.class))
.build();
PostQuery postQuery = new PostQuery(request, "faker");
var listOptions = postQuery.toListOptions();
assertThat(listOptions).isNotNull();
assertThat(listOptions.getFieldSelector()).isNotNull();
var nameEntry =
(Collection<Map.Entry<String, String>>) List.of(Map.entry("metadata.name", "faker"));
var entry = (Collection<Map.Entry<String, String>>) List.of(Map.entry("faker", "faker"));
var indexView =
new QueryIndexViewImpl(Map.of("spec.owner", entry, "metadata.name", nameEntry));
assertThat(listOptions.getFieldSelector().query().matches(indexView))
.containsExactly("faker");
entry = List.of(Map.entry("another-faker", "user1"));
indexView = new QueryIndexViewImpl(Map.of("spec.owner", entry, "metadata.name", nameEntry));
assertThat(listOptions.getFieldSelector().query().matches(indexView)).isEmpty();
}
}

61
application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java
View File

@ -24,6 +24,28 @@ import run.halo.app.infra.exception.DuplicateNameException;
@ExtendWith(MockitoExtension.class)
class DefaultIndexerTest {
private static FakeExtension createFakeExtension() {
var fake = new FakeExtension();
fake.setMetadata(new Metadata());
fake.getMetadata().setName("fake-extension");
fake.setEmail("fake-email");
return fake;
}
private static IndexSpec getNameIndexSpec() {
return getIndexSpec("metadata.name", true,
IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName()));
}
private static IndexSpec getIndexSpec(String name, boolean unique, IndexAttribute attribute) {
return new IndexSpec()
.setName(name)
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(unique)
.setIndexFunc(attribute);
}
@Test
void constructor() {
var spec = getNameIndexSpec();
@ -48,20 +70,29 @@ class DefaultIndexerTest {
.hasMessage("Index entry not found for: metadata.name");
}
@Test
void getIndexEntryTest() {
var spec = getNameIndexSpec();
var descriptor = new IndexDescriptor(spec);
descriptor.setReady(true);
var indexContainer = new IndexEntryContainer();
indexContainer.add(new IndexEntryImpl(descriptor));
var defaultIndexer = new DefaultIndexer(List.of(descriptor), indexContainer);
assertThatThrownBy(() -> defaultIndexer.getIndexEntry("not-exist"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("No index found for fieldPath [not-exist], "
+ "make sure you have created an index for this field.");
assertThat(defaultIndexer.getIndexEntry("metadata.name")).isNotNull();
}
@Test
void getObjectKey() {
var fake = createFakeExtension();
assertThat(DefaultIndexer.getObjectKey(fake)).isEqualTo("fake-extension");
}
private static FakeExtension createFakeExtension() {
var fake = new FakeExtension();
fake.setMetadata(new Metadata());
fake.getMetadata().setName("fake-extension");
fake.setEmail("fake-email");
return fake;
}
@Test
void indexRecord() {
var nameIndex = getNameIndexSpec();
@ -81,12 +112,6 @@ class DefaultIndexerTest {
assertThat(entries).contains(Map.entry("fake-extension", "fake-extension"));
}
private static IndexSpec getNameIndexSpec() {
return getIndexSpec("metadata.name", true,
IndexAttributeFactory.simpleAttribute(FakeExtension.class,
e -> e.getMetadata().getName()));
}
@Test
void indexRecordWithExceptionShouldRollback() {
var indexContainer = new IndexEntryContainer();
@ -242,14 +267,6 @@ class DefaultIndexerTest {
assertThat(iterator.hasNext()).isFalse();
}
private static IndexSpec getIndexSpec(String name, boolean unique, IndexAttribute attribute) {
return new IndexSpec()
.setName(name)
.setOrder(IndexSpec.OrderType.ASC)
.setUnique(unique)
.setIndexFunc(attribute);
}
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakeextensions",

48
application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java
View File

@ -1,12 +1,19 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static run.halo.app.extension.index.query.IndexViewDataSet.createCommentIndexView;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.extension.index.query.IndexViewDataSet;
import run.halo.app.extension.index.query.QueryIndexView;
/**
* Tests for {@link IndexEntryImpl}.
@ -58,7 +65,7 @@ class IndexEntryImplTest {
}
@Test
void getByIndexKey() {
void getObjectIdsTest() {
var spec =
PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class);
var descriptor = new IndexDescriptor(spec);
@ -67,7 +74,7 @@ class IndexEntryImplTest {
assertThat(entry.indexedKeys()).containsExactly("slug-1", "slug-2");
assertThat(entry.entries()).hasSize(2);
assertThat(entry.getByIndexKey("slug-1")).isEqualTo(List.of("fake-name-1"));
assertThat(entry.getObjectNamesBy("slug-1")).isEqualTo(List.of("fake-name-1"));
}
@Test
@ -91,7 +98,6 @@ class IndexEntryImplTest {
assertThat(entry.indexedKeys()).containsSequence("slug-4", "slug-3", "slug-2", "slug-1");
spec.setOrder(IndexSpec.OrderType.ASC);
var descriptor2 = new IndexDescriptor(spec);
var entry2 = new IndexEntryImpl(descriptor2);
@ -105,4 +111,40 @@ class IndexEntryImplTest {
Map.entry("slug-4", "fake-name-3"));
assertThat(entry2.indexedKeys()).containsSequence("slug-1", "slug-2", "slug-3", "slug-4");
}
@Test
void getIdPositionMapTest() {
var indexView = createCommentIndexView();
var topIndexEntry = prepareForPositionMapTest(indexView, "spec.top");
var topIndexEntryFromView = indexView.getIndexEntry("spec.top");
assertThat(topIndexEntry.getIdPositionMap())
.isEqualTo(topIndexEntryFromView.getIdPositionMap());
var creationTimeIndexEntry = prepareForPositionMapTest(indexView, "spec.creationTime");
var creationTimeIndexEntryFromView = indexView.getIndexEntry("spec.creationTime");
assertThat(creationTimeIndexEntry.getIdPositionMap())
.isEqualTo(creationTimeIndexEntryFromView.getIdPositionMap());
var priorityIndexEntry = prepareForPositionMapTest(indexView, "spec.priority");
var priorityIndexEntryFromView = indexView.getIndexEntry("spec.priority");
assertThat(priorityIndexEntry.getIdPositionMap())
.isEqualTo(priorityIndexEntryFromView.getIdPositionMap());
}
IndexEntry prepareForPositionMapTest(QueryIndexView indexView, String property) {
var indexSpec = mock(IndexSpec.class);
var descriptor = mock(IndexDescriptor.class);
when(descriptor.getSpec()).thenReturn(indexSpec);
var indexEntry = new IndexEntryImpl(descriptor);
var indexEntryFromView = indexView.getIndexEntry(property);
var sortedEntries = IndexViewDataSet.sortEntries(indexEntryFromView.entries());
var spyIndexEntry = spy(indexEntry);
doReturn(IndexViewDataSet.toKeyObjectMap(sortedEntries))
.when(spyIndexEntry).getKeyObjectMap();
return spyIndexEntry;
}
}

177
application/src/test/java/run/halo/app/extension/index/IndexEntryOperatorImplTest.java Normal file
View File

@ -0,0 +1,177 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import run.halo.app.extension.index.query.IndexViewDataSet;
/**
* Tests for {@link IndexEntryOperatorImpl}.
*
* @author guqing
* @since 2.17.0
*/
@ExtendWith(MockitoExtension.class)
class IndexEntryOperatorImplTest {
@Mock
private IndexEntry indexEntry;
@InjectMocks
private IndexEntryOperatorImpl indexEntryOperator;
private LinkedHashMap<String, List<String>> createIndexedMapAndPile() {
var entries = new ArrayList<Map.Entry<String, String>>();
entries.add(Map.entry("apple", "A"));
entries.add(Map.entry("banana", "B"));
entries.add(Map.entry("cherry", "C"));
entries.add(Map.entry("date", "D"));
entries.add(Map.entry("egg", "E"));
entries.add(Map.entry("f", "F"));
var indexedMap = IndexViewDataSet.toKeyObjectMap(entries);
lenient().when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet()));
lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> {
var key = (String) invocation.getArgument(0);
return indexedMap.get(key);
});
lenient().when(indexEntry.entries()).thenReturn(entries);
return indexedMap;
}
@Test
void lessThan() {
final var indexedMap = createIndexedMapAndPile();
var result = indexEntryOperator.lessThan("banana", false);
assertThat(result).containsExactly("A");
result = indexEntryOperator.lessThan("banana", true);
assertThat(result).containsExactly("A", "B");
result = indexEntryOperator.lessThan("cherry", true);
assertThat(result).containsExactly("A", "B", "C");
// does not exist key
result = indexEntryOperator.lessThan("z", false);
var objectIds = indexedMap.values().stream()
.flatMap(Collection::stream)
.toArray(String[]::new);
assertThat(result).contains(objectIds);
result = indexEntryOperator.lessThan("a", false);
assertThat(result).isEmpty();
}
@Test
void greaterThan() {
createIndexedMapAndPile();
var result = indexEntryOperator.greaterThan("banana", false);
assertThat(result).containsExactly("C", "D", "E", "F");
result = indexEntryOperator.greaterThan("banana", true);
assertThat(result).containsExactly("B", "C", "D", "E", "F");
result = indexEntryOperator.greaterThan("cherry", true);
assertThat(result).containsExactly("C", "D", "E", "F");
result = indexEntryOperator.greaterThan("cherry", false);
assertThat(result).containsExactly("D", "E", "F");
// does not exist key
result = indexEntryOperator.greaterThan("z", false);
assertThat(result).isEmpty();
}
@Test
void greaterThanForNumberString() {
var entries = List.of(
Map.entry("100", "1"),
Map.entry("101", "2"),
Map.entry("102", "3"),
Map.entry("103", "4"),
Map.entry("110", "5"),
Map.entry("111", "6"),
Map.entry("112", "7"),
Map.entry("120", "8")
);
var indexedMap = IndexViewDataSet.toKeyObjectMap(entries);
when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet()));
lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> {
var key = (String) invocation.getArgument(0);
return indexedMap.get(key);
});
when(indexEntry.entries()).thenReturn(entries);
var result = indexEntryOperator.greaterThan("102", false);
assertThat(result).containsExactly("4", "5", "6", "7", "8");
result = indexEntryOperator.greaterThan("110", false);
assertThat(result).containsExactly("6", "7", "8");
}
@Test
void range() {
createIndexedMapAndPile();
var result = indexEntryOperator.range("banana", "date", true, false);
assertThat(result).containsExactly("B", "C");
result = indexEntryOperator.range("banana", "date", false, false);
assertThat(result).containsExactly("C");
result = indexEntryOperator.range("banana", "date", true, true);
assertThat(result).containsExactly("B", "C", "D");
result = indexEntryOperator.range("apple", "egg", false, true);
assertThat(result).containsExactly("B", "C", "D", "E");
// end not exist
result = indexEntryOperator.range("d", "z", false, false);
assertThat(result).containsExactly("D", "E", "F");
// start key > end key
assertThatThrownBy(() -> indexEntryOperator.range("z", "f", false, false))
.isInstanceOf(IllegalArgumentException.class);
// both not exist
result = indexEntryOperator.range("z", "zz", false, false);
assertThat(result).isEmpty();
}
@Test
void findTest() {
createIndexedMapAndPile();
var result = indexEntryOperator.find("banana");
assertThat(result).containsExactly("B");
result = indexEntryOperator.find("date");
assertThat(result).containsExactly("D");
result = indexEntryOperator.find("z");
assertThat(result).isEmpty();
}
@Test
void findInTest() {
createIndexedMapAndPile();
var result = indexEntryOperator.findIn(List.of("banana", "date"));
assertThat(result).containsExactly("B", "D");
}
}

145
application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java
View File

@ -1,7 +1,6 @@
package run.halo.app.extension.index;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
@ -10,13 +9,12 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.junit.jupiter.api.Nested;
@ -51,16 +49,6 @@ class IndexedQueryEngineImplTest {
@InjectMocks
private IndexedQueryEngineImpl indexedQueryEngine;
@Test
void getIndexEntry() {
Map<String, IndexEntry> indexMap = new HashMap<>();
assertThatThrownBy(() -> IndexedQueryEngineImpl.getIndexEntry("field1", indexMap))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage(
"No index found for fieldPath: field1, make sure you have created an index for "
+ "this field.");
}
@Test
void retrieve() {
var spyIndexedQueryEngine = spy(indexedQueryEngine);
@ -69,8 +57,6 @@ class IndexedQueryEngineImplTest {
var gvk = GroupVersionKind.fromExtension(DemoExtension.class);
when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class));
var pageRequest = mock(PageRequest.class);
when(pageRequest.getPageNumber()).thenReturn(1);
when(pageRequest.getPageSize()).thenReturn(2);
@ -80,8 +66,7 @@ class IndexedQueryEngineImplTest {
assertThat(result.getItems()).containsExactly("object1", "object2");
assertThat(result.getTotal()).isEqualTo(3);
verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted()));
verify(indexerFactory).getIndexer(eq(gvk));
verify(spyIndexedQueryEngine).doRetrieve(eq(gvk), any(), eq(Sort.unsorted()));
verify(pageRequest, times(2)).getPageNumber();
verify(pageRequest, times(2)).getPageSize();
verify(pageRequest).getSort();
@ -95,85 +80,53 @@ class IndexedQueryEngineImplTest {
var gvk = GroupVersionKind.fromExtension(DemoExtension.class);
when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class));
var result = spyIndexedQueryEngine.retrieveAll(gvk, new ListOptions());
var result = spyIndexedQueryEngine.retrieveAll(gvk, new ListOptions(), Sort.unsorted());
assertThat(result).isEmpty();
verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted()));
verify(indexerFactory).getIndexer(eq(gvk));
verify(spyIndexedQueryEngine).doRetrieve(eq(gvk), any(), eq(Sort.unsorted()));
}
@Test
void doRetrieve() {
var indexer = mock(Indexer.class);
var labelEntry = mock(IndexEntry.class);
var fieldSlugEntry = mock(IndexEntry.class);
var nameEntry = mock(IndexEntry.class);
when(indexer.readyIndexesIterator()).thenReturn(
List.of(labelEntry, nameEntry, fieldSlugEntry).iterator());
when(nameEntry.getIndexDescriptor())
.thenReturn(new IndexDescriptor(
PrimaryKeySpecUtils.primaryKeyIndexSpec(DemoExtension.class)));
when(nameEntry.indexedKeys()).thenReturn(Set.of("object1", "object2", "object3"));
var gvk = GroupVersionKind.fromExtension(DemoExtension.class);
when(fieldSlugEntry.getIndexDescriptor())
.thenReturn(new IndexDescriptor(new IndexSpec()
.setName("slug")
.setOrder(IndexSpec.OrderType.ASC)));
when((fieldSlugEntry.immutableEntries())).thenReturn(
List.of(Map.entry("slug1", "object1"), Map.entry("slug2", "object2")));
when(indexerFactory.getIndexer(eq(gvk))).thenReturn(indexer);
when(labelEntry.getIndexDescriptor())
.thenReturn(
new IndexDescriptor(LabelIndexSpecUtils.labelIndexSpec(DemoExtension.class)));
when(labelEntry.entries()).thenReturn(List.of(
pileForIndexer(indexer, PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, List.of(
Map.entry("object1", "object1"),
Map.entry("object2", "object2"),
Map.entry("object3", "object3")
));
pileForIndexer(indexer, LabelIndexSpecUtils.LABEL_PATH, List.of(
Map.entry("key1=value1", "object1"),
Map.entry("key2=value2", "object1"),
Map.entry("key1=value1", "object2"),
Map.entry("key2=value2", "object2"),
Map.entry("key1=value1", "object3")
));
pileForIndexer(indexer, "slug", List.of(
Map.entry("slug1", "object1"),
Map.entry("slug2", "object2")
));
var listOptions = new ListOptions();
listOptions.setLabelSelector(LabelSelector.builder()
.eq("key1", "value1").build());
listOptions.setFieldSelector(FieldSelector.of(equal("slug", "slug1")));
var result = indexedQueryEngine.doRetrieve(indexer, listOptions, Sort.unsorted());
var result = indexedQueryEngine.doRetrieve(gvk, listOptions, Sort.unsorted());
assertThat(result).containsExactly("object1");
}
@Test
void intersection() {
var list1 = Arrays.asList(1, 2, 3, 4);
var list2 = Arrays.asList(3, 4, 5, 6);
var expected = Arrays.asList(3, 4);
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test", version = "v1", kind = "demo", plural = "demos", singular = "demo")
static class DemoExtension extends AbstractExtension {
list1 = Arrays.asList(1, 2, 3);
list2 = Arrays.asList(4, 5, 6);
expected = List.of();
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
list1 = List.of();
list2 = Arrays.asList(1, 2, 3);
expected = List.of();
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
list1 = Arrays.asList(1, 2, 3);
list2 = List.of();
expected = List.of();
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
list1 = List.of();
list2 = List.of();
expected = List.of();
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
list1 = Arrays.asList(1, 2, 2, 3);
list2 = Arrays.asList(2, 3, 3, 4);
expected = Arrays.asList(2, 3);
assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected);
}
@Nested
@ -186,10 +139,6 @@ class IndexedQueryEngineImplTest {
void testRetrieveForLabelMatchers() {
// Setup mocks
IndexEntry indexEntryMock = mock(IndexEntry.class);
Map<String, IndexEntry> fieldPathEntryMap =
Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock);
List<String> allMetadataNames = Arrays.asList("object1", "object2", "object3");
// Setup mock behavior
when(indexEntryMock.entries())
.thenReturn(List.of(Map.entry("key1=value1", "object1"),
@ -203,23 +152,22 @@ class IndexedQueryEngineImplTest {
List<SelectorMatcher> labelMatchers = Arrays.asList(matcher1, matcher2);
// Expected results
List<String> expected = Arrays.asList("object1", "object2");
var indexer = mock(Indexer.class);
when(indexer.getIndexEntry(eq(LabelIndexSpecUtils.LABEL_PATH)))
.thenReturn(indexEntryMock);
var nameIndexEntry = mock(IndexEntry.class);
when(indexer.getIndexEntry(eq(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)))
.thenReturn(nameIndexEntry);
when(nameIndexEntry.entries()).thenReturn(List.of(Map.entry("object1", "object1"),
Map.entry("object2", "object2"), Map.entry("object3", "object3")));
// Test
assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap,
allMetadataNames))
.isEqualTo(expected);
assertThat(indexedQueryEngine.retrieveForLabelMatchers(indexer, labelMatchers))
.containsSequence("object1", "object2");
}
@Test
void testRetrieveForLabelMatchersNoMatch() {
// Setup mocks
IndexEntry indexEntryMock = mock(IndexEntry.class);
Map<String, IndexEntry> fieldPathEntryMap =
Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock);
List<String> allMetadataNames = Arrays.asList("object1", "object2", "object3");
// Setup mock behavior
when(indexEntryMock.entries())
.thenReturn(List.of(Map.entry("key1=value1", "object1"),
@ -230,20 +178,17 @@ class IndexedQueryEngineImplTest {
var matcher1 = EqualityMatcher.equal("key3", "value3");
List<SelectorMatcher> labelMatchers = List.of(matcher1);
// Expected results
List<String> expected = List.of();
var indexer = mock(Indexer.class);
when(indexer.getIndexEntry(eq(LabelIndexSpecUtils.LABEL_PATH)))
.thenReturn(indexEntryMock);
var nameIndexEntry = mock(IndexEntry.class);
when(indexer.getIndexEntry(eq(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME)))
.thenReturn(nameIndexEntry);
when(nameIndexEntry.entries()).thenReturn(List.of(Map.entry("object1", "object1"),
Map.entry("object2", "object2"), Map.entry("object3", "object3")));
// Test
assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap,
allMetadataNames))
.isEqualTo(expected);
assertThat(
indexedQueryEngine.retrieveForLabelMatchers(indexer, labelMatchers)).isEmpty();
}
}
@Data
@EqualsAndHashCode(callSuper = true)
@GVK(group = "test", version = "v1", kind = "demo", plural = "demos", singular = "demo")
static class DemoExtension extends AbstractExtension {
}
}

30
api/src/test/java/run/halo/app/extension/index/query/AndTest.java → application/src/test/java/run/halo/app/extension/index/query/AndTest.java
View File

@ -1,15 +1,17 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.greaterThan;
import static run.halo.app.extension.index.query.QueryFactory.or;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import run.halo.app.extension.index.Indexer;
/**
* Tests for the {@link And} query.
@ -21,33 +23,45 @@ public class AndTest {
@Test
void testMatches() {
Collection<Map.Entry<String, String>> deptEntry = List.of(Map.entry("A", "guqing"),
var deptEntry = List.of(Map.entry("A", "guqing"),
Map.entry("A", "halo"),
Map.entry("B", "lisi"),
Map.entry("B", "zhangsan"),
Map.entry("C", "ryanwang"),
Map.entry("C", "johnniang")
);
Collection<Map.Entry<String, String>> ageEntry = List.of(Map.entry("19", "halo"),
var ageEntry = List.of(Map.entry("19", "halo"),
Map.entry("19", "guqing"),
Map.entry("18", "zhangsan"),
Map.entry("17", "lisi"),
Map.entry("17", "ryanwang"),
Map.entry("17", "johnniang")
);
var entries = Map.of("dept", deptEntry, "age", ageEntry);
var indexView = new QueryIndexViewImpl(entries);
var indexer = mock(Indexer.class);
pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, List.of(
Map.entry("guqing", "guqing"),
Map.entry("halo", "halo"),
Map.entry("lisi", "lisi"),
Map.entry("zhangsan", "zhangsan"),
Map.entry("ryanwang", "ryanwang"),
Map.entry("johnniang", "johnniang")
));
pileForIndexer(indexer, "dept", deptEntry);
pileForIndexer(indexer, "age", ageEntry);
var indexView = new QueryIndexViewImpl(indexer);
var query = and(equal("dept", "B"), equal("age", "18"));
var resultSet = query.matches(indexView);
assertThat(resultSet).containsExactly("zhangsan");
indexView = new QueryIndexViewImpl(entries);
query = and(equal("dept", "C"), equal("age", "18"));
resultSet = query.matches(indexView);
assertThat(resultSet).isEmpty();
indexView = new QueryIndexViewImpl(entries);
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "B")),
@ -57,7 +71,6 @@ public class AndTest {
resultSet = query.matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan");
indexView = new QueryIndexViewImpl(entries);
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "B")),
@ -67,7 +80,6 @@ public class AndTest {
resultSet = query.matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan");
indexView = new QueryIndexViewImpl(entries);
query = and(
// guqing, halo, lisi, zhangsan
or(equal("dept", "A"), equal("dept", "C")),

340
application/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java Normal file
View File

@ -0,0 +1,340 @@
package run.halo.app.extension.index.query;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import java.util.stream.Collectors;
import run.halo.app.extension.index.IndexEntry;
import run.halo.app.extension.index.Indexer;
import run.halo.app.extension.index.KeyComparator;
public class IndexViewDataSet {
/**
* Create a {@link QueryIndexView} for employee to test.
* <pre>
* | id | firstName | lastName | email | hireDate | salary | managerId | departmentId |
* |----|-----------|----------|-------|----------|--------|-----------|--------------|
* | 100| Pat | Fay | p | 17 | 2600 | 101 | 50 |
* | 101| Lee | Day | l | 17 | 2400 | 102 | 40 |
* | 102| William | Jay | w | 19 | 2200 | 102 | 50 |
* | 103| Mary | Day | p | 17 | 2000 | 103 | 50 |
* | 104| John | Fay | j | 17 | 1800 | 103 | 50 |
* | 105| Gon | Fay | p | 18 | 1900 | 101 | 40 |
* </pre>
*
* @return a {@link QueryIndexView} for employee to test
*/
public static QueryIndexView createEmployeeIndexView() {
final var idEntry = List.of(
Map.entry("100", "100"),
Map.entry("101", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("104", "104"),
Map.entry("105", "105")
);
final var firstNameEntry = List.of(
Map.entry("Pat", "100"),
Map.entry("Lee", "101"),
Map.entry("William", "102"),
Map.entry("Mary", "103"),
Map.entry("John", "104"),
Map.entry("Gon", "105")
);
final var lastNameEntry = List.of(
Map.entry("Fay", "100"),
Map.entry("Day", "101"),
Map.entry("Jay", "102"),
Map.entry("Day", "103"),
Map.entry("Fay", "104"),
Map.entry("Fay", "105")
);
final var emailEntry = List.of(
Map.entry("p", "100"),
Map.entry("l", "101"),
Map.entry("w", "102"),
Map.entry("p", "103"),
Map.entry("j", "104"),
Map.entry("p", "105")
);
final var hireDateEntry = List.of(
Map.entry("17", "100"),
Map.entry("17", "101"),
Map.entry("19", "102"),
Map.entry("17", "103"),
Map.entry("17", "104"),
Map.entry("18", "105")
);
final var salaryEntry = List.of(
Map.entry("2600", "100"),
Map.entry("2400", "101"),
Map.entry("2200", "102"),
Map.entry("2000", "103"),
Map.entry("1800", "104"),
Map.entry("1900", "105")
);
final var managerIdEntry = List.of(
Map.entry("101", "100"),
Map.entry("102", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("103", "104"),
Map.entry("101", "105")
);
final var departmentIdEntry = List.of(
Map.entry("50", "100"),
Map.entry("40", "101"),
Map.entry("50", "102"),
Map.entry("50", "103"),
Map.entry("50", "104"),
Map.entry("40", "105")
);
var indexer = mock(Indexer.class);
pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry);
pileForIndexer(indexer, "firstName", firstNameEntry);
pileForIndexer(indexer, "lastName", lastNameEntry);
pileForIndexer(indexer, "email", emailEntry);
pileForIndexer(indexer, "hireDate", hireDateEntry);
pileForIndexer(indexer, "salary", salaryEntry);
pileForIndexer(indexer, "managerId", managerIdEntry);
pileForIndexer(indexer, "departmentId", departmentIdEntry);
return new QueryIndexViewImpl(indexer);
}
/**
* Create a {@link QueryIndexView} for post to test.
* <pre>
* | id | title | published | publishTime | owner |
* |-----|--------|-----------|---------------------|-------|
* | 100 | title1 | true | 2024-01-01T00:00:00 | jack |
* | 101 | title2 | true | 2024-01-02T00:00:00 | rose |
* | 102 | title3 | false | null | smith |
* | 103 | title4 | false | null | peter |
* | 104 | title5 | false | null | john |
* | 105 | title6 | true | 2024-01-05 00:00:00 | tom |
* | 106 | title7 | true | 2024-01-05 13:00:00 | jerry |
* | 107 | title8 | true | 2024-01-05 12:00:00 | jerry |
* | 108 | title9 | false | null | jerry |
* </pre>
*
* @return a {@link QueryIndexView} for post to test
*/
public static QueryIndexView createPostIndexViewWithNullCell() {
final var idEntry = List.of(
Map.entry("100", "100"),
Map.entry("101", "101"),
Map.entry("102", "102"),
Map.entry("103", "103"),
Map.entry("104", "104"),
Map.entry("105", "105"),
Map.entry("106", "106"),
Map.entry("107", "107"),
Map.entry("108", "108")
);
final var titleEntry = List.of(
Map.entry("title1", "100"),
Map.entry("title2", "101"),
Map.entry("title3", "102"),
Map.entry("title4", "103"),
Map.entry("title5", "104"),
Map.entry("title6", "105"),
Map.entry("title7", "106"),
Map.entry("title8", "107"),
Map.entry("title9", "108")
);
final var publishedEntry = List.of(
Map.entry("true", "100"),
Map.entry("true", "101"),
Map.entry("false", "102"),
Map.entry("false", "103"),
Map.entry("false", "104"),
Map.entry("true", "105"),
Map.entry("true", "106"),
Map.entry("true", "107"),
Map.entry("false", "108")
);
final var publishTimeEntry = List.of(
Map.entry("2024-01-01T00:00:00", "100"),
Map.entry("2024-01-02T00:00:00", "101"),
Map.entry("2024-01-05 00:00:00", "105"),
Map.entry("2024-01-05 13:00:00", "106"),
Map.entry("2024-01-05 12:00:00", "107")
);
final var ownerEntry = List.of(
Map.entry("jack", "100"),
Map.entry("rose", "101"),
Map.entry("smith", "102"),
Map.entry("peter", "103"),
Map.entry("john", "104"),
Map.entry("tom", "105"),
Map.entry("jerry", "106"),
Map.entry("jerry", "107"),
Map.entry("jerry", "108")
);
var indexer = mock(Indexer.class);
pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry);
pileForIndexer(indexer, "title", titleEntry);
pileForIndexer(indexer, "published", publishedEntry);
pileForIndexer(indexer, "publishTime", publishTimeEntry);
pileForIndexer(indexer, "owner", ownerEntry);
return new QueryIndexViewImpl(indexer);
}
/**
* Creates a fake comment index view for below data.
* <pre>
* | Name | Top | Priority | Creation Time |
* | ---- | ----- | -------- | -------------------------------- |
* | 1 | True | 0 | 2024-06-05 02:58:15.633165+00:00 |
* | 2 | True | 1 | 2024-06-05 02:58:16.633165+00:00 |
* | 4 | True | 2 | 2024-06-05 02:58:18.633165+00:00 |
* | 3 | True | 2 | 2024-06-05 02:58:17.633165+00:00 |
* | 5 | True | 3 | 2024-06-05 02:58:18.633165+00:00 |
* | 6 | True | 3 | 2024-06-05 02:58:18.633165+00:00 |
* | 10 | False | 0 | 2024-06-05 02:58:17.633165+00:00 |
* | 9 | False | 0 | 2024-06-05 02:58:17.633165+00:00 |
* | 8 | False | 0 | 2024-06-05 02:58:16.633165+00:00 |
* | 7 | False | 0 | 2024-06-05 02:58:15.633165+00:00 |
* | 11 | False | 0 | 2024-06-05 02:58:14.633165+00:00 |
* | 12 | False | 1 | 2024-06-05 02:58:14.633165+00:00 |
* | 14 | False | 3 | 2024-06-05 02:58:17.633165+00:00 |
* | 13 | False | 3 | 2024-06-05 02:58:14.633165+00:00 |
* </pre>
*/
public static QueryIndexView createCommentIndexView() {
final var idEntry = List.of(
Map.entry("1", "1"),
Map.entry("2", "2"),
Map.entry("3", "3"),
Map.entry("4", "4"),
Map.entry("5", "5"),
Map.entry("6", "6"),
Map.entry("7", "7"),
Map.entry("8", "8"),
Map.entry("9", "9"),
Map.entry("10", "10"),
Map.entry("11", "11"),
Map.entry("12", "12"),
Map.entry("13", "13"),
Map.entry("14", "14")
);
final var creationTimeEntry = List.of(
Map.entry("2024-06-05 02:58:15.633165", "1"),
Map.entry("2024-06-05 02:58:16.633165", "2"),
Map.entry("2024-06-05 02:58:17.633165", "3"),
Map.entry("2024-06-05 02:58:18.633165", "4"),
Map.entry("2024-06-05 02:58:18.633165", "5"),
Map.entry("2024-06-05 02:58:18.633165", "6"),
Map.entry("2024-06-05 02:58:15.633165", "7"),
Map.entry("2024-06-05 02:58:16.633165", "8"),
Map.entry("2024-06-05 02:58:17.633165", "9"),
Map.entry("2024-06-05 02:58:17.633165", "10"),
Map.entry("2024-06-05 02:58:14.633165", "11"),
Map.entry("2024-06-05 02:58:14.633165", "12"),
Map.entry("2024-06-05 02:58:14.633165", "13"),
Map.entry("2024-06-05 02:58:17.633165", "14")
);
final var topEntry = List.of(
Map.entry("true", "1"),
Map.entry("true", "2"),
Map.entry("true", "3"),
Map.entry("true", "4"),
Map.entry("true", "5"),
Map.entry("true", "6"),
Map.entry("false", "7"),
Map.entry("false", "8"),
Map.entry("false", "9"),
Map.entry("false", "10"),
Map.entry("false", "11"),
Map.entry("false", "12"),
Map.entry("false", "13"),
Map.entry("false", "14")
);
final var priorityEntry = List.of(
Map.entry("0", "1"),
Map.entry("1", "2"),
Map.entry("2", "3"),
Map.entry("2", "4"),
Map.entry("3", "5"),
Map.entry("3", "6"),
Map.entry("0", "7"),
Map.entry("0", "8"),
Map.entry("0", "9"),
Map.entry("0", "10"),
Map.entry("0", "11"),
Map.entry("1", "12"),
Map.entry("3", "13"),
Map.entry("3", "14")
);
var indexer = mock(Indexer.class);
pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry);
pileForIndexer(indexer, "spec.creationTime", creationTimeEntry);
pileForIndexer(indexer, "spec.top", topEntry);
pileForIndexer(indexer, "spec.priority", priorityEntry);
return new QueryIndexViewImpl(indexer);
}
public static void pileForIndexer(Indexer indexer, String propertyName,
Collection<Map.Entry<String, String>> entries) {
var indexEntry = mock(IndexEntry.class);
lenient().when(indexer.getIndexEntry(propertyName)).thenReturn(indexEntry);
var sortedEntries = sortEntries(entries);
lenient().when(indexEntry.entries()).thenReturn(sortedEntries);
lenient().when(indexEntry.getIdPositionMap()).thenReturn(idPositionMap(sortedEntries));
var indexedMap = toKeyObjectMap(sortedEntries);
lenient().when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet()));
lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> {
var key = (String) invocation.getArgument(0);
return indexedMap.get(key);
});
lenient().when(indexEntry.entries()).thenReturn(entries);
}
public static List<Map.Entry<String, String>> sortEntries(
Collection<Map.Entry<String, String>> entries) {
return entries.stream()
.sorted((a, b) -> KeyComparator.INSTANCE.compare(a.getKey(), b.getKey()))
.toList();
}
public static Map<String, Integer> idPositionMap(
Collection<Map.Entry<String, String>> sortedEntries) {
var asMap = toKeyObjectMap(sortedEntries);
int i = 0;
var idPositionMap = new HashMap<String, Integer>();
for (var valueIdsEntry : asMap.entrySet()) {
var ids = valueIdsEntry.getValue();
for (String id : ids) {
idPositionMap.put(id, i);
}
i++;
}
return idPositionMap;
}
public static LinkedHashMap<String, List<String>> toKeyObjectMap(
Collection<Map.Entry<String, String>> sortedEntries) {
return sortedEntries.stream()
.collect(Collectors.groupingBy(Map.Entry::getKey,
LinkedHashMap::new,
Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
}
}

79
api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java → application/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java
View File

@ -1,6 +1,7 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static run.halo.app.extension.index.query.IndexViewDataSet.createEmployeeIndexView;
import static run.halo.app.extension.index.query.QueryFactory.all;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.between;
@ -35,9 +36,11 @@ import org.junit.jupiter.api.Test;
*/
class QueryFactoryTest {
private final String id = QueryIndexViewImpl.PRIMARY_INDEX_NAME;
@Test
void allTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = all("firstName").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103", "104", "105"
@ -64,7 +67,7 @@ class QueryFactoryTest {
@Test
void equalTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = equal("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "104", "105"
@ -73,8 +76,8 @@ class QueryFactoryTest {
@Test
void equalOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = equalOtherField("managerId", "id").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = equalOtherField("managerId", id).matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
@ -82,7 +85,7 @@ class QueryFactoryTest {
@Test
void notEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = notEqual("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"101", "102", "103"
@ -91,8 +94,8 @@ class QueryFactoryTest {
@Test
void notEqualOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = notEqualOtherField("managerId", "id").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = notEqualOtherField("managerId", id).matches(indexView);
// 103 102 is equal
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "104", "105"
@ -101,8 +104,8 @@ class QueryFactoryTest {
@Test
void lessThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = lessThan("id", "103").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = lessThan(id, "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102"
);
@ -110,8 +113,8 @@ class QueryFactoryTest {
@Test
void lessThanOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = lessThanOtherField("id", "managerId").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = lessThanOtherField(id, "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
@ -119,8 +122,8 @@ class QueryFactoryTest {
@Test
void lessThanOrEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = lessThanOrEqual("id", "103").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = lessThanOrEqual(id, "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
@ -128,9 +131,9 @@ class QueryFactoryTest {
@Test
void lessThanOrEqualOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet =
lessThanOrEqualOtherField("id", "managerId").matches(indexView);
lessThanOrEqualOtherField(id, "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
@ -138,8 +141,8 @@ class QueryFactoryTest {
@Test
void greaterThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = greaterThan("id", "103").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = greaterThan(id, "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
@ -147,8 +150,8 @@ class QueryFactoryTest {
@Test
void greaterThanOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = greaterThanOtherField("id", "managerId").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = greaterThanOtherField(id, "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
@ -156,8 +159,8 @@ class QueryFactoryTest {
@Test
void greaterThanOrEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = greaterThanOrEqual("id", "103").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = greaterThanOrEqual(id, "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104", "105"
);
@ -165,9 +168,9 @@ class QueryFactoryTest {
@Test
void greaterThanOrEqualOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet =
greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
greaterThanOrEqualOtherField(id, "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103", "104", "105"
);
@ -175,8 +178,8 @@ class QueryFactoryTest {
@Test
void inTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = in("id", "103", "104").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = in(id, "103", "104").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104"
);
@ -184,7 +187,7 @@ class QueryFactoryTest {
@Test
void inTest2() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = in("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "104", "105"
@ -193,13 +196,13 @@ class QueryFactoryTest {
@Test
void betweenTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = between("id", "103", "105").matches(indexView);
var indexView = createEmployeeIndexView();
var resultSet = between(id, "103", "105").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104", "105"
);
indexView = IndexViewDataSet.createEmployeeIndexView();
indexView = createEmployeeIndexView();
resultSet = between("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"101", "102", "103"
@ -208,7 +211,7 @@ class QueryFactoryTest {
@Test
void betweenLowerExclusive() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet =
QueryFactory.betweenLowerExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
@ -218,7 +221,7 @@ class QueryFactoryTest {
@Test
void betweenUpperExclusive() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet =
QueryFactory.betweenUpperExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
@ -228,7 +231,7 @@ class QueryFactoryTest {
@Test
void betweenExclusive() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = QueryFactory.betweenExclusive("salary", "2000", "2400").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
@ -237,7 +240,7 @@ class QueryFactoryTest {
@Test
void startsWithTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = startsWith("firstName", "W").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
@ -246,7 +249,7 @@ class QueryFactoryTest {
@Test
void endsWithTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = endsWith("firstName", "y").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103"
@ -255,7 +258,7 @@ class QueryFactoryTest {
@Test
void containsTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet = contains("firstName", "i").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
@ -268,7 +271,7 @@ class QueryFactoryTest {
@Test
void notTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var indexView = createEmployeeIndexView();
var resultSet =
QueryFactory.not(QueryFactory.contains("firstName", "i")).matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
@ -286,10 +289,10 @@ class QueryFactoryTest {
// and composite query
query = and(
and(equal("firstName", "W"), equal("lastName", "Fay")),
or(equalOtherField("id", "userId"), lessThan("age", "123"))
or(equalOtherField(id, "userId"), lessThan("age", "123"))
);
fieldNames = getFieldNamesUsedInQuery(query);
assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId",
assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", id, "userId",
"age");
// or composite query

301
application/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java Normal file
View File

@ -0,0 +1,301 @@
package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static run.halo.app.extension.index.query.IndexViewDataSet.createCommentIndexView;
import static run.halo.app.extension.index.query.IndexViewDataSet.createEmployeeIndexView;
import static run.halo.app.extension.index.query.IndexViewDataSet.createPostIndexViewWithNullCell;
import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer;
import static run.halo.app.extension.index.query.QueryIndexViewImpl.PRIMARY_INDEX_NAME;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.domain.Sort;
import run.halo.app.extension.index.IndexEntry;
import run.halo.app.extension.index.Indexer;
/**
* Tests for {@link QueryIndexViewImpl}.
*
* @author guqing
* @since 2.17.0
*/
class QueryIndexViewImplTest {
final String id = PRIMARY_INDEX_NAME;
@Test
void getAllIdsForFieldTest() {
var indexView = createPostIndexViewWithNullCell();
var resultSet = indexView.getIdsForField("title");
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103", "104", "105", "106", "107", "108"
);
resultSet = indexView.getIdsForField("publishTime");
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "105", "106", "107"
);
}
@Test
void findIdsForValueEqualTest() {
var indexView = createEmployeeIndexView();
var resultSet = indexView.findMatchingIdsWithEqualValues("managerId", id);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103"
);
}
@Test
void findIdsForFieldValueGreaterThanTest() {
var indexView = createEmployeeIndexView();
var resultSet = indexView.findMatchingIdsWithGreaterValues(id, "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = createEmployeeIndexView();
resultSet = indexView.findMatchingIdsWithGreaterValues(id, "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
@Test
void findIdsForFieldValueGreaterThanTest2() {
var indexView = createEmployeeIndexView();
var resultSet = indexView.findMatchingIdsWithGreaterValues("managerId", id, false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = createEmployeeIndexView();
resultSet = indexView.findMatchingIdsWithGreaterValues("managerId", id, true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest() {
var indexView = createEmployeeIndexView();
var resultSet = indexView.findMatchingIdsWithSmallerValues(id, "managerId", false);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
indexView = createEmployeeIndexView();
resultSet = indexView.findMatchingIdsWithSmallerValues(id, "managerId", true);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
}
@Test
void findIdsForFieldValueLessThanTest2() {
var indexView = createEmployeeIndexView();
var resultSet = indexView.findMatchingIdsWithSmallerValues("managerId", id, false);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
indexView = createEmployeeIndexView();
resultSet = indexView.findMatchingIdsWithSmallerValues("managerId", id, true);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "102", "104", "105"
);
}
@Nested
@ExtendWith(MockitoExtension.class)
class SortTest {
@Mock
private Indexer indexer;
@Test
void testSortByUnsorted() {
var idEntry = mock(IndexEntry.class);
when(indexer.getIndexEntry(PRIMARY_INDEX_NAME))
.thenReturn(idEntry);
var indexView = new QueryIndexViewImpl(indexer);
var sort = Sort.unsorted();
var resultSet = new TreeSet<>(List.of("Item1", "Item2"));
List<String> sortedList = indexView.sortBy(resultSet, sort);
assertThat(sortedList).isEqualTo(List.of("Item1", "Item2"));
}
@Test
void testSortBySortedAscending() {
pileForIndexer(indexer, "field1",
List.of(Map.entry("key2", "Item2"), Map.entry("key1", "Item1")));
pileForIndexer(indexer, PRIMARY_INDEX_NAME,
List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2")));
var indexView = new QueryIndexViewImpl(indexer);
var sort = Sort.by(Sort.Order.asc("field1"));
List<String> sortedList = indexView.sortBy(indexView.getAllIds(), sort);
assertThat(sortedList).containsSequence("Item1", "Item2");
}
@Test
void testSortBySortedDescending() {
pileForIndexer(indexer, "field1",
List.of(Map.entry("key1", "Item1"), Map.entry("key2", "Item2")));
pileForIndexer(indexer, PRIMARY_INDEX_NAME,
List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2")));
var indexView = new QueryIndexViewImpl(indexer);
var sort = Sort.by(Sort.Order.desc("field1"));
var resultSet = new TreeSet<>(List.of("Item1", "Item2"));
List<String> sortedList = indexView.sortBy(resultSet, sort);
assertThat(sortedList).containsExactly("Item2", "Item1");
}
@Test
void testSortByMultipleFields() {
pileForIndexer(indexer, "field1",
List.of(Map.entry("k3", "Item3"), Map.entry("k2", "Item2")));
pileForIndexer(indexer, "field2",
List.of(Map.entry("k1", "Item1"), Map.entry("k3", "Item3")));
pileForIndexer(indexer, id,
List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2"),
Map.entry("Item3", "Item3")));
var indexView = new QueryIndexViewImpl(indexer);
var sort = Sort.by(Sort.Order.asc("field1"), Sort.Order.desc("field2"));
var resultSet = new TreeSet<>(List.of("Item1", "Item2", "Item3"));
List<String> sortedList = indexView.sortBy(resultSet, sort);
assertThat(sortedList).containsExactly("Item2", "Item3", "Item1");
}
@Test
void testSortByMultipleFields2() {
pileForIndexer(indexer, id, List.of());
pileForIndexer(indexer, "field1", List.of(Map.entry("John", "John"),
Map.entry("Bob", "Bob"),
Map.entry("Alice", "Alice")
));
pileForIndexer(indexer, "field2", List.of(Map.entry("David", "David"),
Map.entry("Eva", "Eva"),
Map.entry("Frank", "Frank")
));
pileForIndexer(indexer, "field3", List.of(Map.entry("George", "George"),
Map.entry("Helen", "Helen"),
Map.entry("Ivy", "Ivy")
));
/*
* <pre>
* Row Key | field1 | field2 | field3
* -------|-------|-------|-------
* John | John | |
* Bob | Bob | |
* Alice | Alice | |
* David | | David |
* Eva | | Eva |
* Frank | | Frank |
* George | | | George
* Helen | | | Helen
* Ivy | | | Ivy
* </pre>
*/
var indexView = new QueryIndexViewImpl(indexer);
var sort = Sort.by(Sort.Order.desc("field1"), Sort.Order.asc("field2"),
Sort.Order.asc("field3"));
var resultSet = new TreeSet<>(
List.of("Bob", "John", "Eva", "Alice", "Ivy", "David", "Frank", "Helen", "George"));
List<String> sortedList = indexView.sortBy(resultSet, sort);
assertThat(sortedList).containsSequence("David", "Eva", "Frank", "George", "Helen",
"Ivy", "John", "Bob", "Alice");
}
/**
* <p>Result for the following data.</p>
* <pre>
* | id | firstName | lastName | email | hireDate | salary | managerId | departmentId |
* |----|-----------|----------|-------|----------|--------|-----------|--------------|
* | 100| Pat | Fay | p | 17 | 2600 | 101 | 50 |
* | 101| Lee | Day | l | 17 | 2400 | 102 | 40 |
* | 103| Mary | Day | p | 17 | 2000 | 103 | 50 |
* | 104| John | Fay | j | 17 | 1800 | 103 | 50 |
* | 105| Gon | Fay | p | 18 | 1900 | 101 | 40 |
* | 102| William | Jay | w | 19 | 2200 | 102 | 50 |
* </pre>
*/
@Test
void sortByMultipleFieldsWithFirstSame() {
var indexView = createEmployeeIndexView();
var ids = indexView.getAllIds();
var result = indexView.sortBy(ids, Sort.by(Sort.Order.asc("hireDate"),
Sort.Order.asc("lastName"))
);
assertThat(result).containsSequence("101", "103", "100", "104", "105", "102");
}
/**
* <p>Result for the following data.</p>
* <pre>
* | id | title | published | publishTime | owner |
* |-----|--------|-----------|---------------------|-------|
* | 100 | title1 | true | 2024-01-01T00:00:00 | jack |
* | 101 | title2 | true | 2024-01-02T00:00:00 | rose |
* | 105 | title6 | true | 2024-01-05 00:00:00 | tom |
* | 107 | title8 | true | 2024-01-05 12:00:00 | jerry |
* | 106 | title7 | true | 2024-01-05 13:00:00 | jerry |
* | 108 | title9 | false | null | jerry |
* | 104 | title5 | false | null | john |
* | 103 | title4 | false | null | peter |
* | 102 | title3 | false | null | smith |
* </pre>
*/
@Test
void sortByMultipleFieldsForPostDataSet() {
var indexView = createPostIndexViewWithNullCell();
var ids = indexView.getAllIds();
var result = indexView.sortBy(ids, Sort.by(Sort.Order.asc("publishTime"),
Sort.Order.desc("title"))
);
assertThat(result).containsSequence("100", "101", "105", "107", "106", "108", "104",
"103", "102");
}
@Test
void sortByMultipleFieldsForCommentDataSet() {
var indexView = createCommentIndexView();
var ids = indexView.getAllIds();
var sort = Sort.by(Sort.Order.desc("spec.top"),
Sort.Order.asc("spec.priority"),
Sort.Order.desc("spec.creationTime"),
Sort.Order.asc("metadata.name")
);
var result = indexView.sortBy(ids, sort);
assertThat(result).containsSequence("1", "2", "4", "3", "5", "6", "9", "10", "8", "7",
"11", "12", "14", "13");
}
}
}