github
/
halo
mirror of https://github.com/halo-dev/halo
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: only select the required index fields when build query index view (#5312) 

#### What type of PR is this?
/kind improvement
/area core
/milestone 2.13.x

#### What this PR does / why we need it:
构建查询视图时只选择被使用到的索引字段
how to test it?
验证文章和附件列表的查询条件和排序条件不会报错即可

#### Does this PR introduce a user-facing change?
```release-note
优化查询视图构建只选择被使用到的索引字段构建查询视图
```
pull/5413/head^2
guqing 2024-02-27 17:21:12 +08:00 committed by GitHub
parent dc350ae62c
commit 7f4abbba09
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 112 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java
View File

@ -2,7 +2,9 @@ package run.halo.app.extension.index.query;
import java.util.Collection;
import java.util.Objects;
import lombok.Getter;
@Getter
public abstract class LogicalQuery implements Query {
protected final Collection<Query> childQueries;
protected final int size;

23
api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java
View File

@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lombok.experimental.UtilityClass;
import org.springframework.util.Assert;
@ -207,4 +208,26 @@ public class QueryFactory {
public static Query contains(String fieldName, String value) {
return new StringContains(fieldName, value);
}
/**
* Get all the field names used in the given query.
*
* @param query the query
* @return the field names used in the given query
*/
public static List<String> getFieldNamesUsedInQuery(Query query) {
List<String> fieldNames = new ArrayList<>();
if (query instanceof SimpleQuery simpleQuery) {
if (simpleQuery.isFieldRef()) {
fieldNames.add(simpleQuery.getValue());
}
fieldNames.add(simpleQuery.getFieldName());
} else if (query instanceof LogicalQuery logicalQuery) {
for (Query childQuery : logicalQuery.getChildQueries()) {
fieldNames.addAll(getFieldNamesUsedInQuery(childQuery));
}
}
return fieldNames;
}
}

2
api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java
View File

@ -1,8 +1,10 @@
package run.halo.app.extension.index.query;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.Assert;
@Getter
public abstract class SimpleQuery implements Query {
protected final String fieldName;
protected final String value;

78
api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java
View File

@ -2,11 +2,28 @@ package run.halo.app.extension.index.query;
import static org.assertj.core.api.Assertions.assertThat;
import static run.halo.app.extension.index.query.QueryFactory.all;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.between;
import static run.halo.app.extension.index.query.QueryFactory.contains;
import static run.halo.app.extension.index.query.QueryFactory.endsWith;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.equalOtherField;
import static run.halo.app.extension.index.query.QueryFactory.getFieldNamesUsedInQuery;
import static run.halo.app.extension.index.query.QueryFactory.greaterThan;
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqual;
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqualOtherField;
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOtherField;
import static run.halo.app.extension.index.query.QueryFactory.in;
import static run.halo.app.extension.index.query.QueryFactory.isNotNull;
import static run.halo.app.extension.index.query.QueryFactory.isNull;
import static run.halo.app.extension.index.query.QueryFactory.lessThan;
import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqual;
import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqualOtherField;
import static run.halo.app.extension.index.query.QueryFactory.lessThanOtherField;
import static run.halo.app.extension.index.query.QueryFactory.notEqual;
import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField;
import static run.halo.app.extension.index.query.QueryFactory.or;
import static run.halo.app.extension.index.query.QueryFactory.startsWith;
import org.junit.jupiter.api.Test;
@ -30,7 +47,7 @@ class QueryFactoryTest {
@Test
void isNullTest() {
var indexView = IndexViewDataSet.createPostIndexViewWithNullCell();
var resultSet = QueryFactory.isNull("publishTime").matches(indexView);
var resultSet = isNull("publishTime").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103", "104", "108"
);
@ -39,7 +56,7 @@ class QueryFactoryTest {
@Test
void isNotNullTest() {
var indexView = IndexViewDataSet.createPostIndexViewWithNullCell();
var resultSet = QueryFactory.isNotNull("publishTime").matches(indexView);
var resultSet = isNotNull("publishTime").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "105", "106", "107"
);
@ -85,7 +102,7 @@ class QueryFactoryTest {
@Test
void lessThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThan("id", "103").matches(indexView);
var resultSet = lessThan("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102"
);
@ -94,7 +111,7 @@ class QueryFactoryTest {
@Test
void lessThanOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView);
var resultSet = lessThanOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101"
);
@ -103,7 +120,7 @@ class QueryFactoryTest {
@Test
void lessThanOrEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView);
var resultSet = lessThanOrEqual("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
@ -113,7 +130,7 @@ class QueryFactoryTest {
void lessThanOrEqualOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView);
lessThanOrEqualOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "101", "102", "103"
);
@ -122,7 +139,7 @@ class QueryFactoryTest {
@Test
void greaterThanTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView);
var resultSet = greaterThan("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
@ -131,7 +148,7 @@ class QueryFactoryTest {
@Test
void greaterThanOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView);
var resultSet = greaterThanOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
@ -140,7 +157,7 @@ class QueryFactoryTest {
@Test
void greaterThanOrEqualTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView);
var resultSet = greaterThanOrEqual("id", "103").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104", "105"
);
@ -150,7 +167,7 @@ class QueryFactoryTest {
void greaterThanOrEqualOtherFieldTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet =
QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102", "103", "104", "105"
);
@ -159,7 +176,7 @@ class QueryFactoryTest {
@Test
void inTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.in("id", "103", "104").matches(indexView);
var resultSet = in("id", "103", "104").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103", "104"
);
@ -168,7 +185,7 @@ class QueryFactoryTest {
@Test
void inTest2() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView);
var resultSet = in("lastName", "Fay").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"100", "104", "105"
);
@ -221,7 +238,7 @@ class QueryFactoryTest {
@Test
void startsWithTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView);
var resultSet = startsWith("firstName", "W").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
);
@ -230,7 +247,7 @@ class QueryFactoryTest {
@Test
void endsWithTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView);
var resultSet = endsWith("firstName", "y").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"103"
);
@ -239,11 +256,11 @@ class QueryFactoryTest {
@Test
void containsTest() {
var indexView = IndexViewDataSet.createEmployeeIndexView();
var resultSet = QueryFactory.contains("firstName", "i").matches(indexView);
var resultSet = contains("firstName", "i").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"102"
);
resultSet = QueryFactory.contains("firstName", "N").matches(indexView);
resultSet = contains("firstName", "N").matches(indexView);
assertThat(resultSet).containsExactlyInAnyOrder(
"104", "105"
);
@ -258,4 +275,33 @@ class QueryFactoryTest {
"100", "101", "103", "104", "105"
);
}
@Test
void getUsedFieldNamesTest() {
// single query
var query = equal("firstName", "W");
var fieldNames = getFieldNamesUsedInQuery(query);
assertThat(fieldNames).containsExactlyInAnyOrder("firstName");
// and composite query
query = and(
and(equal("firstName", "W"), equal("lastName", "Fay")),
or(equalOtherField("id", "userId"), lessThan("age", "123"))
);
fieldNames = getFieldNamesUsedInQuery(query);
assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId",
"age");
// or composite query
var complexQuery = or(
equal("field1", "value1"),
and(
equal("field2", "value2"),
equal("field3", "value3")
),
equal("field4", "value4")
);
fieldNames = getFieldNamesUsedInQuery(complexQuery);
assertThat(fieldNames).containsExactlyInAnyOrder("field1", "field2", "field3", "field4");
}
}

23
application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java
View File

@ -4,12 +4,14 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.domain.Sort;
@ -20,6 +22,7 @@ import run.halo.app.extension.GroupVersionKind;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.index.query.QueryFactory;
import run.halo.app.extension.index.query.QueryIndexViewImpl;
import run.halo.app.extension.router.selector.FieldSelector;
import run.halo.app.extension.router.selector.LabelSelector;
@ -145,8 +148,12 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
stopWatch.stop();
stopWatch.start("build index view");
var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort);
var indexViewMap = new HashMap<String, Collection<Map.Entry<String, String>>>();
for (Map.Entry<String, IndexEntry> entry : fieldPathEntryMap.entrySet()) {
if (!fieldNamesUsedInQuery.contains(entry.getKey())) {
continue;
}
indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries());
}
// TODO optimize build indexView time
@ -183,6 +190,22 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
return result;
}
@NonNull
private Set<String> getFieldNamesUsedInListOptions(ListOptions options, Sort sort) {
var fieldNamesUsedInQuery = new HashSet<String>();
fieldNamesUsedInQuery.add(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME);
for (Sort.Order order : sort) {
fieldNamesUsedInQuery.add(order.getProperty());
}
var hasFieldSelector = hasFieldSelector(options.getFieldSelector());
if (hasFieldSelector) {
var fieldQuery = options.getFieldSelector().query();
var fieldNames = QueryFactory.getFieldNamesUsedInQuery(fieldQuery);
fieldNamesUsedInQuery.addAll(fieldNames);
}
return fieldNamesUsedInQuery;
}
boolean hasLabelSelector(LabelSelector labelSelector) {
return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers());
}