mirror of https://github.com/halo-dev/halo
refactor: only select the required index fields when build query index view (#5312)
#### What type of PR is this? /kind improvement /area core /milestone 2.13.x #### What this PR does / why we need it: 构建查询视图时只选择被使用到的索引字段 how to test it? 验证文章和附件列表的查询条件和排序条件不会报错即可 #### Does this PR introduce a user-facing change? ```release-note 优化查询视图构建只选择被使用到的索引字段构建查询视图 ```pull/5413/head^2
parent
dc350ae62c
commit
7f4abbba09
|
@ -2,7 +2,9 @@ package run.halo.app.extension.index.query;
|
|||
|
||||
import java.util.Collection;
|
||||
import java.util.Objects;
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public abstract class LogicalQuery implements Query {
|
||||
protected final Collection<Query> childQueries;
|
||||
protected final int size;
|
||||
|
|
|
@ -5,6 +5,7 @@ import java.util.Arrays;
|
|||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import lombok.experimental.UtilityClass;
|
||||
import org.springframework.util.Assert;
|
||||
|
@ -207,4 +208,26 @@ public class QueryFactory {
|
|||
public static Query contains(String fieldName, String value) {
|
||||
return new StringContains(fieldName, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all the field names used in the given query.
|
||||
*
|
||||
* @param query the query
|
||||
* @return the field names used in the given query
|
||||
*/
|
||||
public static List<String> getFieldNamesUsedInQuery(Query query) {
|
||||
List<String> fieldNames = new ArrayList<>();
|
||||
|
||||
if (query instanceof SimpleQuery simpleQuery) {
|
||||
if (simpleQuery.isFieldRef()) {
|
||||
fieldNames.add(simpleQuery.getValue());
|
||||
}
|
||||
fieldNames.add(simpleQuery.getFieldName());
|
||||
} else if (query instanceof LogicalQuery logicalQuery) {
|
||||
for (Query childQuery : logicalQuery.getChildQueries()) {
|
||||
fieldNames.addAll(getFieldNamesUsedInQuery(childQuery));
|
||||
}
|
||||
}
|
||||
return fieldNames;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
package run.halo.app.extension.index.query;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
@Getter
|
||||
public abstract class SimpleQuery implements Query {
|
||||
protected final String fieldName;
|
||||
protected final String value;
|
||||
|
|
|
@ -2,11 +2,28 @@ package run.halo.app.extension.index.query;
|
|||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.all;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.and;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.between;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.contains;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.endsWith;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.equal;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.equalOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.getFieldNamesUsedInQuery;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.greaterThan;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqual;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqualOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.greaterThanOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.in;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.isNotNull;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.isNull;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.lessThan;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqual;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqualOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.lessThanOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.notEqual;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.or;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.startsWith;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
@ -30,7 +47,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void isNullTest() {
|
||||
var indexView = IndexViewDataSet.createPostIndexViewWithNullCell();
|
||||
var resultSet = QueryFactory.isNull("publishTime").matches(indexView);
|
||||
var resultSet = isNull("publishTime").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"102", "103", "104", "108"
|
||||
);
|
||||
|
@ -39,7 +56,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void isNotNullTest() {
|
||||
var indexView = IndexViewDataSet.createPostIndexViewWithNullCell();
|
||||
var resultSet = QueryFactory.isNotNull("publishTime").matches(indexView);
|
||||
var resultSet = isNotNull("publishTime").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "101", "105", "106", "107"
|
||||
);
|
||||
|
@ -85,7 +102,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void lessThanTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.lessThan("id", "103").matches(indexView);
|
||||
var resultSet = lessThan("id", "103").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "101", "102"
|
||||
);
|
||||
|
@ -94,7 +111,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void lessThanOtherFieldTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView);
|
||||
var resultSet = lessThanOtherField("id", "managerId").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "101"
|
||||
);
|
||||
|
@ -103,7 +120,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void lessThanOrEqualTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView);
|
||||
var resultSet = lessThanOrEqual("id", "103").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "101", "102", "103"
|
||||
);
|
||||
|
@ -113,7 +130,7 @@ class QueryFactoryTest {
|
|||
void lessThanOrEqualOtherFieldTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet =
|
||||
QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView);
|
||||
lessThanOrEqualOtherField("id", "managerId").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "101", "102", "103"
|
||||
);
|
||||
|
@ -122,7 +139,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void greaterThanTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView);
|
||||
var resultSet = greaterThan("id", "103").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"104", "105"
|
||||
);
|
||||
|
@ -131,7 +148,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void greaterThanOtherFieldTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView);
|
||||
var resultSet = greaterThanOtherField("id", "managerId").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"104", "105"
|
||||
);
|
||||
|
@ -140,7 +157,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void greaterThanOrEqualTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView);
|
||||
var resultSet = greaterThanOrEqual("id", "103").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"103", "104", "105"
|
||||
);
|
||||
|
@ -150,7 +167,7 @@ class QueryFactoryTest {
|
|||
void greaterThanOrEqualOtherFieldTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet =
|
||||
QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
|
||||
greaterThanOrEqualOtherField("id", "managerId").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"102", "103", "104", "105"
|
||||
);
|
||||
|
@ -159,7 +176,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void inTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.in("id", "103", "104").matches(indexView);
|
||||
var resultSet = in("id", "103", "104").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"103", "104"
|
||||
);
|
||||
|
@ -168,7 +185,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void inTest2() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView);
|
||||
var resultSet = in("lastName", "Fay").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"100", "104", "105"
|
||||
);
|
||||
|
@ -221,7 +238,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void startsWithTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView);
|
||||
var resultSet = startsWith("firstName", "W").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"102"
|
||||
);
|
||||
|
@ -230,7 +247,7 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void endsWithTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView);
|
||||
var resultSet = endsWith("firstName", "y").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"103"
|
||||
);
|
||||
|
@ -239,11 +256,11 @@ class QueryFactoryTest {
|
|||
@Test
|
||||
void containsTest() {
|
||||
var indexView = IndexViewDataSet.createEmployeeIndexView();
|
||||
var resultSet = QueryFactory.contains("firstName", "i").matches(indexView);
|
||||
var resultSet = contains("firstName", "i").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"102"
|
||||
);
|
||||
resultSet = QueryFactory.contains("firstName", "N").matches(indexView);
|
||||
resultSet = contains("firstName", "N").matches(indexView);
|
||||
assertThat(resultSet).containsExactlyInAnyOrder(
|
||||
"104", "105"
|
||||
);
|
||||
|
@ -258,4 +275,33 @@ class QueryFactoryTest {
|
|||
"100", "101", "103", "104", "105"
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void getUsedFieldNamesTest() {
|
||||
// single query
|
||||
var query = equal("firstName", "W");
|
||||
var fieldNames = getFieldNamesUsedInQuery(query);
|
||||
assertThat(fieldNames).containsExactlyInAnyOrder("firstName");
|
||||
|
||||
// and composite query
|
||||
query = and(
|
||||
and(equal("firstName", "W"), equal("lastName", "Fay")),
|
||||
or(equalOtherField("id", "userId"), lessThan("age", "123"))
|
||||
);
|
||||
fieldNames = getFieldNamesUsedInQuery(query);
|
||||
assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId",
|
||||
"age");
|
||||
|
||||
// or composite query
|
||||
var complexQuery = or(
|
||||
equal("field1", "value1"),
|
||||
and(
|
||||
equal("field2", "value2"),
|
||||
equal("field3", "value3")
|
||||
),
|
||||
equal("field4", "value4")
|
||||
);
|
||||
fieldNames = getFieldNamesUsedInQuery(complexQuery);
|
||||
assertThat(fieldNames).containsExactlyInAnyOrder("field1", "field2", "field3", "field4");
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,12 +4,14 @@ import java.util.ArrayList;
|
|||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.data.domain.Sort;
|
||||
|
@ -20,6 +22,7 @@ import run.halo.app.extension.GroupVersionKind;
|
|||
import run.halo.app.extension.ListOptions;
|
||||
import run.halo.app.extension.ListResult;
|
||||
import run.halo.app.extension.PageRequest;
|
||||
import run.halo.app.extension.index.query.QueryFactory;
|
||||
import run.halo.app.extension.index.query.QueryIndexViewImpl;
|
||||
import run.halo.app.extension.router.selector.FieldSelector;
|
||||
import run.halo.app.extension.router.selector.LabelSelector;
|
||||
|
@ -145,8 +148,12 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
|
|||
stopWatch.stop();
|
||||
|
||||
stopWatch.start("build index view");
|
||||
var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort);
|
||||
var indexViewMap = new HashMap<String, Collection<Map.Entry<String, String>>>();
|
||||
for (Map.Entry<String, IndexEntry> entry : fieldPathEntryMap.entrySet()) {
|
||||
if (!fieldNamesUsedInQuery.contains(entry.getKey())) {
|
||||
continue;
|
||||
}
|
||||
indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries());
|
||||
}
|
||||
// TODO optimize build indexView time
|
||||
|
@ -183,6 +190,22 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine {
|
|||
return result;
|
||||
}
|
||||
|
||||
@NonNull
|
||||
private Set<String> getFieldNamesUsedInListOptions(ListOptions options, Sort sort) {
|
||||
var fieldNamesUsedInQuery = new HashSet<String>();
|
||||
fieldNamesUsedInQuery.add(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME);
|
||||
for (Sort.Order order : sort) {
|
||||
fieldNamesUsedInQuery.add(order.getProperty());
|
||||
}
|
||||
var hasFieldSelector = hasFieldSelector(options.getFieldSelector());
|
||||
if (hasFieldSelector) {
|
||||
var fieldQuery = options.getFieldSelector().query();
|
||||
var fieldNames = QueryFactory.getFieldNamesUsedInQuery(fieldQuery);
|
||||
fieldNamesUsedInQuery.addAll(fieldNames);
|
||||
}
|
||||
return fieldNamesUsedInQuery;
|
||||
}
|
||||
|
||||
boolean hasLabelSelector(LabelSelector labelSelector) {
|
||||
return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers());
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue