From 7f4abbba0983ff24121c37ae0c3c2c08f44d2f43 Mon Sep 17 00:00:00 2001 From: guqing <38999863+guqing@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:21:12 +0800 Subject: [PATCH] refactor: only select the required index fields when build query index view (#5312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind improvement /area core /milestone 2.13.x #### What this PR does / why we need it: 构建查询视图时只选择被使用到的索引字段 how to test it? 验证文章和附件列表的查询条件和排序条件不会报错即可 #### Does this PR introduce a user-facing change? ```release-note 优化查询视图构建只选择被使用到的索引字段构建查询视图 ``` --- .../extension/index/query/LogicalQuery.java | 2 + .../extension/index/query/QueryFactory.java | 23 ++++++ .../extension/index/query/SimpleQuery.java | 2 + .../index/query/QueryFactoryTest.java | 78 +++++++++++++++---- .../index/IndexedQueryEngineImpl.java | 23 ++++++ 5 files changed, 112 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java index 4d81a1ef8..2311109b5 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/LogicalQuery.java @@ -2,7 +2,9 @@ package run.halo.app.extension.index.query; import java.util.Collection; import java.util.Objects; +import lombok.Getter; +@Getter public abstract class LogicalQuery implements Query { protected final Collection childQueries; protected final int size; diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java index 8ac83ef2f..741d96c50 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryFactory.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import lombok.experimental.UtilityClass; import org.springframework.util.Assert; @@ -207,4 +208,26 @@ public class QueryFactory { public static Query contains(String fieldName, String value) { return new StringContains(fieldName, value); } + + /** + * Get all the field names used in the given query. + * + * @param query the query + * @return the field names used in the given query + */ + public static List getFieldNamesUsedInQuery(Query query) { + List fieldNames = new ArrayList<>(); + + if (query instanceof SimpleQuery simpleQuery) { + if (simpleQuery.isFieldRef()) { + fieldNames.add(simpleQuery.getValue()); + } + fieldNames.add(simpleQuery.getFieldName()); + } else if (query instanceof LogicalQuery logicalQuery) { + for (Query childQuery : logicalQuery.getChildQueries()) { + fieldNames.addAll(getFieldNamesUsedInQuery(childQuery)); + } + } + return fieldNames; + } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java index da41561fe..931066161 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/SimpleQuery.java @@ -1,8 +1,10 @@ package run.halo.app.extension.index.query; +import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.springframework.util.Assert; +@Getter public abstract class SimpleQuery implements Query { protected final String fieldName; protected final String value; diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java index acce9acd1..e7b8918d7 100644 --- a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java +++ b/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java @@ -2,11 +2,28 @@ package run.halo.app.extension.index.query; import static org.assertj.core.api.Assertions.assertThat; import static run.halo.app.extension.index.query.QueryFactory.all; +import static run.halo.app.extension.index.query.QueryFactory.and; import static run.halo.app.extension.index.query.QueryFactory.between; +import static run.halo.app.extension.index.query.QueryFactory.contains; +import static run.halo.app.extension.index.query.QueryFactory.endsWith; import static run.halo.app.extension.index.query.QueryFactory.equal; import static run.halo.app.extension.index.query.QueryFactory.equalOtherField; +import static run.halo.app.extension.index.query.QueryFactory.getFieldNamesUsedInQuery; +import static run.halo.app.extension.index.query.QueryFactory.greaterThan; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqual; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOrEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.greaterThanOtherField; +import static run.halo.app.extension.index.query.QueryFactory.in; +import static run.halo.app.extension.index.query.QueryFactory.isNotNull; +import static run.halo.app.extension.index.query.QueryFactory.isNull; +import static run.halo.app.extension.index.query.QueryFactory.lessThan; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqual; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOrEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.lessThanOtherField; import static run.halo.app.extension.index.query.QueryFactory.notEqual; import static run.halo.app.extension.index.query.QueryFactory.notEqualOtherField; +import static run.halo.app.extension.index.query.QueryFactory.or; +import static run.halo.app.extension.index.query.QueryFactory.startsWith; import org.junit.jupiter.api.Test; @@ -30,7 +47,7 @@ class QueryFactoryTest { @Test void isNullTest() { var indexView = IndexViewDataSet.createPostIndexViewWithNullCell(); - var resultSet = QueryFactory.isNull("publishTime").matches(indexView); + var resultSet = isNull("publishTime").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103", "104", "108" ); @@ -39,7 +56,7 @@ class QueryFactoryTest { @Test void isNotNullTest() { var indexView = IndexViewDataSet.createPostIndexViewWithNullCell(); - var resultSet = QueryFactory.isNotNull("publishTime").matches(indexView); + var resultSet = isNotNull("publishTime").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "105", "106", "107" ); @@ -85,7 +102,7 @@ class QueryFactoryTest { @Test void lessThanTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThan("id", "103").matches(indexView); + var resultSet = lessThan("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102" ); @@ -94,7 +111,7 @@ class QueryFactoryTest { @Test void lessThanOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThanOtherField("id", "managerId").matches(indexView); + var resultSet = lessThanOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101" ); @@ -103,7 +120,7 @@ class QueryFactoryTest { @Test void lessThanOrEqualTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.lessThanOrEqual("id", "103").matches(indexView); + var resultSet = lessThanOrEqual("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -113,7 +130,7 @@ class QueryFactoryTest { void lessThanOrEqualOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); var resultSet = - QueryFactory.lessThanOrEqualOtherField("id", "managerId").matches(indexView); + lessThanOrEqualOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -122,7 +139,7 @@ class QueryFactoryTest { @Test void greaterThanTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThan("id", "103").matches(indexView); + var resultSet = greaterThan("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -131,7 +148,7 @@ class QueryFactoryTest { @Test void greaterThanOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThanOtherField("id", "managerId").matches(indexView); + var resultSet = greaterThanOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -140,7 +157,7 @@ class QueryFactoryTest { @Test void greaterThanOrEqualTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.greaterThanOrEqual("id", "103").matches(indexView); + var resultSet = greaterThanOrEqual("id", "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104", "105" ); @@ -150,7 +167,7 @@ class QueryFactoryTest { void greaterThanOrEqualOtherFieldTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); var resultSet = - QueryFactory.greaterThanOrEqualOtherField("id", "managerId").matches(indexView); + greaterThanOrEqualOtherField("id", "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103", "104", "105" ); @@ -159,7 +176,7 @@ class QueryFactoryTest { @Test void inTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.in("id", "103", "104").matches(indexView); + var resultSet = in("id", "103", "104").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104" ); @@ -168,7 +185,7 @@ class QueryFactoryTest { @Test void inTest2() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.in("lastName", "Fay").matches(indexView); + var resultSet = in("lastName", "Fay").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "104", "105" ); @@ -221,7 +238,7 @@ class QueryFactoryTest { @Test void startsWithTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.startsWith("firstName", "W").matches(indexView); + var resultSet = startsWith("firstName", "W").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" ); @@ -230,7 +247,7 @@ class QueryFactoryTest { @Test void endsWithTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.endsWith("firstName", "y").matches(indexView); + var resultSet = endsWith("firstName", "y").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103" ); @@ -239,11 +256,11 @@ class QueryFactoryTest { @Test void containsTest() { var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = QueryFactory.contains("firstName", "i").matches(indexView); + var resultSet = contains("firstName", "i").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" ); - resultSet = QueryFactory.contains("firstName", "N").matches(indexView); + resultSet = contains("firstName", "N").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -258,4 +275,33 @@ class QueryFactoryTest { "100", "101", "103", "104", "105" ); } + + @Test + void getUsedFieldNamesTest() { + // single query + var query = equal("firstName", "W"); + var fieldNames = getFieldNamesUsedInQuery(query); + assertThat(fieldNames).containsExactlyInAnyOrder("firstName"); + + // and composite query + query = and( + and(equal("firstName", "W"), equal("lastName", "Fay")), + or(equalOtherField("id", "userId"), lessThan("age", "123")) + ); + fieldNames = getFieldNamesUsedInQuery(query); + assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId", + "age"); + + // or composite query + var complexQuery = or( + equal("field1", "value1"), + and( + equal("field2", "value2"), + equal("field3", "value3") + ), + equal("field4", "value4") + ); + fieldNames = getFieldNamesUsedInQuery(complexQuery); + assertThat(fieldNames).containsExactlyInAnyOrder("field1", "field2", "field3", "field4"); + } } diff --git a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java index b3723fcb8..1e30d05d8 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java @@ -4,12 +4,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Sort; @@ -20,6 +22,7 @@ import run.halo.app.extension.GroupVersionKind; import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; import run.halo.app.extension.PageRequest; +import run.halo.app.extension.index.query.QueryFactory; import run.halo.app.extension.index.query.QueryIndexViewImpl; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.extension.router.selector.LabelSelector; @@ -145,8 +148,12 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine { stopWatch.stop(); stopWatch.start("build index view"); + var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort); var indexViewMap = new HashMap>>(); for (Map.Entry entry : fieldPathEntryMap.entrySet()) { + if (!fieldNamesUsedInQuery.contains(entry.getKey())) { + continue; + } indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries()); } // TODO optimize build indexView time @@ -183,6 +190,22 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine { return result; } + @NonNull + private Set getFieldNamesUsedInListOptions(ListOptions options, Sort sort) { + var fieldNamesUsedInQuery = new HashSet(); + fieldNamesUsedInQuery.add(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME); + for (Sort.Order order : sort) { + fieldNamesUsedInQuery.add(order.getProperty()); + } + var hasFieldSelector = hasFieldSelector(options.getFieldSelector()); + if (hasFieldSelector) { + var fieldQuery = options.getFieldSelector().query(); + var fieldNames = QueryFactory.getFieldNamesUsedInQuery(fieldQuery); + fieldNamesUsedInQuery.addAll(fieldNames); + } + return fieldNamesUsedInQuery; + } + boolean hasLabelSelector(LabelSelector labelSelector) { return labelSelector != null && !CollectionUtils.isEmpty(labelSelector.getMatchers()); }