mirror of https://github.com/halo-dev/halo
feat: optimized post reconciliation process for enhanced performance and resource utilization (#5250)
#### What type of PR is this? /kind feature /milestone 2.12.x /area core #### What this PR does / why we need it: 我们为文章自定义模型的数据调协过程引入了重要的优化。 在以前,当数据量大(例如,50,000篇文章)的情况下,每次系统重启都会触发耗时且资源密集的所有数据的协调过程,即使大部分数据并不需要调协。这导致了不必要的数据库查询和高资源消耗。 为了解决这个问题,我们在文章自定义模型的 status 中添加了一个新的 `Long observedVersion` 属性。 每次协调后,此属性将更新为 `metadata.version`,还调整了 `syncAllOnStart` 条件,只有当 `status.observedVersion < metadata.version` 时才会调协数据。 这个改变确保了只有在启动时需要的数据会被协调,从而减少了资源使用和不必要的协调过程。 因此,Halo 的数据承载能力得到了显著提高。 **how to test it?** 使用此 PR 测试:启动时文章只有首次会执行 reconcile,再次重启时则不会再执行,如果直接修改数据去除掉 `status.observedVersion` 来模拟迁移或漏 reconcile 的过程则启动时该数据会被再次执行 reconcile #### Which issue(s) this PR fixes: Fixes #5147 #### Does this PR introduce a user-facing change? ```release-note 优化文章数据的调协过程以降低 Halo 启动时文章的调协耗时同时提高性能和资源利用率 ```pull/5256/head
parent
22104fe636
commit
17a0fb9e05
|
@ -16,7 +16,6 @@ import run.halo.app.extension.AbstractExtension;
|
|||
import run.halo.app.extension.GVK;
|
||||
import run.halo.app.extension.GroupVersionKind;
|
||||
import run.halo.app.extension.MetadataOperator;
|
||||
import run.halo.app.extension.MetadataUtil;
|
||||
import run.halo.app.infra.ConditionList;
|
||||
|
||||
/**
|
||||
|
@ -35,6 +34,8 @@ public class Post extends AbstractExtension {
|
|||
|
||||
public static final String KIND = "Post";
|
||||
|
||||
public static final String REQUIRE_SYNC_ON_STARTUP_INDEX_NAME = "requireSyncOnStartup";
|
||||
|
||||
public static final GroupVersionKind GVK = GroupVersionKind.fromExtension(Post.class);
|
||||
|
||||
public static final String CATEGORIES_ANNO = "content.halo.run/categories";
|
||||
|
@ -158,6 +159,8 @@ public class Post extends AbstractExtension {
|
|||
|
||||
private Instant lastModifyTime;
|
||||
|
||||
private Long observedVersion;
|
||||
|
||||
@JsonIgnore
|
||||
public ConditionList getConditionsOrDefault() {
|
||||
if (this.conditions == null) {
|
||||
|
@ -268,9 +271,4 @@ public class Post extends AbstractExtension {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void changePublishedState(Post post, boolean value) {
|
||||
Map<String, String> labels = MetadataUtil.nullSafeLabels(post);
|
||||
labels.put(PUBLISHED_LABEL, String.valueOf(value));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
|||
import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
|
||||
import static run.halo.app.extension.ExtensionUtil.addFinalizers;
|
||||
import static run.halo.app.extension.ExtensionUtil.removeFinalizers;
|
||||
import static run.halo.app.extension.index.query.QueryFactory.equal;
|
||||
|
||||
import com.google.common.hash.Hashing;
|
||||
import java.time.Instant;
|
||||
|
@ -36,6 +37,7 @@ import run.halo.app.event.post.PostPublishedEvent;
|
|||
import run.halo.app.event.post.PostUnpublishedEvent;
|
||||
import run.halo.app.event.post.PostUpdatedEvent;
|
||||
import run.halo.app.event.post.PostVisibleChangedEvent;
|
||||
import run.halo.app.extension.DefaultExtensionMatcher;
|
||||
import run.halo.app.extension.ExtensionClient;
|
||||
import run.halo.app.extension.ExtensionOperator;
|
||||
import run.halo.app.extension.ListOptions;
|
||||
|
@ -211,7 +213,11 @@ public class PostReconciler implements Reconciler<Reconciler.Request> {
|
|||
status.setInProgress(
|
||||
!StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot()));
|
||||
|
||||
// version + 1 is required to truly equal version
|
||||
// as a version will be incremented after the update
|
||||
status.setObservedVersion(post.getMetadata().getVersion() + 1);
|
||||
client.update(post);
|
||||
|
||||
// fire event after updating post
|
||||
events.forEach(eventPublisher::publishEvent);
|
||||
});
|
||||
|
@ -222,8 +228,12 @@ public class PostReconciler implements Reconciler<Reconciler.Request> {
|
|||
public Controller setupWith(ControllerBuilder builder) {
|
||||
return builder
|
||||
.extension(new Post())
|
||||
// TODO Make it configurable
|
||||
.workerCount(1)
|
||||
.onAddMatcher(DefaultExtensionMatcher.builder(client, Post.GVK)
|
||||
.fieldSelector(FieldSelector.of(
|
||||
equal(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME, BooleanUtils.TRUE))
|
||||
)
|
||||
.build()
|
||||
)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ import static run.halo.app.extension.index.IndexAttributeFactory.multiValueAttri
|
|||
import static run.halo.app.extension.index.IndexAttributeFactory.simpleAttribute;
|
||||
|
||||
import java.util.Set;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.springframework.boot.context.event.ApplicationContextInitializedEvent;
|
||||
import org.springframework.context.ApplicationListener;
|
||||
import org.springframework.lang.NonNull;
|
||||
|
@ -146,6 +147,18 @@ public class SchemeInitializer implements ApplicationListener<ApplicationContext
|
|||
.setName("status.excerpt")
|
||||
.setIndexFunc(
|
||||
simpleAttribute(Post.class, post -> post.getStatusOrDefault().getExcerpt())));
|
||||
|
||||
indexSpecs.add(new IndexSpec()
|
||||
.setName(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME)
|
||||
.setIndexFunc(simpleAttribute(Post.class, post -> {
|
||||
var version = post.getMetadata().getVersion();
|
||||
var observedVersion = post.getStatusOrDefault().getObservedVersion();
|
||||
if (observedVersion == null || observedVersion < version) {
|
||||
return BooleanUtils.TRUE;
|
||||
}
|
||||
// do not care about the false case so return null to avoid indexing
|
||||
return null;
|
||||
})));
|
||||
});
|
||||
schemeManager.register(Category.class, indexSpecs -> {
|
||||
indexSpecs.add(new IndexSpec()
|
||||
|
|
|
@ -19,6 +19,7 @@ public class TestPost {
|
|||
post.setApiVersion(getApiVersion(Post.class));
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setName("post-A");
|
||||
metadata.setVersion(1L);
|
||||
post.setMetadata(metadata);
|
||||
|
||||
Post.PostSpec postSpec = new Post.PostSpec();
|
||||
|
@ -38,6 +39,7 @@ public class TestPost {
|
|||
snapshot.setApiVersion(getApiVersion(Snapshot.class));
|
||||
Metadata metadata = new Metadata();
|
||||
metadata.setName("snapshot-A");
|
||||
metadata.setVersion(1L);
|
||||
metadata.setCreationTimestamp(Instant.now());
|
||||
snapshot.setMetadata(metadata);
|
||||
MetadataUtil.nullSafeAnnotations(snapshot).put(Snapshot.KEEP_RAW_ANNO, "true");
|
||||
|
|
|
@ -358,7 +358,8 @@ class CommentServiceImplTest {
|
|||
"apiVersion": "content.halo.run/v1alpha1",
|
||||
"kind": "Post",
|
||||
"metadata": {
|
||||
"name": "fake-post"
|
||||
"name": "fake-post",
|
||||
"version": 1
|
||||
}
|
||||
},
|
||||
"stats": {
|
||||
|
@ -403,7 +404,8 @@ class CommentServiceImplTest {
|
|||
"apiVersion": "content.halo.run/v1alpha1",
|
||||
"kind": "Post",
|
||||
"metadata": {
|
||||
"name": "fake-post"
|
||||
"name": "fake-post",
|
||||
"version": 1
|
||||
}
|
||||
},
|
||||
"stats": {
|
||||
|
@ -447,7 +449,8 @@ class CommentServiceImplTest {
|
|||
"apiVersion": "content.halo.run/v1alpha1",
|
||||
"kind": "Post",
|
||||
"metadata": {
|
||||
"name": "fake-post"
|
||||
"name": "fake-post",
|
||||
"version": 1
|
||||
}
|
||||
},
|
||||
"stats": {
|
||||
|
|
Loading…
Reference in New Issue