From 17a0fb9e05de5ee100871c06bd5c63a709380701 Mon Sep 17 00:00:00 2001 From: guqing <38999863+guqing@users.noreply.github.com> Date: Fri, 26 Jan 2024 17:02:10 +0800 Subject: [PATCH] feat: optimized post reconciliation process for enhanced performance and resource utilization (#5250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind feature /milestone 2.12.x /area core #### What this PR does / why we need it: 我们为文章自定义模型的数据调协过程引入了重要的优化。 在以前，当数据量大（例如，50,000篇文章）的情况下，每次系统重启都会触发耗时且资源密集的所有数据的协调过程，即使大部分数据并不需要调协。这导致了不必要的数据库查询和高资源消耗。 为了解决这个问题，我们在文章自定义模型的 status 中添加了一个新的 `Long observedVersion` 属性。 每次协调后，此属性将更新为 `metadata.version`，还调整了 `syncAllOnStart` 条件，只有当 `status.observedVersion < metadata.version` 时才会调协数据。 这个改变确保了只有在启动时需要的数据会被协调，从而减少了资源使用和不必要的协调过程。 因此，Halo 的数据承载能力得到了显著提高。 **how to test it?** 使用此 PR 测试：启动时文章只有首次会执行 reconcile，再次重启时则不会再执行，如果直接修改数据去除掉 `status.observedVersion` 来模拟迁移或漏 reconcile 的过程则启动时该数据会被再次执行 reconcile #### Which issue(s) this PR fixes: Fixes #5147 #### Does this PR introduce a user-facing change? ```release-note 优化文章数据的调协过程以降低 Halo 启动时文章的调协耗时同时提高性能和资源利用率 ``` --- .../run/halo/app/core/extension/content/Post.java | 10 ++++------ .../core/extension/reconciler/PostReconciler.java | 14 ++++++++++++-- .../java/run/halo/app/infra/SchemeInitializer.java | 13 +++++++++++++ .../test/java/run/halo/app/content/TestPost.java | 2 ++ .../content/comment/CommentServiceImplTest.java | 9 ++++++--- 5 files changed, 37 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/run/halo/app/core/extension/content/Post.java b/api/src/main/java/run/halo/app/core/extension/content/Post.java index ebe6b8860..6a616e933 100644 --- a/api/src/main/java/run/halo/app/core/extension/content/Post.java +++ b/api/src/main/java/run/halo/app/core/extension/content/Post.java @@ -16,7 +16,6 @@ import run.halo.app.extension.AbstractExtension; import run.halo.app.extension.GVK; import run.halo.app.extension.GroupVersionKind; import run.halo.app.extension.MetadataOperator; -import run.halo.app.extension.MetadataUtil; import run.halo.app.infra.ConditionList; /** @@ -35,6 +34,8 @@ public class Post extends AbstractExtension { public static final String KIND = "Post"; + public static final String REQUIRE_SYNC_ON_STARTUP_INDEX_NAME = "requireSyncOnStartup"; + public static final GroupVersionKind GVK = GroupVersionKind.fromExtension(Post.class); public static final String CATEGORIES_ANNO = "content.halo.run/categories"; @@ -158,6 +159,8 @@ public class Post extends AbstractExtension { private Instant lastModifyTime; + private Long observedVersion; + @JsonIgnore public ConditionList getConditionsOrDefault() { if (this.conditions == null) { @@ -268,9 +271,4 @@ public class Post extends AbstractExtension { } } } - - public static void changePublishedState(Post post, boolean value) { - Map labels = MetadataUtil.nullSafeLabels(post); - labels.put(PUBLISHED_LABEL, String.valueOf(value)); - } } diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java index 8c9a95095..aeb56dc57 100644 --- a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java +++ b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java @@ -4,6 +4,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; import static run.halo.app.extension.ExtensionUtil.addFinalizers; import static run.halo.app.extension.ExtensionUtil.removeFinalizers; +import static run.halo.app.extension.index.query.QueryFactory.equal; import com.google.common.hash.Hashing; import java.time.Instant; @@ -36,6 +37,7 @@ import run.halo.app.event.post.PostPublishedEvent; import run.halo.app.event.post.PostUnpublishedEvent; import run.halo.app.event.post.PostUpdatedEvent; import run.halo.app.event.post.PostVisibleChangedEvent; +import run.halo.app.extension.DefaultExtensionMatcher; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionOperator; import run.halo.app.extension.ListOptions; @@ -211,7 +213,11 @@ public class PostReconciler implements Reconciler { status.setInProgress( !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot())); + // version + 1 is required to truly equal version + // as a version will be incremented after the update + status.setObservedVersion(post.getMetadata().getVersion() + 1); client.update(post); + // fire event after updating post events.forEach(eventPublisher::publishEvent); }); @@ -222,8 +228,12 @@ public class PostReconciler implements Reconciler { public Controller setupWith(ControllerBuilder builder) { return builder .extension(new Post()) - // TODO Make it configurable - .workerCount(1) + .onAddMatcher(DefaultExtensionMatcher.builder(client, Post.GVK) + .fieldSelector(FieldSelector.of( + equal(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME, BooleanUtils.TRUE)) + ) + .build() + ) .build(); } diff --git a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java index 729d07f18..ad73b1288 100644 --- a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java +++ b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java @@ -5,6 +5,7 @@ import static run.halo.app.extension.index.IndexAttributeFactory.multiValueAttri import static run.halo.app.extension.index.IndexAttributeFactory.simpleAttribute; import java.util.Set; +import org.apache.commons.lang3.BooleanUtils; import org.springframework.boot.context.event.ApplicationContextInitializedEvent; import org.springframework.context.ApplicationListener; import org.springframework.lang.NonNull; @@ -146,6 +147,18 @@ public class SchemeInitializer implements ApplicationListener post.getStatusOrDefault().getExcerpt()))); + + indexSpecs.add(new IndexSpec() + .setName(Post.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME) + .setIndexFunc(simpleAttribute(Post.class, post -> { + var version = post.getMetadata().getVersion(); + var observedVersion = post.getStatusOrDefault().getObservedVersion(); + if (observedVersion == null || observedVersion < version) { + return BooleanUtils.TRUE; + } + // do not care about the false case so return null to avoid indexing + return null; + }))); }); schemeManager.register(Category.class, indexSpecs -> { indexSpecs.add(new IndexSpec() diff --git a/application/src/test/java/run/halo/app/content/TestPost.java b/application/src/test/java/run/halo/app/content/TestPost.java index 806c7605f..0dfeadc5b 100644 --- a/application/src/test/java/run/halo/app/content/TestPost.java +++ b/application/src/test/java/run/halo/app/content/TestPost.java @@ -19,6 +19,7 @@ public class TestPost { post.setApiVersion(getApiVersion(Post.class)); Metadata metadata = new Metadata(); metadata.setName("post-A"); + metadata.setVersion(1L); post.setMetadata(metadata); Post.PostSpec postSpec = new Post.PostSpec(); @@ -38,6 +39,7 @@ public class TestPost { snapshot.setApiVersion(getApiVersion(Snapshot.class)); Metadata metadata = new Metadata(); metadata.setName("snapshot-A"); + metadata.setVersion(1L); metadata.setCreationTimestamp(Instant.now()); snapshot.setMetadata(metadata); MetadataUtil.nullSafeAnnotations(snapshot).put(Snapshot.KEEP_RAW_ANNO, "true"); diff --git a/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java b/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java index c7b8e965d..81c8a0e9c 100644 --- a/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java +++ b/application/src/test/java/run/halo/app/content/comment/CommentServiceImplTest.java @@ -358,7 +358,8 @@ class CommentServiceImplTest { "apiVersion": "content.halo.run/v1alpha1", "kind": "Post", "metadata": { - "name": "fake-post" + "name": "fake-post", + "version": 1 } }, "stats": { @@ -403,7 +404,8 @@ class CommentServiceImplTest { "apiVersion": "content.halo.run/v1alpha1", "kind": "Post", "metadata": { - "name": "fake-post" + "name": "fake-post", + "version": 1 } }, "stats": { @@ -447,7 +449,8 @@ class CommentServiceImplTest { "apiVersion": "content.halo.run/v1alpha1", "kind": "Post", "metadata": { - "name": "fake-post" + "name": "fake-post", + "version": 1 } }, "stats": {