From 3b61807e8b69fdc06f654efd1f8fea65f5d82438 Mon Sep 17 00:00:00 2001 From: John Niang Date: Tue, 9 May 2023 10:49:43 +0800 Subject: [PATCH] Fix the problem of being able to search deleted posts (#3877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind bug /kind improvement /area core #### What this PR does / why we need it: This PR refactors post reconciler to reduce post updates and refines post events. Previously, we need 3 - 4 updates per reconciliation, but now we only need 1. And all events collected in reconciler will be fired after updating post. #### Which issue(s) this PR fixes: Fixes https://github.com/halo-dev/halo/issues/3121 #### Special notes for your reviewer: 0. Install search plugin 1. Create a public post and publish it 2. Search posts 3. Try to make the post private 4. Search posts 5. Try to make the post public 6. Search posts 7. Try to delete the post 8. Search posts 9. Try to recover the post 10. Search posts #### Does this PR introduce a user-facing change? ```release-note 修复依然能搜索到已删除文章的问题 ``` --- .../app/core/extension/content/Constant.java | 2 + .../halo/app/extension/ExtensionOperator.java | 3 +- .../run/halo/app/extension/ExtensionUtil.java | 31 ++ .../app/search/post/PostSearchService.java | 2 + .../halo/app/extension/ExtensionUtilTest.java | 59 +++ .../core/extension/endpoint/PostEndpoint.java | 8 - .../extension/reconciler/PostReconciler.java | 478 +++++++----------- ...cycledEvent.java => PostUpdatedEvent.java} | 5 +- .../event/post/PostVisibleChangedEvent.java | 2 + ...nsionUtil.java => ExtensionStoreUtil.java} | 4 +- .../app/extension/JSONExtensionConverter.java | 3 +- .../ReactiveExtensionClientImpl.java | 6 +- .../halo/app/search/IndicesServiceImpl.java | 61 ++- .../search/post/LucenePostSearchService.java | 23 +- .../app/search/post/PostEventReconciler.java | 27 +- .../halo/app/theme/finders/PostFinder.java | 8 + .../reconciler/PostReconcilerTest.java | 8 +- ...lTest.java => ExtensionStoreUtilTest.java} | 10 +- 18 files changed, 380 insertions(+), 360 deletions(-) create mode 100644 api/src/main/java/run/halo/app/extension/ExtensionUtil.java create mode 100644 api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java rename application/src/main/java/run/halo/app/event/post/{PostRecycledEvent.java => PostUpdatedEvent.java} (62%) rename application/src/main/java/run/halo/app/extension/{ExtensionUtil.java => ExtensionStoreUtil.java} (93%) rename application/src/test/java/run/halo/app/extension/{ExtensionUtilTest.java => ExtensionStoreUtilTest.java} (77%) diff --git a/api/src/main/java/run/halo/app/core/extension/content/Constant.java b/api/src/main/java/run/halo/app/core/extension/content/Constant.java index 93b4ba1c6..154b4ba28 100644 --- a/api/src/main/java/run/halo/app/core/extension/content/Constant.java +++ b/api/src/main/java/run/halo/app/core/extension/content/Constant.java @@ -8,4 +8,6 @@ public enum Constant { public static final String LAST_READ_TIME_ANNO = "content.halo.run/last-read-time"; public static final String PERMALINK_PATTERN_ANNO = "content.halo.run/permalink-pattern"; + + public static final String CHECKSUM_CONFIG_ANNO = "checksum/config"; } diff --git a/api/src/main/java/run/halo/app/extension/ExtensionOperator.java b/api/src/main/java/run/halo/app/extension/ExtensionOperator.java index 75b361f21..d54e780d6 100644 --- a/api/src/main/java/run/halo/app/extension/ExtensionOperator.java +++ b/api/src/main/java/run/halo/app/extension/ExtensionOperator.java @@ -97,7 +97,6 @@ public interface ExtensionOperator { } static boolean isDeleted(ExtensionOperator extension) { - return extension.getMetadata() != null - && extension.getMetadata().getDeletionTimestamp() != null; + return ExtensionUtil.isDeleted(extension); } } diff --git a/api/src/main/java/run/halo/app/extension/ExtensionUtil.java b/api/src/main/java/run/halo/app/extension/ExtensionUtil.java new file mode 100644 index 000000000..66d730e54 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/ExtensionUtil.java @@ -0,0 +1,31 @@ +package run.halo.app.extension; + +import java.util.HashSet; +import java.util.Set; + +public enum ExtensionUtil { + ; + + public static boolean isDeleted(ExtensionOperator extension) { + return extension.getMetadata() != null + && extension.getMetadata().getDeletionTimestamp() != null; + } + + public static void addFinalizers(MetadataOperator metadata, Set finalizers) { + var existingFinalizers = metadata.getFinalizers(); + if (existingFinalizers == null) { + existingFinalizers = new HashSet<>(); + } + existingFinalizers.addAll(finalizers); + metadata.setFinalizers(existingFinalizers); + } + + public static void removeFinalizers(MetadataOperator metadata, Set finalizers) { + var existingFinalizers = metadata.getFinalizers(); + if (existingFinalizers != null) { + existingFinalizers.removeAll(finalizers); + } + metadata.setFinalizers(existingFinalizers); + } + +} diff --git a/api/src/main/java/run/halo/app/search/post/PostSearchService.java b/api/src/main/java/run/halo/app/search/post/PostSearchService.java index 37f5b4c1f..22ac532be 100644 --- a/api/src/main/java/run/halo/app/search/post/PostSearchService.java +++ b/api/src/main/java/run/halo/app/search/post/PostSearchService.java @@ -14,4 +14,6 @@ public interface PostSearchService extends ExtensionPoint { void removeDocuments(Set postNames) throws Exception; + void removeAllDocuments() throws Exception; + } diff --git a/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java b/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java new file mode 100644 index 000000000..3529f8210 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java @@ -0,0 +1,59 @@ +package run.halo.app.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class ExtensionUtilTest { + + @Test + void testIsNotDeleted() { + var ext = mock(ExtensionOperator.class); + + when(ext.getMetadata()).thenReturn(null); + assertFalse(ExtensionUtil.isDeleted(ext)); + + var metadata = mock(Metadata.class); + when(ext.getMetadata()).thenReturn(metadata); + when(metadata.getDeletionTimestamp()).thenReturn(null); + assertFalse(ExtensionUtil.isDeleted(ext)); + + when(metadata.getDeletionTimestamp()).thenReturn(Instant.now()); + assertTrue(ExtensionUtil.isDeleted(ext)); + } + + @Test + void addFinalizers() { + var metadata = new Metadata(); + assertNull(metadata.getFinalizers()); + ExtensionUtil.addFinalizers(metadata, Set.of("fake")); + + assertEquals(Set.of("fake"), metadata.getFinalizers()); + + ExtensionUtil.addFinalizers(metadata, Set.of("fake")); + assertEquals(Set.of("fake"), metadata.getFinalizers()); + + ExtensionUtil.addFinalizers(metadata, Set.of("another-fake")); + assertEquals(Set.of("fake", "another-fake"), metadata.getFinalizers()); + } + + @Test + void removeFinalizers() { + var metadata = new Metadata(); + ExtensionUtil.removeFinalizers(metadata, Set.of("fake")); + assertNull(metadata.getFinalizers()); + + metadata.setFinalizers(new HashSet<>(Set.of("fake"))); + ExtensionUtil.removeFinalizers(metadata, Set.of("fake")); + assertEquals(Set.of(), metadata.getFinalizers()); + } + +} diff --git a/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java b/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java index 98af24bba..d228a866b 100644 --- a/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java +++ b/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java @@ -30,8 +30,6 @@ import run.halo.app.content.PostQuery; import run.halo.app.content.PostRequest; import run.halo.app.content.PostService; import run.halo.app.core.extension.content.Post; -import run.halo.app.event.post.PostRecycledEvent; -import run.halo.app.event.post.PostUnpublishedEvent; import run.halo.app.extension.ListResult; import run.halo.app.extension.MetadataUtil; import run.halo.app.extension.ReactiveExtensionClient; @@ -262,9 +260,6 @@ public class PostEndpoint implements CustomEndpoint { .flatMap(client::update)) .retryWhen(Retry.backoff(3, Duration.ofMillis(100)) .filter(t -> t instanceof OptimisticLockingFailureException)) - // TODO Fire unpublished event in reconciler in the future - .doOnNext(post -> eventPublisher.publishEvent( - new PostUnpublishedEvent(this, post.getMetadata().getName()))) .flatMap(post -> ServerResponse.ok().bodyValue(post)); } @@ -278,9 +273,6 @@ public class PostEndpoint implements CustomEndpoint { .flatMap(client::update)) .retryWhen(Retry.backoff(3, Duration.ofMillis(100)) .filter(t -> t instanceof OptimisticLockingFailureException)) - // TODO Fire recycled event in reconciler in the future - .doOnNext(post -> eventPublisher.publishEvent( - new PostRecycledEvent(this, post.getMetadata().getName()))) .flatMap(post -> ServerResponse.ok().bodyValue(post)); } diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java index df279d46d..d1fdec041 100644 --- a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java +++ b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java @@ -1,39 +1,45 @@ package run.halo.app.core.extension.reconciler; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; +import static run.halo.app.extension.ExtensionUtil.addFinalizers; +import static run.halo.app.extension.ExtensionUtil.removeFinalizers; + +import com.google.common.hash.Hashing; import java.time.Instant; +import java.util.HashMap; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; +import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Component; -import org.springframework.util.Assert; +import run.halo.app.content.ContentWrapper; import run.halo.app.content.PostService; import run.halo.app.content.permalinks.PostPermalinkPolicy; import run.halo.app.core.extension.content.Comment; import run.halo.app.core.extension.content.Constant; import run.halo.app.core.extension.content.Post; +import run.halo.app.core.extension.content.Post.PostPhase; +import run.halo.app.core.extension.content.Post.VisibleEnum; import run.halo.app.core.extension.content.Snapshot; import run.halo.app.event.post.PostPublishedEvent; import run.halo.app.event.post.PostUnpublishedEvent; +import run.halo.app.event.post.PostUpdatedEvent; import run.halo.app.event.post.PostVisibleChangedEvent; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionOperator; -import run.halo.app.extension.MetadataUtil; import run.halo.app.extension.Ref; import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.ControllerBuilder; import run.halo.app.extension.controller.Reconciler; import run.halo.app.infra.Condition; -import run.halo.app.infra.ConditionList; import run.halo.app.infra.ConditionStatus; import run.halo.app.infra.utils.HaloUtils; -import run.halo.app.infra.utils.JsonUtils; import run.halo.app.metrics.CounterService; import run.halo.app.metrics.MeterUtils; @@ -62,20 +68,134 @@ public class PostReconciler implements Reconciler { @Override public Result reconcile(Request request) { + var events = new HashSet(); client.fetch(Post.class, request.name()) .ifPresent(post -> { if (ExtensionOperator.isDeleted(post)) { - cleanUpResourcesAndRemoveFinalizer(request.name()); + removeFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME)); + unPublishPost(post, events); + cleanUpResources(post); + // update post to be able to be collected by gc collector. + client.update(post); + // fire event after updating post + events.forEach(eventPublisher::publishEvent); return; } - addFinalizerIfNecessary(post); + addFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME)); - // reconcile spec first - reconcileSpec(request.name()); - reconcileMetadata(request.name()); - reconcileStatus(request.name()); + var labels = post.getMetadata().getLabels(); + if (labels == null) { + labels = new HashMap<>(); + post.getMetadata().setLabels(labels); + } + + var annotations = post.getMetadata().getAnnotations(); + if (annotations == null) { + annotations = new HashMap<>(); + post.getMetadata().setAnnotations(annotations); + } + + var status = post.getStatus(); + if (status == null) { + status = new Post.PostStatus(); + post.setStatus(status); + } + + // calculate the sha256sum + var configSha256sum = Hashing.sha256().hashString(post.getSpec().toString(), UTF_8) + .toString(); + + var oldConfigChecksum = annotations.get(Constant.CHECKSUM_CONFIG_ANNO); + if (!Objects.equals(oldConfigChecksum, configSha256sum)) { + // if the checksum doesn't match + events.add(new PostUpdatedEvent(this, post.getMetadata().getName())); + annotations.put(Constant.CHECKSUM_CONFIG_ANNO, configSha256sum); + } + + var expectDelete = defaultIfNull(post.getSpec().getDeleted(), false); + var expectPublish = defaultIfNull(post.getSpec().getPublish(), false); + + if (expectDelete || !expectPublish) { + unPublishPost(post, events); + } else { + publishPost(post, events); + } + + labels.put(Post.DELETED_LABEL, expectDelete.toString()); + + var expectVisible = defaultIfNull(post.getSpec().getVisible(), VisibleEnum.PUBLIC); + var oldVisible = VisibleEnum.from(labels.get(Post.VISIBLE_LABEL)); + if (!Objects.equals(oldVisible, expectVisible)) { + eventPublisher.publishEvent( + new PostVisibleChangedEvent(request.name(), oldVisible, expectVisible)); + } + labels.put(Post.VISIBLE_LABEL, expectVisible.toString()); + + var ownerName = post.getSpec().getOwner(); + if (StringUtils.isNotBlank(ownerName)) { + labels.put(Post.OWNER_LABEL, ownerName); + } + + var publishTime = post.getSpec().getPublishTime(); + if (publishTime != null) { + labels.put(Post.ARCHIVE_YEAR_LABEL, HaloUtils.getYearText(publishTime)); + labels.put(Post.ARCHIVE_MONTH_LABEL, HaloUtils.getMonthText(publishTime)); + labels.put(Post.ARCHIVE_DAY_LABEL, HaloUtils.getDayText(publishTime)); + } + + var permalinkPattern = postPermalinkPolicy.pattern(); + annotations.put(Constant.PERMALINK_PATTERN_ANNO, permalinkPattern); + + status.setPermalink(postPermalinkPolicy.permalink(post)); + if (status.getPhase() == null) { + status.setPhase(PostPhase.DRAFT.toString()); + } + + var excerpt = post.getSpec().getExcerpt(); + if (excerpt == null) { + excerpt = new Post.Excerpt(); + } + var isAutoGenerate = defaultIfNull(excerpt.getAutoGenerate(), true); + if (isAutoGenerate) { + Optional contentWrapper = + postService.getContent(post.getSpec().getReleaseSnapshot(), + post.getSpec().getBaseSnapshot()) + .blockOptional(); + if (contentWrapper.isPresent()) { + String contentRevised = contentWrapper.get().getContent(); + status.setExcerpt(getExcerpt(contentRevised)); + } + } else { + status.setExcerpt(excerpt.getRaw()); + } + + + var ref = Ref.of(post); + // handle contributors + var headSnapshot = post.getSpec().getHeadSnapshot(); + var contributors = client.list(Snapshot.class, + snapshot -> ref.equals(snapshot.getSpec().getSubjectRef()), null) + .stream() + .map(snapshot -> { + Set usernames = snapshot.getSpec().getContributors(); + return Objects.requireNonNullElseGet(usernames, + () -> new HashSet()); + }) + .flatMap(Set::stream) + .distinct() + .sorted() + .toList(); + status.setContributors(contributors); + + // update in progress status + status.setInProgress( + !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot())); + + client.update(post); + // fire event after updating post + events.forEach(eventPublisher::publishEvent); }); - return new Result(false, null); + return Result.doNotRetry(); } @Override @@ -83,290 +203,78 @@ public class PostReconciler implements Reconciler { return builder .extension(new Post()) // TODO Make it configurable - .workerCount(10) + .workerCount(1) .build(); } - private void reconcileSpec(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - // un-publish post if necessary - if (post.isPublished() - && Objects.equals(false, post.getSpec().getPublish())) { - boolean success = unPublishReconcile(name); - if (success) { - eventPublisher.publishEvent(new PostUnpublishedEvent(this, name)); - } - return; - } - - try { - publishPost(name); - } catch (Throwable e) { - publishFailed(name, e); - throw e; - } - }); - } - - private void publishPost(String name) { - client.fetch(Post.class, name) - .filter(post -> Objects.equals(true, post.getSpec().getPublish())) - .ifPresent(post -> { - Map annotations = MetadataUtil.nullSafeAnnotations(post); - String lastReleasedSnapshot = annotations.get(Post.LAST_RELEASED_SNAPSHOT_ANNO); - String releaseSnapshot = post.getSpec().getReleaseSnapshot(); - if (StringUtils.isBlank(releaseSnapshot)) { - return; - } - - // do nothing if release snapshot is not changed and post is published - if (post.isPublished() - && StringUtils.equals(lastReleasedSnapshot, releaseSnapshot)) { - return; - } - Post.PostStatus status = post.getStatusOrDefault(); - - // validate release snapshot - Optional releasedSnapshotOpt = - client.fetch(Snapshot.class, releaseSnapshot); - if (releasedSnapshotOpt.isEmpty()) { - Condition condition = Condition.builder() - .type(Post.PostPhase.FAILED.name()) - .reason("SnapshotNotFound") - .message( - String.format("Snapshot [%s] not found for publish", releaseSnapshot)) - .status(ConditionStatus.FALSE) - .lastTransitionTime(Instant.now()) - .build(); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - status.setPhase(Post.PostPhase.FAILED.name()); - client.update(post); - return; - } - // do publish - annotations.put(Post.LAST_RELEASED_SNAPSHOT_ANNO, releaseSnapshot); - status.setPhase(Post.PostPhase.PUBLISHED.name()); - Condition condition = Condition.builder() - .type(Post.PostPhase.PUBLISHED.name()) - .reason("Published") - .message("Post published successfully.") - .lastTransitionTime(Instant.now()) - .status(ConditionStatus.TRUE) - .build(); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - - Post.changePublishedState(post, true); - if (post.getSpec().getPublishTime() == null) { - post.getSpec().setPublishTime(Instant.now()); - } - - // populate lastModifyTime - status.setLastModifyTime(releasedSnapshotOpt.get().getSpec().getLastModifyTime()); - - client.update(post); - eventPublisher.publishEvent(new PostPublishedEvent(this, name)); - }); - } - - private boolean unPublishReconcile(String name) { - return client.fetch(Post.class, name) - .map(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - Post.changePublishedState(post, false); - - final Post.PostStatus status = post.getStatusOrDefault(); - Condition condition = new Condition(); - condition.setType("CancelledPublish"); - condition.setStatus(ConditionStatus.TRUE); - condition.setReason(condition.getType()); - condition.setMessage("CancelledPublish"); - condition.setLastTransitionTime(Instant.now()); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - - status.setPhase(Post.PostPhase.DRAFT.name()); - if (!oldPost.equals(post)) { - client.update(post); - } - return true; - }) - .orElse(false); - } - - private void publishFailed(String name, Throwable error) { - Assert.notNull(name, "Name must not be null"); - Assert.notNull(error, "Error must not be null"); - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - - Post.PostStatus status = post.getStatusOrDefault(); - Post.PostPhase phase = Post.PostPhase.FAILED; - status.setPhase(phase.name()); - - final ConditionList conditions = status.getConditionsOrDefault(); + private void publishPost(Post post, Set events) { + var expectReleaseSnapshot = post.getSpec().getReleaseSnapshot(); + if (StringUtils.isBlank(expectReleaseSnapshot)) { + // Do nothing if release snapshot is not set + return; + } + var annotations = post.getMetadata().getAnnotations(); + var lastReleaseSnapshot = annotations.get(Post.LAST_RELEASED_SNAPSHOT_ANNO); + if (post.isPublished() + && Objects.equals(expectReleaseSnapshot, lastReleaseSnapshot)) { + // If the release snapshot is not change + return; + } + var status = post.getStatus(); + // validate the release snapshot + var snapshot = client.fetch(Snapshot.class, expectReleaseSnapshot); + if (snapshot.isEmpty()) { Condition condition = Condition.builder() - .type(phase.name()) - .reason("PublishFailed") - .message(error.getMessage()) + .type(PostPhase.FAILED.name()) + .reason("SnapshotNotFound") + .message( + String.format("Snapshot [%s] not found for publish", expectReleaseSnapshot)) .status(ConditionStatus.FALSE) .lastTransitionTime(Instant.now()) .build(); - conditions.addAndEvictFIFO(condition); - - post.setStatus(status); - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void reconcileMetadata(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - Post.PostSpec spec = post.getSpec(); - - // handle logic delete - Map labels = MetadataUtil.nullSafeLabels(post); - if (Objects.equals(spec.getDeleted(), true)) { - labels.put(Post.DELETED_LABEL, Boolean.TRUE.toString()); - } else { - labels.put(Post.DELETED_LABEL, Boolean.FALSE.toString()); - } - - fireVisibleChangedEventIfChanged(post); - labels.put(Post.VISIBLE_LABEL, - Objects.requireNonNullElse(spec.getVisible(), Post.VisibleEnum.PUBLIC).name()); - - labels.put(Post.OWNER_LABEL, spec.getOwner()); - Instant publishTime = post.getSpec().getPublishTime(); - if (publishTime != null) { - labels.put(Post.ARCHIVE_YEAR_LABEL, HaloUtils.getYearText(publishTime)); - labels.put(Post.ARCHIVE_MONTH_LABEL, HaloUtils.getMonthText(publishTime)); - labels.put(Post.ARCHIVE_DAY_LABEL, HaloUtils.getDayText(publishTime)); - } - if (!labels.containsKey(Post.PUBLISHED_LABEL)) { - labels.put(Post.PUBLISHED_LABEL, Boolean.FALSE.toString()); - } - - Map annotations = MetadataUtil.nullSafeAnnotations(post); - String newPattern = postPermalinkPolicy.pattern(); - annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern); - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void fireVisibleChangedEventIfChanged(Post post) { + status.getConditionsOrDefault().addAndEvictFIFO(condition); + status.setPhase(PostPhase.FAILED.name()); + return; + } + annotations.put(Post.LAST_RELEASED_SNAPSHOT_ANNO, expectReleaseSnapshot); + status.setPhase(PostPhase.PUBLISHED.toString()); + var condition = Condition.builder() + .type(PostPhase.PUBLISHED.name()) + .reason("Published") + .message("Post published successfully.") + .lastTransitionTime(Instant.now()) + .status(ConditionStatus.TRUE) + .build(); + status.getConditionsOrDefault().addAndEvictFIFO(condition); var labels = post.getMetadata().getLabels(); - if (labels == null) { + labels.put(Post.PUBLISHED_LABEL, Boolean.TRUE.toString()); + if (post.getSpec().getPublishTime() == null) { + // TODO Set the field in creation hook in the future. + post.getSpec().setPublishTime(Instant.now()); + } + status.setLastModifyTime(snapshot.get().getSpec().getLastModifyTime()); + events.add(new PostPublishedEvent(this, post.getMetadata().getName())); + } + + void unPublishPost(Post post, Set events) { + if (!post.isPublished()) { return; } - var name = post.getMetadata().getName(); - var oldVisibleStr = labels.get(Post.VISIBLE_LABEL); - if (oldVisibleStr != null) { - var oldVisible = Post.VisibleEnum.valueOf(oldVisibleStr); - var expectVisible = post.getSpec().getVisible(); - if (!Objects.equals(oldVisible, expectVisible)) { - eventPublisher.publishEvent( - new PostVisibleChangedEvent(name, oldVisible, expectVisible)); - } - } - } + var labels = post.getMetadata().getLabels(); + labels.put(Post.PUBLISHED_LABEL, Boolean.FALSE.toString()); + var status = post.getStatus(); - private void reconcileStatus(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); + var condition = new Condition(); + condition.setType("CancelledPublish"); + condition.setStatus(ConditionStatus.TRUE); + condition.setReason(condition.getType()); + condition.setMessage("CancelledPublish"); + condition.setLastTransitionTime(Instant.now()); + status.getConditionsOrDefault().addAndEvictFIFO(condition); - post.getStatusOrDefault() - .setPermalink(postPermalinkPolicy.permalink(post)); + status.setPhase(PostPhase.DRAFT.toString()); - Post.PostStatus status = post.getStatusOrDefault(); - if (status.getPhase() == null) { - status.setPhase(Post.PostPhase.DRAFT.name()); - } - Post.PostSpec spec = post.getSpec(); - // handle excerpt - Post.Excerpt excerpt = spec.getExcerpt(); - if (excerpt == null) { - excerpt = new Post.Excerpt(); - excerpt.setAutoGenerate(true); - spec.setExcerpt(excerpt); - } - if (excerpt.getAutoGenerate()) { - postService.getContent(spec.getReleaseSnapshot(), spec.getBaseSnapshot()) - .blockOptional() - .ifPresent(content -> { - String contentRevised = content.getContent(); - status.setExcerpt(getExcerpt(contentRevised)); - }); - } else { - status.setExcerpt(excerpt.getRaw()); - } - - Ref ref = Ref.of(post); - // handle contributors - String headSnapshot = post.getSpec().getHeadSnapshot(); - List contributors = client.list(Snapshot.class, - snapshot -> ref.equals(snapshot.getSpec().getSubjectRef()), null) - .stream() - .peek(snapshot -> { - snapshot.getSpec().setContentPatch(StringUtils.EMPTY); - snapshot.getSpec().setRawPatch(StringUtils.EMPTY); - }) - .map(snapshot -> { - Set usernames = snapshot.getSpec().getContributors(); - return Objects.requireNonNullElseGet(usernames, - () -> new HashSet()); - }) - .flatMap(Set::stream) - .distinct() - .sorted() - .toList(); - status.setContributors(contributors); - - // update in progress status - status.setInProgress( - !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot())); - - if (post.isPublished() && status.getLastModifyTime() == null) { - client.fetch(Snapshot.class, post.getSpec().getReleaseSnapshot()) - .ifPresent(releasedSnapshot -> - status.setLastModifyTime(releasedSnapshot.getSpec().getLastModifyTime())); - } - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void addFinalizerIfNecessary(Post oldPost) { - Set finalizers = oldPost.getMetadata().getFinalizers(); - if (finalizers != null && finalizers.contains(FINALIZER_NAME)) { - return; - } - client.fetch(Post.class, oldPost.getMetadata().getName()) - .ifPresent(post -> { - Set newFinalizers = post.getMetadata().getFinalizers(); - if (newFinalizers == null) { - newFinalizers = new HashSet<>(); - post.getMetadata().setFinalizers(newFinalizers); - } - newFinalizers.add(FINALIZER_NAME); - client.update(post); - }); - } - - private void cleanUpResourcesAndRemoveFinalizer(String postName) { - client.fetch(Post.class, postName).ifPresent(post -> { - cleanUpResources(post); - if (post.getMetadata().getFinalizers() != null) { - post.getMetadata().getFinalizers().remove(FINALIZER_NAME); - } - client.update(post); - }); + events.add(new PostUnpublishedEvent(this, post.getMetadata().getName())); } private void cleanUpResources(Post post) { @@ -377,7 +285,7 @@ public class PostReconciler implements Reconciler { .forEach(client::delete); // clean up comments - client.list(Comment.class, comment -> comment.getSpec().getSubjectRef().equals(ref), + client.list(Comment.class, comment -> ref.equals(comment.getSpec().getSubjectRef()), null) .forEach(client::delete); diff --git a/application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java b/application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java similarity index 62% rename from application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java rename to application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java index 6d5e3bd3f..a17d8677c 100644 --- a/application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java +++ b/application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java @@ -2,15 +2,16 @@ package run.halo.app.event.post; import org.springframework.context.ApplicationEvent; -public class PostRecycledEvent extends ApplicationEvent implements PostEvent { +public class PostUpdatedEvent extends ApplicationEvent implements PostEvent { private final String postName; - public PostRecycledEvent(Object source, String postName) { + public PostUpdatedEvent(Object source, String postName) { super(source); this.postName = postName; } + @Override public String getName() { return postName; } diff --git a/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java b/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java index 4b1355cd1..6e2d1c2fb 100644 --- a/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java +++ b/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java @@ -1,6 +1,7 @@ package run.halo.app.event.post; import lombok.Data; +import org.springframework.lang.Nullable; import run.halo.app.core.extension.content.Post; @Data @@ -8,6 +9,7 @@ public class PostVisibleChangedEvent implements PostEvent { private final String postName; + @Nullable private final Post.VisibleEnum oldVisible; private final Post.VisibleEnum newVisible; diff --git a/application/src/main/java/run/halo/app/extension/ExtensionUtil.java b/application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java similarity index 93% rename from application/src/main/java/run/halo/app/extension/ExtensionUtil.java rename to application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java index 93e35b696..0f73a63b0 100644 --- a/application/src/main/java/run/halo/app/extension/ExtensionUtil.java +++ b/application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java @@ -7,9 +7,9 @@ import org.springframework.util.StringUtils; * * @author johnniang */ -public final class ExtensionUtil { +public final class ExtensionStoreUtil { - private ExtensionUtil() { + private ExtensionStoreUtil() { } /** diff --git a/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java b/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java index 0cfa1be10..eb9107db2 100644 --- a/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java +++ b/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java @@ -2,6 +2,7 @@ package run.halo.app.extension; import static org.openapi4j.core.validation.ValidationSeverity.ERROR; import static org.springframework.util.StringUtils.arrayToCommaDelimitedString; +import static run.halo.app.extension.ExtensionStoreUtil.buildStoreName; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -65,7 +66,7 @@ public class JSONExtensionConverter implements ExtensionConverter { } var version = extension.getMetadata().getVersion(); - var storeName = ExtensionUtil.buildStoreName(scheme, extension.getMetadata().getName()); + var storeName = buildStoreName(scheme, extension.getMetadata().getName()); var data = objectMapper.writeValueAsBytes(extensionJsonNode); return new ExtensionStore(storeName, data, version); } catch (IOException e) { diff --git a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java index 7466e958b..ad1a16497 100644 --- a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java +++ b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java @@ -46,7 +46,7 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { public Flux list(Class type, Predicate predicate, Comparator comparator) { var scheme = schemeManager.get(type); - var prefix = ExtensionUtil.buildStoreNamePrefix(scheme); + var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme); return client.listByNamePrefix(prefix) .map(extensionStore -> converter.convertFrom(type, extensionStore)) @@ -75,14 +75,14 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { @Override public Mono fetch(Class type, String name) { - var storeName = ExtensionUtil.buildStoreName(schemeManager.get(type), name); + var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(type), name); return client.fetchByName(storeName) .map(extensionStore -> converter.convertFrom(type, extensionStore)); } @Override public Mono fetch(GroupVersionKind gvk, String name) { - var storeName = ExtensionUtil.buildStoreName(schemeManager.get(gvk), name); + var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(gvk), name); return client.fetchByName(storeName) .map(extensionStore -> converter.convertFrom(Unstructured.class, extensionStore)); } diff --git a/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java b/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java index 5e8cf2a84..9c52f6039 100644 --- a/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java +++ b/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java @@ -3,6 +3,7 @@ package run.halo.app.search; import org.springframework.stereotype.Service; import reactor.core.Exceptions; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; import run.halo.app.core.extension.content.Post; import run.halo.app.plugin.extensionpoint.ExtensionGetter; import run.halo.app.search.post.PostSearchService; @@ -23,28 +24,42 @@ public class IndicesServiceImpl implements IndicesService { @Override public Mono rebuildPostIndices() { return extensionGetter.getEnabledExtension(PostSearchService.class) - .flatMap(searchService -> postFinder.listAll() - .filter(post -> Post.isPublished(post.getMetadata())) - .flatMap(listedPostVo -> { - PostVo postVo = PostVo.from(listedPostVo); - return postFinder.content(postVo.getMetadata().getName()) - .map(content -> { - postVo.setContent(content); - return postVo; - }) - .defaultIfEmpty(postVo); - }) - .map(PostDocUtils::from) - .limitRate(100) - .buffer(100) - .doOnNext(postDocs -> { - try { - searchService.addDocuments(postDocs); - } catch (Exception e) { - throw Exceptions.propagate(e); - } - }) - .then() - ); + .flatMap(searchService -> Mono.fromRunnable( + () -> { + try { + // remove all docs before rebuilding + searchService.removeAllDocuments(); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }) + .then(rebuildPostIndices(searchService)) + ) + .subscribeOn(Schedulers.boundedElastic()); + } + + private Mono rebuildPostIndices(PostSearchService searchService) { + return postFinder.listAll() + .filter(post -> Post.isPublished(post.getMetadata())) + .flatMap(listedPostVo -> { + PostVo postVo = PostVo.from(listedPostVo); + return postFinder.content(postVo.getMetadata().getName()) + .map(content -> { + postVo.setContent(content); + return postVo; + }) + .defaultIfEmpty(postVo); + }) + .map(PostDocUtils::from) + .limitRate(100) + .buffer(100) + .doOnNext(postDocs -> { + try { + searchService.addDocuments(postDocs); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }) + .then(); } } diff --git a/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java b/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java index 423d77a8d..274c2c75c 100644 --- a/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java +++ b/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java @@ -126,6 +126,15 @@ public class LucenePostSearchService implements PostSearchService, DisposableBea } } + @Override + public void removeAllDocuments() throws Exception { + var writeConfig = new IndexWriterConfig(analyzer); + writeConfig.setOpenMode(APPEND); + try (var writer = new IndexWriter(postIndexDir, writeConfig)) { + writer.deleteAll(); + } + } + @Override public void destroy() throws Exception { analyzer.close(); @@ -145,11 +154,19 @@ public class LucenePostSearchService implements PostSearchService, DisposableBea doc.add(new StringField("name", post.name(), YES)); doc.add(new StoredField("title", post.title())); - var content = Jsoup.clean(stripToEmpty(post.excerpt()) + stripToEmpty(post.content()), - Safelist.none()); + var cleanExcerpt = Jsoup.clean(stripToEmpty(post.excerpt()), Safelist.none()); + var cleanContent = Jsoup.clean(stripToEmpty(post.content()), Safelist.none()); + + var contentBuilder = new StringBuilder(cleanExcerpt); + if (!contentBuilder.isEmpty()) { + contentBuilder.append(' '); + } + contentBuilder.append(cleanContent); + + var content = contentBuilder.toString(); doc.add(new StoredField("content", content)); - doc.add(new TextField("searchable", post.title() + content, NO)); + doc.add(new TextField("searchable", post.title() + " " + content, NO)); long publishTimestamp = post.publishTimestamp().toEpochMilli(); doc.add(new LongPoint("publishTimestamp", publishTimestamp)); diff --git a/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java b/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java index 320310e97..b96f12287 100644 --- a/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java +++ b/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java @@ -13,8 +13,8 @@ import org.springframework.stereotype.Component; import reactor.core.Exceptions; import run.halo.app.event.post.PostEvent; import run.halo.app.event.post.PostPublishedEvent; -import run.halo.app.event.post.PostRecycledEvent; import run.halo.app.event.post.PostUnpublishedEvent; +import run.halo.app.event.post.PostUpdatedEvent; import run.halo.app.event.post.PostVisibleChangedEvent; import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.ControllerBuilder; @@ -52,11 +52,10 @@ public class PostEventReconciler implements Reconciler, SmartLifecycl @Override public Result reconcile(PostEvent postEvent) { - if (postEvent instanceof PostPublishedEvent) { + if (postEvent instanceof PostPublishedEvent || postEvent instanceof PostUpdatedEvent) { addPostDoc(postEvent.getName()); } - if (postEvent instanceof PostUnpublishedEvent - || postEvent instanceof PostRecycledEvent) { + if (postEvent instanceof PostUnpublishedEvent) { deletePostDoc(postEvent.getName()); } if (postEvent instanceof PostVisibleChangedEvent visibleChangedEvent) { @@ -81,29 +80,13 @@ public class PostEventReconciler implements Reconciler, SmartLifecycl ); } - @EventListener(PostPublishedEvent.class) - public void handlePostPublished(PostPublishedEvent publishedEvent) { - postEventQueue.addImmediately(publishedEvent); - } - - @EventListener(PostUnpublishedEvent.class) - public void handlePostUnpublished(PostUnpublishedEvent unpublishedEvent) { - postEventQueue.addImmediately(unpublishedEvent); - } - - @EventListener(PostRecycledEvent.class) - public void handlePostRecycled(PostRecycledEvent recycledEvent) { - postEventQueue.addImmediately(recycledEvent); - } - - @EventListener(PostVisibleChangedEvent.class) - public void handlePostVisibleChanged(PostVisibleChangedEvent event) { + @EventListener(PostEvent.class) + public void handlePostEvent(PostEvent event) { postEventQueue.addImmediately(event); } void addPostDoc(String postName) { postFinder.getByName(postName) - .filter(postVo -> PUBLIC.equals(postVo.getSpec().getVisible())) .map(PostDocUtils::from) .flatMap(postDoc -> extensionGetter.getEnabledExtension(PostSearchService.class) .doOnNext(searchService -> { diff --git a/application/src/main/java/run/halo/app/theme/finders/PostFinder.java b/application/src/main/java/run/halo/app/theme/finders/PostFinder.java index be2ba02a0..834797610 100644 --- a/application/src/main/java/run/halo/app/theme/finders/PostFinder.java +++ b/application/src/main/java/run/halo/app/theme/finders/PostFinder.java @@ -19,6 +19,14 @@ import run.halo.app.theme.finders.vo.PostVo; */ public interface PostFinder { + /** + * Gets post detail by name. + *

+ * We ensure the post is public, non-deleted and published. + * + * @param postName is post name + * @return post detail + */ Mono getByName(String postName); Mono content(String postName); diff --git a/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java b/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java index 9cd06b045..adb9fbc77 100644 --- a/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java +++ b/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java @@ -77,7 +77,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(3)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); verify(postPermalinkPolicy, times(1)).permalink(any()); @@ -118,7 +118,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(4)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getExcerpt()).isEqualTo("hello world"); } @@ -154,7 +154,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(4)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getLastModifyTime()).isEqualTo(lastModifyTime); verify(eventPublisher).publishEvent(any(PostPublishedEvent.class)); @@ -183,7 +183,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(3)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getLastModifyTime()).isNull(); } diff --git a/application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java b/application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java similarity index 77% rename from application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java rename to application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java index 8c71678b6..1f70b3f24 100644 --- a/application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java +++ b/application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class ExtensionUtilTest { +class ExtensionStoreUtilTest { Scheme scheme; @@ -28,19 +28,19 @@ class ExtensionUtilTest { @Test void buildStoreNamePrefix() { - var prefix = ExtensionUtil.buildStoreNamePrefix(scheme); + var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme); assertEquals("/registry/fake.halo.run/fakes", prefix); - prefix = ExtensionUtil.buildStoreNamePrefix(grouplessScheme); + prefix = ExtensionStoreUtil.buildStoreNamePrefix(grouplessScheme); assertEquals("/registry/fakes", prefix); } @Test void buildStoreName() { - var storeName = ExtensionUtil.buildStoreName(scheme, "fake-name"); + var storeName = ExtensionStoreUtil.buildStoreName(scheme, "fake-name"); assertEquals("/registry/fake.halo.run/fakes/fake-name", storeName); - storeName = ExtensionUtil.buildStoreName(grouplessScheme, "fake-name"); + storeName = ExtensionStoreUtil.buildStoreName(grouplessScheme, "fake-name"); assertEquals("/registry/fakes/fake-name", storeName); }