diff --git a/api/src/main/java/run/halo/app/core/extension/content/Constant.java b/api/src/main/java/run/halo/app/core/extension/content/Constant.java index 93b4ba1c6..154b4ba28 100644 --- a/api/src/main/java/run/halo/app/core/extension/content/Constant.java +++ b/api/src/main/java/run/halo/app/core/extension/content/Constant.java @@ -8,4 +8,6 @@ public enum Constant { public static final String LAST_READ_TIME_ANNO = "content.halo.run/last-read-time"; public static final String PERMALINK_PATTERN_ANNO = "content.halo.run/permalink-pattern"; + + public static final String CHECKSUM_CONFIG_ANNO = "checksum/config"; } diff --git a/api/src/main/java/run/halo/app/extension/ExtensionOperator.java b/api/src/main/java/run/halo/app/extension/ExtensionOperator.java index 75b361f21..d54e780d6 100644 --- a/api/src/main/java/run/halo/app/extension/ExtensionOperator.java +++ b/api/src/main/java/run/halo/app/extension/ExtensionOperator.java @@ -97,7 +97,6 @@ public interface ExtensionOperator { } static boolean isDeleted(ExtensionOperator extension) { - return extension.getMetadata() != null - && extension.getMetadata().getDeletionTimestamp() != null; + return ExtensionUtil.isDeleted(extension); } } diff --git a/api/src/main/java/run/halo/app/extension/ExtensionUtil.java b/api/src/main/java/run/halo/app/extension/ExtensionUtil.java new file mode 100644 index 000000000..66d730e54 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/ExtensionUtil.java @@ -0,0 +1,31 @@ +package run.halo.app.extension; + +import java.util.HashSet; +import java.util.Set; + +public enum ExtensionUtil { + ; + + public static boolean isDeleted(ExtensionOperator extension) { + return extension.getMetadata() != null + && extension.getMetadata().getDeletionTimestamp() != null; + } + + public static void addFinalizers(MetadataOperator metadata, Set finalizers) { + var existingFinalizers = metadata.getFinalizers(); + if (existingFinalizers == null) { + existingFinalizers = new HashSet<>(); + } + existingFinalizers.addAll(finalizers); + metadata.setFinalizers(existingFinalizers); + } + + public static void removeFinalizers(MetadataOperator metadata, Set finalizers) { + var existingFinalizers = metadata.getFinalizers(); + if (existingFinalizers != null) { + existingFinalizers.removeAll(finalizers); + } + metadata.setFinalizers(existingFinalizers); + } + +} diff --git a/api/src/main/java/run/halo/app/search/post/PostSearchService.java b/api/src/main/java/run/halo/app/search/post/PostSearchService.java index 37f5b4c1f..22ac532be 100644 --- a/api/src/main/java/run/halo/app/search/post/PostSearchService.java +++ b/api/src/main/java/run/halo/app/search/post/PostSearchService.java @@ -14,4 +14,6 @@ public interface PostSearchService extends ExtensionPoint { void removeDocuments(Set postNames) throws Exception; + void removeAllDocuments() throws Exception; + } diff --git a/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java b/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java new file mode 100644 index 000000000..3529f8210 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java @@ -0,0 +1,59 @@ +package run.halo.app.extension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class ExtensionUtilTest { + + @Test + void testIsNotDeleted() { + var ext = mock(ExtensionOperator.class); + + when(ext.getMetadata()).thenReturn(null); + assertFalse(ExtensionUtil.isDeleted(ext)); + + var metadata = mock(Metadata.class); + when(ext.getMetadata()).thenReturn(metadata); + when(metadata.getDeletionTimestamp()).thenReturn(null); + assertFalse(ExtensionUtil.isDeleted(ext)); + + when(metadata.getDeletionTimestamp()).thenReturn(Instant.now()); + assertTrue(ExtensionUtil.isDeleted(ext)); + } + + @Test + void addFinalizers() { + var metadata = new Metadata(); + assertNull(metadata.getFinalizers()); + ExtensionUtil.addFinalizers(metadata, Set.of("fake")); + + assertEquals(Set.of("fake"), metadata.getFinalizers()); + + ExtensionUtil.addFinalizers(metadata, Set.of("fake")); + assertEquals(Set.of("fake"), metadata.getFinalizers()); + + ExtensionUtil.addFinalizers(metadata, Set.of("another-fake")); + assertEquals(Set.of("fake", "another-fake"), metadata.getFinalizers()); + } + + @Test + void removeFinalizers() { + var metadata = new Metadata(); + ExtensionUtil.removeFinalizers(metadata, Set.of("fake")); + assertNull(metadata.getFinalizers()); + + metadata.setFinalizers(new HashSet<>(Set.of("fake"))); + ExtensionUtil.removeFinalizers(metadata, Set.of("fake")); + assertEquals(Set.of(), metadata.getFinalizers()); + } + +} diff --git a/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java b/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java index 98af24bba..d228a866b 100644 --- a/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java +++ b/application/src/main/java/run/halo/app/core/extension/endpoint/PostEndpoint.java @@ -30,8 +30,6 @@ import run.halo.app.content.PostQuery; import run.halo.app.content.PostRequest; import run.halo.app.content.PostService; import run.halo.app.core.extension.content.Post; -import run.halo.app.event.post.PostRecycledEvent; -import run.halo.app.event.post.PostUnpublishedEvent; import run.halo.app.extension.ListResult; import run.halo.app.extension.MetadataUtil; import run.halo.app.extension.ReactiveExtensionClient; @@ -262,9 +260,6 @@ public class PostEndpoint implements CustomEndpoint { .flatMap(client::update)) .retryWhen(Retry.backoff(3, Duration.ofMillis(100)) .filter(t -> t instanceof OptimisticLockingFailureException)) - // TODO Fire unpublished event in reconciler in the future - .doOnNext(post -> eventPublisher.publishEvent( - new PostUnpublishedEvent(this, post.getMetadata().getName()))) .flatMap(post -> ServerResponse.ok().bodyValue(post)); } @@ -278,9 +273,6 @@ public class PostEndpoint implements CustomEndpoint { .flatMap(client::update)) .retryWhen(Retry.backoff(3, Duration.ofMillis(100)) .filter(t -> t instanceof OptimisticLockingFailureException)) - // TODO Fire recycled event in reconciler in the future - .doOnNext(post -> eventPublisher.publishEvent( - new PostRecycledEvent(this, post.getMetadata().getName()))) .flatMap(post -> ServerResponse.ok().bodyValue(post)); } diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java index df279d46d..d1fdec041 100644 --- a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java +++ b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java @@ -1,39 +1,45 @@ package run.halo.app.core.extension.reconciler; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; +import static run.halo.app.extension.ExtensionUtil.addFinalizers; +import static run.halo.app.extension.ExtensionUtil.removeFinalizers; + +import com.google.common.hash.Hashing; import java.time.Instant; +import java.util.HashMap; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; +import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Component; -import org.springframework.util.Assert; +import run.halo.app.content.ContentWrapper; import run.halo.app.content.PostService; import run.halo.app.content.permalinks.PostPermalinkPolicy; import run.halo.app.core.extension.content.Comment; import run.halo.app.core.extension.content.Constant; import run.halo.app.core.extension.content.Post; +import run.halo.app.core.extension.content.Post.PostPhase; +import run.halo.app.core.extension.content.Post.VisibleEnum; import run.halo.app.core.extension.content.Snapshot; import run.halo.app.event.post.PostPublishedEvent; import run.halo.app.event.post.PostUnpublishedEvent; +import run.halo.app.event.post.PostUpdatedEvent; import run.halo.app.event.post.PostVisibleChangedEvent; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionOperator; -import run.halo.app.extension.MetadataUtil; import run.halo.app.extension.Ref; import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.ControllerBuilder; import run.halo.app.extension.controller.Reconciler; import run.halo.app.infra.Condition; -import run.halo.app.infra.ConditionList; import run.halo.app.infra.ConditionStatus; import run.halo.app.infra.utils.HaloUtils; -import run.halo.app.infra.utils.JsonUtils; import run.halo.app.metrics.CounterService; import run.halo.app.metrics.MeterUtils; @@ -62,20 +68,134 @@ public class PostReconciler implements Reconciler { @Override public Result reconcile(Request request) { + var events = new HashSet(); client.fetch(Post.class, request.name()) .ifPresent(post -> { if (ExtensionOperator.isDeleted(post)) { - cleanUpResourcesAndRemoveFinalizer(request.name()); + removeFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME)); + unPublishPost(post, events); + cleanUpResources(post); + // update post to be able to be collected by gc collector. + client.update(post); + // fire event after updating post + events.forEach(eventPublisher::publishEvent); return; } - addFinalizerIfNecessary(post); + addFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME)); - // reconcile spec first - reconcileSpec(request.name()); - reconcileMetadata(request.name()); - reconcileStatus(request.name()); + var labels = post.getMetadata().getLabels(); + if (labels == null) { + labels = new HashMap<>(); + post.getMetadata().setLabels(labels); + } + + var annotations = post.getMetadata().getAnnotations(); + if (annotations == null) { + annotations = new HashMap<>(); + post.getMetadata().setAnnotations(annotations); + } + + var status = post.getStatus(); + if (status == null) { + status = new Post.PostStatus(); + post.setStatus(status); + } + + // calculate the sha256sum + var configSha256sum = Hashing.sha256().hashString(post.getSpec().toString(), UTF_8) + .toString(); + + var oldConfigChecksum = annotations.get(Constant.CHECKSUM_CONFIG_ANNO); + if (!Objects.equals(oldConfigChecksum, configSha256sum)) { + // if the checksum doesn't match + events.add(new PostUpdatedEvent(this, post.getMetadata().getName())); + annotations.put(Constant.CHECKSUM_CONFIG_ANNO, configSha256sum); + } + + var expectDelete = defaultIfNull(post.getSpec().getDeleted(), false); + var expectPublish = defaultIfNull(post.getSpec().getPublish(), false); + + if (expectDelete || !expectPublish) { + unPublishPost(post, events); + } else { + publishPost(post, events); + } + + labels.put(Post.DELETED_LABEL, expectDelete.toString()); + + var expectVisible = defaultIfNull(post.getSpec().getVisible(), VisibleEnum.PUBLIC); + var oldVisible = VisibleEnum.from(labels.get(Post.VISIBLE_LABEL)); + if (!Objects.equals(oldVisible, expectVisible)) { + eventPublisher.publishEvent( + new PostVisibleChangedEvent(request.name(), oldVisible, expectVisible)); + } + labels.put(Post.VISIBLE_LABEL, expectVisible.toString()); + + var ownerName = post.getSpec().getOwner(); + if (StringUtils.isNotBlank(ownerName)) { + labels.put(Post.OWNER_LABEL, ownerName); + } + + var publishTime = post.getSpec().getPublishTime(); + if (publishTime != null) { + labels.put(Post.ARCHIVE_YEAR_LABEL, HaloUtils.getYearText(publishTime)); + labels.put(Post.ARCHIVE_MONTH_LABEL, HaloUtils.getMonthText(publishTime)); + labels.put(Post.ARCHIVE_DAY_LABEL, HaloUtils.getDayText(publishTime)); + } + + var permalinkPattern = postPermalinkPolicy.pattern(); + annotations.put(Constant.PERMALINK_PATTERN_ANNO, permalinkPattern); + + status.setPermalink(postPermalinkPolicy.permalink(post)); + if (status.getPhase() == null) { + status.setPhase(PostPhase.DRAFT.toString()); + } + + var excerpt = post.getSpec().getExcerpt(); + if (excerpt == null) { + excerpt = new Post.Excerpt(); + } + var isAutoGenerate = defaultIfNull(excerpt.getAutoGenerate(), true); + if (isAutoGenerate) { + Optional contentWrapper = + postService.getContent(post.getSpec().getReleaseSnapshot(), + post.getSpec().getBaseSnapshot()) + .blockOptional(); + if (contentWrapper.isPresent()) { + String contentRevised = contentWrapper.get().getContent(); + status.setExcerpt(getExcerpt(contentRevised)); + } + } else { + status.setExcerpt(excerpt.getRaw()); + } + + + var ref = Ref.of(post); + // handle contributors + var headSnapshot = post.getSpec().getHeadSnapshot(); + var contributors = client.list(Snapshot.class, + snapshot -> ref.equals(snapshot.getSpec().getSubjectRef()), null) + .stream() + .map(snapshot -> { + Set usernames = snapshot.getSpec().getContributors(); + return Objects.requireNonNullElseGet(usernames, + () -> new HashSet()); + }) + .flatMap(Set::stream) + .distinct() + .sorted() + .toList(); + status.setContributors(contributors); + + // update in progress status + status.setInProgress( + !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot())); + + client.update(post); + // fire event after updating post + events.forEach(eventPublisher::publishEvent); }); - return new Result(false, null); + return Result.doNotRetry(); } @Override @@ -83,290 +203,78 @@ public class PostReconciler implements Reconciler { return builder .extension(new Post()) // TODO Make it configurable - .workerCount(10) + .workerCount(1) .build(); } - private void reconcileSpec(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - // un-publish post if necessary - if (post.isPublished() - && Objects.equals(false, post.getSpec().getPublish())) { - boolean success = unPublishReconcile(name); - if (success) { - eventPublisher.publishEvent(new PostUnpublishedEvent(this, name)); - } - return; - } - - try { - publishPost(name); - } catch (Throwable e) { - publishFailed(name, e); - throw e; - } - }); - } - - private void publishPost(String name) { - client.fetch(Post.class, name) - .filter(post -> Objects.equals(true, post.getSpec().getPublish())) - .ifPresent(post -> { - Map annotations = MetadataUtil.nullSafeAnnotations(post); - String lastReleasedSnapshot = annotations.get(Post.LAST_RELEASED_SNAPSHOT_ANNO); - String releaseSnapshot = post.getSpec().getReleaseSnapshot(); - if (StringUtils.isBlank(releaseSnapshot)) { - return; - } - - // do nothing if release snapshot is not changed and post is published - if (post.isPublished() - && StringUtils.equals(lastReleasedSnapshot, releaseSnapshot)) { - return; - } - Post.PostStatus status = post.getStatusOrDefault(); - - // validate release snapshot - Optional releasedSnapshotOpt = - client.fetch(Snapshot.class, releaseSnapshot); - if (releasedSnapshotOpt.isEmpty()) { - Condition condition = Condition.builder() - .type(Post.PostPhase.FAILED.name()) - .reason("SnapshotNotFound") - .message( - String.format("Snapshot [%s] not found for publish", releaseSnapshot)) - .status(ConditionStatus.FALSE) - .lastTransitionTime(Instant.now()) - .build(); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - status.setPhase(Post.PostPhase.FAILED.name()); - client.update(post); - return; - } - // do publish - annotations.put(Post.LAST_RELEASED_SNAPSHOT_ANNO, releaseSnapshot); - status.setPhase(Post.PostPhase.PUBLISHED.name()); - Condition condition = Condition.builder() - .type(Post.PostPhase.PUBLISHED.name()) - .reason("Published") - .message("Post published successfully.") - .lastTransitionTime(Instant.now()) - .status(ConditionStatus.TRUE) - .build(); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - - Post.changePublishedState(post, true); - if (post.getSpec().getPublishTime() == null) { - post.getSpec().setPublishTime(Instant.now()); - } - - // populate lastModifyTime - status.setLastModifyTime(releasedSnapshotOpt.get().getSpec().getLastModifyTime()); - - client.update(post); - eventPublisher.publishEvent(new PostPublishedEvent(this, name)); - }); - } - - private boolean unPublishReconcile(String name) { - return client.fetch(Post.class, name) - .map(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - Post.changePublishedState(post, false); - - final Post.PostStatus status = post.getStatusOrDefault(); - Condition condition = new Condition(); - condition.setType("CancelledPublish"); - condition.setStatus(ConditionStatus.TRUE); - condition.setReason(condition.getType()); - condition.setMessage("CancelledPublish"); - condition.setLastTransitionTime(Instant.now()); - status.getConditionsOrDefault().addAndEvictFIFO(condition); - - status.setPhase(Post.PostPhase.DRAFT.name()); - if (!oldPost.equals(post)) { - client.update(post); - } - return true; - }) - .orElse(false); - } - - private void publishFailed(String name, Throwable error) { - Assert.notNull(name, "Name must not be null"); - Assert.notNull(error, "Error must not be null"); - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - - Post.PostStatus status = post.getStatusOrDefault(); - Post.PostPhase phase = Post.PostPhase.FAILED; - status.setPhase(phase.name()); - - final ConditionList conditions = status.getConditionsOrDefault(); + private void publishPost(Post post, Set events) { + var expectReleaseSnapshot = post.getSpec().getReleaseSnapshot(); + if (StringUtils.isBlank(expectReleaseSnapshot)) { + // Do nothing if release snapshot is not set + return; + } + var annotations = post.getMetadata().getAnnotations(); + var lastReleaseSnapshot = annotations.get(Post.LAST_RELEASED_SNAPSHOT_ANNO); + if (post.isPublished() + && Objects.equals(expectReleaseSnapshot, lastReleaseSnapshot)) { + // If the release snapshot is not change + return; + } + var status = post.getStatus(); + // validate the release snapshot + var snapshot = client.fetch(Snapshot.class, expectReleaseSnapshot); + if (snapshot.isEmpty()) { Condition condition = Condition.builder() - .type(phase.name()) - .reason("PublishFailed") - .message(error.getMessage()) + .type(PostPhase.FAILED.name()) + .reason("SnapshotNotFound") + .message( + String.format("Snapshot [%s] not found for publish", expectReleaseSnapshot)) .status(ConditionStatus.FALSE) .lastTransitionTime(Instant.now()) .build(); - conditions.addAndEvictFIFO(condition); - - post.setStatus(status); - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void reconcileMetadata(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); - Post.PostSpec spec = post.getSpec(); - - // handle logic delete - Map labels = MetadataUtil.nullSafeLabels(post); - if (Objects.equals(spec.getDeleted(), true)) { - labels.put(Post.DELETED_LABEL, Boolean.TRUE.toString()); - } else { - labels.put(Post.DELETED_LABEL, Boolean.FALSE.toString()); - } - - fireVisibleChangedEventIfChanged(post); - labels.put(Post.VISIBLE_LABEL, - Objects.requireNonNullElse(spec.getVisible(), Post.VisibleEnum.PUBLIC).name()); - - labels.put(Post.OWNER_LABEL, spec.getOwner()); - Instant publishTime = post.getSpec().getPublishTime(); - if (publishTime != null) { - labels.put(Post.ARCHIVE_YEAR_LABEL, HaloUtils.getYearText(publishTime)); - labels.put(Post.ARCHIVE_MONTH_LABEL, HaloUtils.getMonthText(publishTime)); - labels.put(Post.ARCHIVE_DAY_LABEL, HaloUtils.getDayText(publishTime)); - } - if (!labels.containsKey(Post.PUBLISHED_LABEL)) { - labels.put(Post.PUBLISHED_LABEL, Boolean.FALSE.toString()); - } - - Map annotations = MetadataUtil.nullSafeAnnotations(post); - String newPattern = postPermalinkPolicy.pattern(); - annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern); - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void fireVisibleChangedEventIfChanged(Post post) { + status.getConditionsOrDefault().addAndEvictFIFO(condition); + status.setPhase(PostPhase.FAILED.name()); + return; + } + annotations.put(Post.LAST_RELEASED_SNAPSHOT_ANNO, expectReleaseSnapshot); + status.setPhase(PostPhase.PUBLISHED.toString()); + var condition = Condition.builder() + .type(PostPhase.PUBLISHED.name()) + .reason("Published") + .message("Post published successfully.") + .lastTransitionTime(Instant.now()) + .status(ConditionStatus.TRUE) + .build(); + status.getConditionsOrDefault().addAndEvictFIFO(condition); var labels = post.getMetadata().getLabels(); - if (labels == null) { + labels.put(Post.PUBLISHED_LABEL, Boolean.TRUE.toString()); + if (post.getSpec().getPublishTime() == null) { + // TODO Set the field in creation hook in the future. + post.getSpec().setPublishTime(Instant.now()); + } + status.setLastModifyTime(snapshot.get().getSpec().getLastModifyTime()); + events.add(new PostPublishedEvent(this, post.getMetadata().getName())); + } + + void unPublishPost(Post post, Set events) { + if (!post.isPublished()) { return; } - var name = post.getMetadata().getName(); - var oldVisibleStr = labels.get(Post.VISIBLE_LABEL); - if (oldVisibleStr != null) { - var oldVisible = Post.VisibleEnum.valueOf(oldVisibleStr); - var expectVisible = post.getSpec().getVisible(); - if (!Objects.equals(oldVisible, expectVisible)) { - eventPublisher.publishEvent( - new PostVisibleChangedEvent(name, oldVisible, expectVisible)); - } - } - } + var labels = post.getMetadata().getLabels(); + labels.put(Post.PUBLISHED_LABEL, Boolean.FALSE.toString()); + var status = post.getStatus(); - private void reconcileStatus(String name) { - client.fetch(Post.class, name).ifPresent(post -> { - final Post oldPost = JsonUtils.deepCopy(post); + var condition = new Condition(); + condition.setType("CancelledPublish"); + condition.setStatus(ConditionStatus.TRUE); + condition.setReason(condition.getType()); + condition.setMessage("CancelledPublish"); + condition.setLastTransitionTime(Instant.now()); + status.getConditionsOrDefault().addAndEvictFIFO(condition); - post.getStatusOrDefault() - .setPermalink(postPermalinkPolicy.permalink(post)); + status.setPhase(PostPhase.DRAFT.toString()); - Post.PostStatus status = post.getStatusOrDefault(); - if (status.getPhase() == null) { - status.setPhase(Post.PostPhase.DRAFT.name()); - } - Post.PostSpec spec = post.getSpec(); - // handle excerpt - Post.Excerpt excerpt = spec.getExcerpt(); - if (excerpt == null) { - excerpt = new Post.Excerpt(); - excerpt.setAutoGenerate(true); - spec.setExcerpt(excerpt); - } - if (excerpt.getAutoGenerate()) { - postService.getContent(spec.getReleaseSnapshot(), spec.getBaseSnapshot()) - .blockOptional() - .ifPresent(content -> { - String contentRevised = content.getContent(); - status.setExcerpt(getExcerpt(contentRevised)); - }); - } else { - status.setExcerpt(excerpt.getRaw()); - } - - Ref ref = Ref.of(post); - // handle contributors - String headSnapshot = post.getSpec().getHeadSnapshot(); - List contributors = client.list(Snapshot.class, - snapshot -> ref.equals(snapshot.getSpec().getSubjectRef()), null) - .stream() - .peek(snapshot -> { - snapshot.getSpec().setContentPatch(StringUtils.EMPTY); - snapshot.getSpec().setRawPatch(StringUtils.EMPTY); - }) - .map(snapshot -> { - Set usernames = snapshot.getSpec().getContributors(); - return Objects.requireNonNullElseGet(usernames, - () -> new HashSet()); - }) - .flatMap(Set::stream) - .distinct() - .sorted() - .toList(); - status.setContributors(contributors); - - // update in progress status - status.setInProgress( - !StringUtils.equals(headSnapshot, post.getSpec().getReleaseSnapshot())); - - if (post.isPublished() && status.getLastModifyTime() == null) { - client.fetch(Snapshot.class, post.getSpec().getReleaseSnapshot()) - .ifPresent(releasedSnapshot -> - status.setLastModifyTime(releasedSnapshot.getSpec().getLastModifyTime())); - } - - if (!oldPost.equals(post)) { - client.update(post); - } - }); - } - - private void addFinalizerIfNecessary(Post oldPost) { - Set finalizers = oldPost.getMetadata().getFinalizers(); - if (finalizers != null && finalizers.contains(FINALIZER_NAME)) { - return; - } - client.fetch(Post.class, oldPost.getMetadata().getName()) - .ifPresent(post -> { - Set newFinalizers = post.getMetadata().getFinalizers(); - if (newFinalizers == null) { - newFinalizers = new HashSet<>(); - post.getMetadata().setFinalizers(newFinalizers); - } - newFinalizers.add(FINALIZER_NAME); - client.update(post); - }); - } - - private void cleanUpResourcesAndRemoveFinalizer(String postName) { - client.fetch(Post.class, postName).ifPresent(post -> { - cleanUpResources(post); - if (post.getMetadata().getFinalizers() != null) { - post.getMetadata().getFinalizers().remove(FINALIZER_NAME); - } - client.update(post); - }); + events.add(new PostUnpublishedEvent(this, post.getMetadata().getName())); } private void cleanUpResources(Post post) { @@ -377,7 +285,7 @@ public class PostReconciler implements Reconciler { .forEach(client::delete); // clean up comments - client.list(Comment.class, comment -> comment.getSpec().getSubjectRef().equals(ref), + client.list(Comment.class, comment -> ref.equals(comment.getSpec().getSubjectRef()), null) .forEach(client::delete); diff --git a/application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java b/application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java similarity index 62% rename from application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java rename to application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java index 6d5e3bd3f..a17d8677c 100644 --- a/application/src/main/java/run/halo/app/event/post/PostRecycledEvent.java +++ b/application/src/main/java/run/halo/app/event/post/PostUpdatedEvent.java @@ -2,15 +2,16 @@ package run.halo.app.event.post; import org.springframework.context.ApplicationEvent; -public class PostRecycledEvent extends ApplicationEvent implements PostEvent { +public class PostUpdatedEvent extends ApplicationEvent implements PostEvent { private final String postName; - public PostRecycledEvent(Object source, String postName) { + public PostUpdatedEvent(Object source, String postName) { super(source); this.postName = postName; } + @Override public String getName() { return postName; } diff --git a/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java b/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java index 4b1355cd1..6e2d1c2fb 100644 --- a/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java +++ b/application/src/main/java/run/halo/app/event/post/PostVisibleChangedEvent.java @@ -1,6 +1,7 @@ package run.halo.app.event.post; import lombok.Data; +import org.springframework.lang.Nullable; import run.halo.app.core.extension.content.Post; @Data @@ -8,6 +9,7 @@ public class PostVisibleChangedEvent implements PostEvent { private final String postName; + @Nullable private final Post.VisibleEnum oldVisible; private final Post.VisibleEnum newVisible; diff --git a/application/src/main/java/run/halo/app/extension/ExtensionUtil.java b/application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java similarity index 93% rename from application/src/main/java/run/halo/app/extension/ExtensionUtil.java rename to application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java index 93e35b696..0f73a63b0 100644 --- a/application/src/main/java/run/halo/app/extension/ExtensionUtil.java +++ b/application/src/main/java/run/halo/app/extension/ExtensionStoreUtil.java @@ -7,9 +7,9 @@ import org.springframework.util.StringUtils; * * @author johnniang */ -public final class ExtensionUtil { +public final class ExtensionStoreUtil { - private ExtensionUtil() { + private ExtensionStoreUtil() { } /** diff --git a/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java b/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java index 0cfa1be10..eb9107db2 100644 --- a/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java +++ b/application/src/main/java/run/halo/app/extension/JSONExtensionConverter.java @@ -2,6 +2,7 @@ package run.halo.app.extension; import static org.openapi4j.core.validation.ValidationSeverity.ERROR; import static org.springframework.util.StringUtils.arrayToCommaDelimitedString; +import static run.halo.app.extension.ExtensionStoreUtil.buildStoreName; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -65,7 +66,7 @@ public class JSONExtensionConverter implements ExtensionConverter { } var version = extension.getMetadata().getVersion(); - var storeName = ExtensionUtil.buildStoreName(scheme, extension.getMetadata().getName()); + var storeName = buildStoreName(scheme, extension.getMetadata().getName()); var data = objectMapper.writeValueAsBytes(extensionJsonNode); return new ExtensionStore(storeName, data, version); } catch (IOException e) { diff --git a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java index 7466e958b..ad1a16497 100644 --- a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java +++ b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java @@ -46,7 +46,7 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { public Flux list(Class type, Predicate predicate, Comparator comparator) { var scheme = schemeManager.get(type); - var prefix = ExtensionUtil.buildStoreNamePrefix(scheme); + var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme); return client.listByNamePrefix(prefix) .map(extensionStore -> converter.convertFrom(type, extensionStore)) @@ -75,14 +75,14 @@ public class ReactiveExtensionClientImpl implements ReactiveExtensionClient { @Override public Mono fetch(Class type, String name) { - var storeName = ExtensionUtil.buildStoreName(schemeManager.get(type), name); + var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(type), name); return client.fetchByName(storeName) .map(extensionStore -> converter.convertFrom(type, extensionStore)); } @Override public Mono fetch(GroupVersionKind gvk, String name) { - var storeName = ExtensionUtil.buildStoreName(schemeManager.get(gvk), name); + var storeName = ExtensionStoreUtil.buildStoreName(schemeManager.get(gvk), name); return client.fetchByName(storeName) .map(extensionStore -> converter.convertFrom(Unstructured.class, extensionStore)); } diff --git a/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java b/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java index 5e8cf2a84..9c52f6039 100644 --- a/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java +++ b/application/src/main/java/run/halo/app/search/IndicesServiceImpl.java @@ -3,6 +3,7 @@ package run.halo.app.search; import org.springframework.stereotype.Service; import reactor.core.Exceptions; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; import run.halo.app.core.extension.content.Post; import run.halo.app.plugin.extensionpoint.ExtensionGetter; import run.halo.app.search.post.PostSearchService; @@ -23,28 +24,42 @@ public class IndicesServiceImpl implements IndicesService { @Override public Mono rebuildPostIndices() { return extensionGetter.getEnabledExtension(PostSearchService.class) - .flatMap(searchService -> postFinder.listAll() - .filter(post -> Post.isPublished(post.getMetadata())) - .flatMap(listedPostVo -> { - PostVo postVo = PostVo.from(listedPostVo); - return postFinder.content(postVo.getMetadata().getName()) - .map(content -> { - postVo.setContent(content); - return postVo; - }) - .defaultIfEmpty(postVo); - }) - .map(PostDocUtils::from) - .limitRate(100) - .buffer(100) - .doOnNext(postDocs -> { - try { - searchService.addDocuments(postDocs); - } catch (Exception e) { - throw Exceptions.propagate(e); - } - }) - .then() - ); + .flatMap(searchService -> Mono.fromRunnable( + () -> { + try { + // remove all docs before rebuilding + searchService.removeAllDocuments(); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }) + .then(rebuildPostIndices(searchService)) + ) + .subscribeOn(Schedulers.boundedElastic()); + } + + private Mono rebuildPostIndices(PostSearchService searchService) { + return postFinder.listAll() + .filter(post -> Post.isPublished(post.getMetadata())) + .flatMap(listedPostVo -> { + PostVo postVo = PostVo.from(listedPostVo); + return postFinder.content(postVo.getMetadata().getName()) + .map(content -> { + postVo.setContent(content); + return postVo; + }) + .defaultIfEmpty(postVo); + }) + .map(PostDocUtils::from) + .limitRate(100) + .buffer(100) + .doOnNext(postDocs -> { + try { + searchService.addDocuments(postDocs); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + }) + .then(); } } diff --git a/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java b/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java index 423d77a8d..274c2c75c 100644 --- a/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java +++ b/application/src/main/java/run/halo/app/search/post/LucenePostSearchService.java @@ -126,6 +126,15 @@ public class LucenePostSearchService implements PostSearchService, DisposableBea } } + @Override + public void removeAllDocuments() throws Exception { + var writeConfig = new IndexWriterConfig(analyzer); + writeConfig.setOpenMode(APPEND); + try (var writer = new IndexWriter(postIndexDir, writeConfig)) { + writer.deleteAll(); + } + } + @Override public void destroy() throws Exception { analyzer.close(); @@ -145,11 +154,19 @@ public class LucenePostSearchService implements PostSearchService, DisposableBea doc.add(new StringField("name", post.name(), YES)); doc.add(new StoredField("title", post.title())); - var content = Jsoup.clean(stripToEmpty(post.excerpt()) + stripToEmpty(post.content()), - Safelist.none()); + var cleanExcerpt = Jsoup.clean(stripToEmpty(post.excerpt()), Safelist.none()); + var cleanContent = Jsoup.clean(stripToEmpty(post.content()), Safelist.none()); + + var contentBuilder = new StringBuilder(cleanExcerpt); + if (!contentBuilder.isEmpty()) { + contentBuilder.append(' '); + } + contentBuilder.append(cleanContent); + + var content = contentBuilder.toString(); doc.add(new StoredField("content", content)); - doc.add(new TextField("searchable", post.title() + content, NO)); + doc.add(new TextField("searchable", post.title() + " " + content, NO)); long publishTimestamp = post.publishTimestamp().toEpochMilli(); doc.add(new LongPoint("publishTimestamp", publishTimestamp)); diff --git a/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java b/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java index 320310e97..b96f12287 100644 --- a/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java +++ b/application/src/main/java/run/halo/app/search/post/PostEventReconciler.java @@ -13,8 +13,8 @@ import org.springframework.stereotype.Component; import reactor.core.Exceptions; import run.halo.app.event.post.PostEvent; import run.halo.app.event.post.PostPublishedEvent; -import run.halo.app.event.post.PostRecycledEvent; import run.halo.app.event.post.PostUnpublishedEvent; +import run.halo.app.event.post.PostUpdatedEvent; import run.halo.app.event.post.PostVisibleChangedEvent; import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.ControllerBuilder; @@ -52,11 +52,10 @@ public class PostEventReconciler implements Reconciler, SmartLifecycl @Override public Result reconcile(PostEvent postEvent) { - if (postEvent instanceof PostPublishedEvent) { + if (postEvent instanceof PostPublishedEvent || postEvent instanceof PostUpdatedEvent) { addPostDoc(postEvent.getName()); } - if (postEvent instanceof PostUnpublishedEvent - || postEvent instanceof PostRecycledEvent) { + if (postEvent instanceof PostUnpublishedEvent) { deletePostDoc(postEvent.getName()); } if (postEvent instanceof PostVisibleChangedEvent visibleChangedEvent) { @@ -81,29 +80,13 @@ public class PostEventReconciler implements Reconciler, SmartLifecycl ); } - @EventListener(PostPublishedEvent.class) - public void handlePostPublished(PostPublishedEvent publishedEvent) { - postEventQueue.addImmediately(publishedEvent); - } - - @EventListener(PostUnpublishedEvent.class) - public void handlePostUnpublished(PostUnpublishedEvent unpublishedEvent) { - postEventQueue.addImmediately(unpublishedEvent); - } - - @EventListener(PostRecycledEvent.class) - public void handlePostRecycled(PostRecycledEvent recycledEvent) { - postEventQueue.addImmediately(recycledEvent); - } - - @EventListener(PostVisibleChangedEvent.class) - public void handlePostVisibleChanged(PostVisibleChangedEvent event) { + @EventListener(PostEvent.class) + public void handlePostEvent(PostEvent event) { postEventQueue.addImmediately(event); } void addPostDoc(String postName) { postFinder.getByName(postName) - .filter(postVo -> PUBLIC.equals(postVo.getSpec().getVisible())) .map(PostDocUtils::from) .flatMap(postDoc -> extensionGetter.getEnabledExtension(PostSearchService.class) .doOnNext(searchService -> { diff --git a/application/src/main/java/run/halo/app/theme/finders/PostFinder.java b/application/src/main/java/run/halo/app/theme/finders/PostFinder.java index be2ba02a0..834797610 100644 --- a/application/src/main/java/run/halo/app/theme/finders/PostFinder.java +++ b/application/src/main/java/run/halo/app/theme/finders/PostFinder.java @@ -19,6 +19,14 @@ import run.halo.app.theme.finders.vo.PostVo; */ public interface PostFinder { + /** + * Gets post detail by name. + *

+ * We ensure the post is public, non-deleted and published. + * + * @param postName is post name + * @return post detail + */ Mono getByName(String postName); Mono content(String postName); diff --git a/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java b/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java index 9cd06b045..adb9fbc77 100644 --- a/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java +++ b/application/src/test/java/run/halo/app/core/extension/reconciler/PostReconcilerTest.java @@ -77,7 +77,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(3)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); verify(postPermalinkPolicy, times(1)).permalink(any()); @@ -118,7 +118,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(4)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getExcerpt()).isEqualTo("hello world"); } @@ -154,7 +154,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(4)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getLastModifyTime()).isEqualTo(lastModifyTime); verify(eventPublisher).publishEvent(any(PostPublishedEvent.class)); @@ -183,7 +183,7 @@ class PostReconcilerTest { ArgumentCaptor captor = ArgumentCaptor.forClass(Post.class); postReconciler.reconcile(new Reconciler.Request(name)); - verify(client, times(3)).update(captor.capture()); + verify(client, times(1)).update(captor.capture()); Post value = captor.getValue(); assertThat(value.getStatus().getLastModifyTime()).isNull(); } diff --git a/application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java b/application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java similarity index 77% rename from application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java rename to application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java index 8c71678b6..1f70b3f24 100644 --- a/application/src/test/java/run/halo/app/extension/ExtensionUtilTest.java +++ b/application/src/test/java/run/halo/app/extension/ExtensionStoreUtilTest.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class ExtensionUtilTest { +class ExtensionStoreUtilTest { Scheme scheme; @@ -28,19 +28,19 @@ class ExtensionUtilTest { @Test void buildStoreNamePrefix() { - var prefix = ExtensionUtil.buildStoreNamePrefix(scheme); + var prefix = ExtensionStoreUtil.buildStoreNamePrefix(scheme); assertEquals("/registry/fake.halo.run/fakes", prefix); - prefix = ExtensionUtil.buildStoreNamePrefix(grouplessScheme); + prefix = ExtensionStoreUtil.buildStoreNamePrefix(grouplessScheme); assertEquals("/registry/fakes", prefix); } @Test void buildStoreName() { - var storeName = ExtensionUtil.buildStoreName(scheme, "fake-name"); + var storeName = ExtensionStoreUtil.buildStoreName(scheme, "fake-name"); assertEquals("/registry/fake.halo.run/fakes/fake-name", storeName); - storeName = ExtensionUtil.buildStoreName(grouplessScheme, "fake-name"); + storeName = ExtensionStoreUtil.buildStoreName(grouplessScheme, "fake-name"); assertEquals("/registry/fakes/fake-name", storeName); }