diff --git a/src/main/java/run/halo/app/core/extension/reconciler/ThemeReconciler.java b/src/main/java/run/halo/app/core/extension/reconciler/ThemeReconciler.java index 4812a38b6..412e08c56 100644 --- a/src/main/java/run/halo/app/core/extension/reconciler/ThemeReconciler.java +++ b/src/main/java/run/halo/app/core/extension/reconciler/ThemeReconciler.java @@ -8,6 +8,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import org.apache.commons.lang3.StringUtils; +import org.springframework.retry.support.RetryTemplate; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import org.springframework.util.FileSystemUtils; @@ -41,6 +42,12 @@ public class ThemeReconciler implements Reconciler { private final ExtensionClient client; private final ThemePathPolicy themePathPolicy; + private final RetryTemplate retryTemplate = RetryTemplate.builder() + .maxAttempts(20) + .fixedBackoff(300) + .retryOn(IllegalStateException.class) + .build(); + public ThemeReconciler(ExtensionClient client, HaloProperties haloProperties) { this.client = client; themePathPolicy = new ThemePathPolicy(haloProperties.getWorkDir()); @@ -157,20 +164,39 @@ public class ThemeReconciler implements Reconciler { if (StringUtils.isNotBlank(settingName)) { client.fetch(Setting.class, settingName) .ifPresent(client::delete); + retryTemplate.execute(callback -> { + client.fetch(Setting.class, settingName).ifPresent(setting -> { + throw new IllegalStateException("Waiting for setting to be deleted."); + }); + return null; + }); } // delete annotation setting deleteAnnotationSettings(theme.getMetadata().getName()); } private void deleteAnnotationSettings(String themeName) { - List result = client.list(AnnotationSetting.class, annotationSetting -> { - Map labels = ExtensionUtil.nullSafeLabels(annotationSetting); - return themeName.equals(labels.get(Theme.THEME_NAME_LABEL)); - }, null); + List result = listAnnotationSettingsByThemeName(themeName); for (AnnotationSetting annotationSetting : result) { client.delete(annotationSetting); } + + retryTemplate.execute(callback -> { + List annotationSettings = + listAnnotationSettingsByThemeName(themeName); + if (annotationSettings.isEmpty()) { + return null; + } + throw new IllegalStateException("Waiting for annotation settings to be deleted."); + }); + } + + private List listAnnotationSettingsByThemeName(String themeName) { + return client.list(AnnotationSetting.class, annotationSetting -> { + Map labels = ExtensionUtil.nullSafeLabels(annotationSetting); + return themeName.equals(labels.get(Theme.THEME_NAME_LABEL)); + }, null); } private void deleteThemeFiles(Theme theme) { diff --git a/src/test/java/run/halo/app/core/extension/reconciler/ThemeReconcilerTest.java b/src/test/java/run/halo/app/core/extension/reconciler/ThemeReconcilerTest.java index b2988e6a8..de4772d72 100644 --- a/src/test/java/run/halo/app/core/extension/reconciler/ThemeReconcilerTest.java +++ b/src/test/java/run/halo/app/core/extension/reconciler/ThemeReconcilerTest.java @@ -1,6 +1,7 @@ package run.halo.app.core.extension.reconciler; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; @@ -12,6 +13,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.List; import java.util.Map; import java.util.Optional; import org.json.JSONException; @@ -23,7 +25,9 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.stubbing.Answer; import org.skyscreamer.jsonassert.JSONAssert; +import org.springframework.retry.RetryException; import org.springframework.util.FileSystemUtils; import org.springframework.util.ResourceUtils; import run.halo.app.core.extension.AnnotationSetting; @@ -32,6 +36,7 @@ import run.halo.app.core.extension.Theme; import run.halo.app.extension.ConfigMap; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.Metadata; +import run.halo.app.extension.MetadataOperator; import run.halo.app.extension.controller.Reconciler; import run.halo.app.infra.properties.HaloProperties; import run.halo.app.infra.utils.JsonUtils; @@ -105,14 +110,100 @@ class ThemeReconcilerTest { themeReconciler.reconcile(new Reconciler.Request(metadata.getName())); verify(extensionClient, times(2)).fetch(eq(Theme.class), eq(metadata.getName())); - verify(extensionClient, times(1)).fetch(eq(Setting.class), eq(themeSpec.getSettingName())); + verify(extensionClient, times(2)).fetch(eq(Setting.class), eq(themeSpec.getSettingName())); - verify(extensionClient, times(1)).list(eq(AnnotationSetting.class), any(), any()); + verify(extensionClient, times(2)).list(eq(AnnotationSetting.class), any(), any()); assertThat(Files.exists(testWorkDir)).isTrue(); assertThat(Files.exists(defaultThemePath)).isFalse(); } + @Test + void reconcileDeleteRetry() { + Theme theme = fakeTheme(); + final MetadataOperator metadata = theme.getMetadata(); + + Path testWorkDir = tempDirectory.resolve("reconcile-delete"); + when(haloProperties.getWorkDir()).thenReturn(testWorkDir); + + final ThemeReconciler themeReconciler = + new ThemeReconciler(extensionClient, haloProperties); + + final int[] retryFlags = {0, 0}; + when(extensionClient.fetch(eq(Setting.class), eq("theme-test-setting"))) + .thenAnswer((Answer>) invocation -> { + retryFlags[0]++; + // retry 2 times + if (retryFlags[0] < 3) { + return Optional.of(new Setting()); + } + return Optional.empty(); + }); + + when(extensionClient.list(eq(AnnotationSetting.class), any(), eq(null))) + .thenAnswer((Answer>) invocation -> { + retryFlags[1]++; + // retry 2 times + if (retryFlags[1] < 3) { + return List.of(new AnnotationSetting()); + } + return List.of(); + }); + + themeReconciler.reconcile(new Reconciler.Request(metadata.getName())); + + String settingName = theme.getSpec().getSettingName(); + verify(extensionClient, times(2)).fetch(eq(Theme.class), eq(metadata.getName())); + verify(extensionClient, times(3)).fetch(eq(Setting.class), eq(settingName)); + verify(extensionClient, times(3)).list(eq(AnnotationSetting.class), any(), eq(null)); + } + + @Test + void reconcileDeleteRetryWhenThrowException() { + Theme theme = fakeTheme(); + + Path testWorkDir = tempDirectory.resolve("reconcile-delete"); + when(haloProperties.getWorkDir()).thenReturn(testWorkDir); + + final ThemeReconciler themeReconciler = + new ThemeReconciler(extensionClient, haloProperties); + + final int[] retryFlags = {0}; + when(extensionClient.fetch(eq(Setting.class), eq("theme-test-setting"))) + .thenAnswer((Answer>) invocation -> { + retryFlags[0]++; + // retry 2 times + if (retryFlags[0] < 2) { + return Optional.of(new Setting()); + } + throw new RetryException("retry exception."); + }); + + String settingName = theme.getSpec().getSettingName(); + assertThatThrownBy( + () -> themeReconciler.reconcile(new Reconciler.Request(theme.getMetadata().getName()))) + .isInstanceOf(RetryException.class) + .hasMessage("retry exception."); + + verify(extensionClient, times(2)).fetch(eq(Setting.class), eq(settingName)); + } + + private Theme fakeTheme() { + Theme theme = new Theme(); + Metadata metadata = new Metadata(); + metadata.setName("theme-test"); + metadata.setDeletionTimestamp(Instant.now()); + theme.setMetadata(metadata); + theme.setKind(Theme.KIND); + theme.setApiVersion("theme.halo.run/v1alpha1"); + Theme.ThemeSpec themeSpec = new Theme.ThemeSpec(); + themeSpec.setSettingName("theme-test-setting"); + theme.setSpec(themeSpec); + when(extensionClient.fetch(eq(Theme.class), eq(metadata.getName()))) + .thenReturn(Optional.of(theme)); + return theme; + } + @Test void themeSettingDefaultValue() throws IOException, JSONException { Path testWorkDir = tempDirectory.resolve("reconcile-setting-value"); @@ -169,10 +260,10 @@ class ThemeReconcilerTest { ConfigMap defaultValueConfigMap = configMapCaptor.getValue(); Map data = defaultValueConfigMap.getData(); JSONAssert.assertEquals(""" - { - "sns": "{\\"email\\":\\"example@exmple.com\\"}" - } - """, + { + "sns": "{\\"email\\":\\"example@exmple.com\\"}" + } + """, JsonUtils.objectToJson(data), true); }