From 0b505a90506c4b6b5af37fddb4552f859a36d137 Mon Sep 17 00:00:00 2001 From: guqing <38999863+guqing@users.noreply.github.com> Date: Sun, 24 Nov 2024 23:50:22 +0800 Subject: [PATCH] refactor: add uniqueness check for local thumbnail original links and dimensions to avoid duplication (#7031) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind improvement /area core /milestone 2.20.x #### What this PR does / why we need it: 对本地缩略图的原图链接和尺寸增加唯一性检查避免重复 #### Does this PR introduce a user-facing change? ```release-note 对本地缩略图的原图链接和尺寸增加唯一性检查避免重复 ``` --- .../attachment/extension/LocalThumbnail.java | 10 +++++++++ .../impl/LocalThumbnailServiceImpl.java | 21 ++++++++++++------- .../run/halo/app/infra/SchemeInitializer.java | 8 +++++++ .../impl/LocalThumbnailServiceImplTest.java | 10 +++++---- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/run/halo/app/core/attachment/extension/LocalThumbnail.java b/application/src/main/java/run/halo/app/core/attachment/extension/LocalThumbnail.java index 97ee3974b..7cf66c1b1 100644 --- a/application/src/main/java/run/halo/app/core/attachment/extension/LocalThumbnail.java +++ b/application/src/main/java/run/halo/app/core/attachment/extension/LocalThumbnail.java @@ -19,6 +19,7 @@ import run.halo.app.extension.GVK; @GVK(group = "storage.halo.run", version = "v1alpha1", kind = "LocalThumbnail", plural = "localthumbnails", singular = "localthumbnail") public class LocalThumbnail extends AbstractExtension { + public static final String UNIQUE_IMAGE_AND_SIZE_INDEX = "uniqueImageAndSize"; public static final String REQUEST_TO_GENERATE_ANNO = "storage.halo.run/request-to-generate"; @Schema(requiredMode = REQUIRED) @@ -80,4 +81,13 @@ public class LocalThumbnail extends AbstractExtension { public enum Phase { PENDING, SUCCEEDED, FAILED } + + public static String uniqueImageAndSize(LocalThumbnail localThumbnail) { + return uniqueImageAndSize(localThumbnail.getSpec().getImageSignature(), + localThumbnail.getSpec().getSize()); + } + + public static String uniqueImageAndSize(String imageSignature, ThumbnailSize size) { + return imageSignature + "-" + size.name(); + } } diff --git a/application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java b/application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java index 5b25d4b01..3ed0ddf2b 100644 --- a/application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java +++ b/application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java @@ -64,8 +64,8 @@ public class LocalThumbnailServiceImpl implements LocalThumbnailService { .resolve(fileName); } - static String geImageFileName(URL imageUrl) { - var fileName = substringAfterLast(imageUrl.getPath(), "/"); + static String geImageFileName(URI imageUri) { + var fileName = substringAfterLast(imageUri.getPath(), "/"); fileName = defaultIfBlank(fileName, randomAlphanumeric(10)); return ThumbnailGenerator.sanitizeFileName(fileName); } @@ -113,11 +113,12 @@ public class LocalThumbnailServiceImpl implements LocalThumbnailService { private Mono fetchByImageHashAndSize(String imageSignature, ThumbnailSize size) { + var indexValue = LocalThumbnail.uniqueImageAndSize(imageSignature, size); return client.listBy(LocalThumbnail.class, ListOptions.builder() - .fieldQuery(equal("spec.imageSignature", imageSignature)) - .build(), PageRequestImpl.ofSize(ThumbnailSize.values().length)) + .fieldQuery(equal(LocalThumbnail.UNIQUE_IMAGE_AND_SIZE_INDEX, indexValue)) + .build(), PageRequestImpl.ofSize(ThumbnailSize.values().length) + ) .flatMapMany(result -> Flux.fromIterable(result.getItems())) - .filter(thumbnail -> thumbnail.getSpec().getSize().equals(size)) .next(); } @@ -158,9 +159,15 @@ public class LocalThumbnailServiceImpl implements LocalThumbnailService { public Mono create(URL imageUrl, ThumbnailSize size) { Assert.notNull(imageUrl, "Image URL must not be null."); Assert.notNull(size, "Thumbnail size must not be null."); - var year = getYear(); - var originalFileName = geImageFileName(imageUrl); var imageUri = URI.create(imageUrl.toString()); + var imageHash = signatureForImageUri(imageUri); + return fetchByImageHashAndSize(imageHash, size) + .switchIfEmpty(Mono.defer(() -> doCreate(imageUri, size))); + } + + private Mono doCreate(URI imageUri, ThumbnailSize size) { + var year = getYear(); + var originalFileName = geImageFileName(imageUri); return generateUniqueThumbFileName(originalFileName, year, size) .flatMap(thumbFileName -> { var filePath = diff --git a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java index 49ab3d0a6..c8dd622f0 100644 --- a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java +++ b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java @@ -612,6 +612,14 @@ public class SchemeInitializer implements ApplicationListener { + // make sure image and size are unique + indexSpec.add(new IndexSpec() + .setUnique(true) + .setName(LocalThumbnail.UNIQUE_IMAGE_AND_SIZE_INDEX) + .setIndexFunc(simpleAttribute(LocalThumbnail.class, + LocalThumbnail::uniqueImageAndSize) + ) + ); indexSpec.add(new IndexSpec() .setName("spec.imageSignature") .setIndexFunc(simpleAttribute(LocalThumbnail.class, diff --git a/application/src/test/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImplTest.java b/application/src/test/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImplTest.java index 54d3d2c43..6970e89ab 100644 --- a/application/src/test/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImplTest.java +++ b/application/src/test/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImplTest.java @@ -12,6 +12,7 @@ import static run.halo.app.core.attachment.ThumbnailSigner.generateSignature; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -20,6 +21,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.web.util.UriUtils; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; import run.halo.app.core.attachment.AttachmentRootGetter; @@ -61,14 +63,14 @@ class LocalThumbnailServiceImplTest { @Test void geImageFileNameTest() throws MalformedURLException { var fileName = - LocalThumbnailServiceImpl.geImageFileName(new URL("https://halo.run/example.jpg")); + LocalThumbnailServiceImpl.geImageFileName(URI.create("https://halo.run/example.jpg")); assertThat(fileName).isEqualTo("example.jpg"); - fileName = LocalThumbnailServiceImpl.geImageFileName(new URL("https://halo.run/")); + fileName = LocalThumbnailServiceImpl.geImageFileName(URI.create("https://halo.run/")); assertThat(fileName).isNotBlank(); - fileName = LocalThumbnailServiceImpl.geImageFileName( - new URL("https://halo.run/.1fasfg(*&^%$.jpg")); + var encoded = UriUtils.encode("https://halo.run/.1fasfg(*&^%$.jpg", StandardCharsets.UTF_8); + fileName = LocalThumbnailServiceImpl.geImageFileName(URI.create(encoded)); assertThat(fileName).isNotBlank(); }