From 39545a1e4c1fdce312fb47b82969eea3b5c6493c Mon Sep 17 00:00:00 2001 From: guqing <38999863+guqing@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:28:10 +0800 Subject: [PATCH] refactor: optimize request headers when generating thumbnails from URI (#6628) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### What type of PR is this? /kind improvement /area core /milestone 2.20.x #### What this PR does / why we need it: 优化根据 URI 生成缩略图时的请求头 1. 由于之前 attachment 的 permalink 是 `URI.toString` 会导致根据 permalink 索引查询附件可能由于编码问题无法查询到导致生成缩略图只能根据 URI 生成 2. 可能配置了 nginx 判断请求头不允许脚本请求如导致根据 URI 访问图片无法访问导致无法生成，如 ``` if ($http_user agent ~*(python curlljava wget go-http-client httpclient okhttp)){ } ``` Fixes #6627 #### Does this PR introduce a user-facing change? ```release-note None ``` --- .../halo/app/core/attachment/ThumbnailGenerator.java | 7 ++++++- .../attachment/reconciler/AttachmentReconciler.java | 2 +- .../reconciler/LocalThumbnailsReconciler.java | 4 ++++ .../app/core/attachment/ThumbnailGeneratorTest.java | 11 ++++++++--- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/run/halo/app/core/attachment/ThumbnailGenerator.java b/application/src/main/java/run/halo/app/core/attachment/ThumbnailGenerator.java index 59f543e75..ad822f05c 100644 --- a/application/src/main/java/run/halo/app/core/attachment/ThumbnailGenerator.java +++ b/application/src/main/java/run/halo/app/core/attachment/ThumbnailGenerator.java @@ -6,6 +6,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -172,7 +173,11 @@ public class ThumbnailGenerator { File tempFile = File.createTempFile("halo-image-thumb-", ".tmp"); long totalBytesDownloaded = 0; var tempFilePath = tempFile.toPath(); - try (InputStream inputStream = url.openStream(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)" + + " Chrome/92.0.4515.131 Safari/537.36"); + try (InputStream inputStream = connection.getInputStream(); FileOutputStream outputStream = new FileOutputStream(tempFile)) { byte[] buffer = new byte[4096]; diff --git a/application/src/main/java/run/halo/app/core/attachment/reconciler/AttachmentReconciler.java b/application/src/main/java/run/halo/app/core/attachment/reconciler/AttachmentReconciler.java index 0441390cc..fa5b722ba 100644 --- a/application/src/main/java/run/halo/app/core/attachment/reconciler/AttachmentReconciler.java +++ b/application/src/main/java/run/halo/app/core/attachment/reconciler/AttachmentReconciler.java @@ -58,7 +58,7 @@ public class AttachmentReconciler implements Reconciler { var annotations = attachment.getMetadata().getAnnotations(); if (annotations != null) { attachmentService.getPermalink(attachment) - .map(URI::toString) + .map(URI::toASCIIString) .switchIfEmpty(Mono.fromSupplier(() -> { // Only for back-compatibility return annotations.get(Constant.EXTERNAL_LINK_ANNO_KEY); diff --git a/application/src/main/java/run/halo/app/core/attachment/reconciler/LocalThumbnailsReconciler.java b/application/src/main/java/run/halo/app/core/attachment/reconciler/LocalThumbnailsReconciler.java index 1f1eb7b38..ce2d90f82 100644 --- a/application/src/main/java/run/halo/app/core/attachment/reconciler/LocalThumbnailsReconciler.java +++ b/application/src/main/java/run/halo/app/core/attachment/reconciler/LocalThumbnailsReconciler.java @@ -27,6 +27,7 @@ import run.halo.app.core.extension.attachment.Attachment; import run.halo.app.core.extension.attachment.Constant; import run.halo.app.core.extension.attachment.LocalThumbnail; import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.ExtensionUtil; import run.halo.app.extension.ListOptions; import run.halo.app.extension.PageRequestImpl; import run.halo.app.extension.controller.Controller; @@ -47,6 +48,9 @@ public class LocalThumbnailsReconciler implements Reconciler public Result reconcile(Request request) { client.fetch(LocalThumbnail.class, request.name()) .ifPresent(thumbnail -> { + if (ExtensionUtil.isDeleted(thumbnail)) { + return; + } if (shouldGenerate(thumbnail)) { requestGenerateThumbnail(thumbnail); nullSafeAnnotations(thumbnail).remove(REQUEST_TO_GENERATE_ANNO); diff --git a/application/src/test/java/run/halo/app/core/attachment/ThumbnailGeneratorTest.java b/application/src/test/java/run/halo/app/core/attachment/ThumbnailGeneratorTest.java index e50da39ba..a87dca886 100644 --- a/application/src/test/java/run/halo/app/core/attachment/ThumbnailGeneratorTest.java +++ b/application/src/test/java/run/halo/app/core/attachment/ThumbnailGeneratorTest.java @@ -4,11 +4,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -57,7 +59,9 @@ class ThumbnailGeneratorTest { String mockImageData = "fakeImageData"; InputStream mockInputStream = new ByteArrayInputStream(mockImageData.getBytes()); - doAnswer(invocation -> mockInputStream).when(spyImageUrl).openStream(); + var urlConnection = mock(HttpURLConnection.class); + doAnswer(invocation -> urlConnection).when(spyImageUrl).openConnection(); + doReturn(mockInputStream).when(urlConnection).getInputStream(); var path = imageDownloader.downloadFileInternal(spyImageUrl); assertThat(path).isNotNull(); @@ -81,8 +85,9 @@ class ThumbnailGeneratorTest { var fileSizeByte = ThumbnailGenerator.MAX_FILE_SIZE + 10; byte[] largeImageData = new byte[fileSizeByte]; InputStream mockInputStream = new ByteArrayInputStream(largeImageData); - - doReturn(mockInputStream).when(spyImageUrl).openStream(); + var urlConnection = mock(HttpURLConnection.class); + doAnswer(invocation -> urlConnection).when(spyImageUrl).openConnection(); + doReturn(mockInputStream).when(urlConnection).getInputStream(); assertThatThrownBy(() -> imageDownloader.downloadFileInternal(spyImageUrl)) .isInstanceOf(IOException.class) .hasMessageContaining("File size exceeds the limit");