mirror of https://github.com/halo-dev/halo
Generate JS and CSS bundle with fixed buffer size (#6573)
#### What type of PR is this? /kind improvement /area core /milestone 2.20.x #### What this PR does / why we need it: If we are running Halo instance in machine with small memory available, the JS/CSS bundle might not be accessible. This RP refactors generation of JS and CSS bundle with fixed buffer size rather than length of original resources. ```java 2024-09-02T15:01:27.667+08:00 WARN 62039 --- [boundedElastic-3] reactor.core.Exceptions : throwIfFatal detected a jvm fatal exception, which is thrown and logged below: java.lang.OutOfMemoryError: Java heap space at java.base/java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:64) ~[na:na] at java.base/java.nio.ByteBuffer.allocate(ByteBuffer.java:363) ~[na:na] at org.springframework.core.io.buffer.DefaultDataBuffer.allocate(DefaultDataBuffer.java:234) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DefaultDataBuffer.setCapacity(DefaultDataBuffer.java:196) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DefaultDataBuffer.ensureWritable(DefaultDataBuffer.java:228) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:296) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:289) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:43) ~[spring-core-6.1.12.jar:6.1.12] at run.halo.app.core.extension.service.impl.PluginServiceImpl.lambda$uglifyJsBundle$17(PluginServiceImpl.java:257) ~[classes/:na] at run.halo.app.core.extension.service.impl.PluginServiceImpl$$Lambda$4661/0x000000c80214e298.accept(Unknown Source) ~[na:na] at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onNext(FluxPeekFuseable.java:196) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.onNext(FluxUsing.java:353) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxGenerate$GenerateSubscription.next(FluxGenerate.java:178) ~[reactor-core-3.6.9.jar:3.6.9] at org.springframework.core.io.buffer.DataBufferUtils$ReadableByteChannelGenerator.accept(DataBufferUtils.java:1002) ~[spring-core-6.1.12.jar:6.1.12] at org.springframework.core.io.buffer.DataBufferUtils$ReadableByteChannelGenerator.accept(DataBufferUtils.java:974) ~[spring-core-6.1.12.jar:6.1.12] at reactor.core.publisher.FluxGenerate.lambda$new$1(FluxGenerate.java:58) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxGenerate$$Lambda$4155/0x000000c802069228.apply(Unknown Source) ~[na:na] at reactor.core.publisher.FluxGenerate$GenerateSubscription.slowPath(FluxGenerate.java:271) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxGenerate$GenerateSubscription.request(FluxGenerate.java:213) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.request(FluxUsing.java:320) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.request(FluxPeekFuseable.java:144) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:968) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onSubscribe(FluxPeekFuseable.java:178) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.onSubscribe(FluxUsing.java:347) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxGenerate.subscribe(FluxGenerate.java:85) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxUsing.subscribe(FluxUsing.java:102) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.Flux.subscribe(Flux.java:8848) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:430) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxHandleFuseable$HandleFuseableSubscriber.tryOnNext(FluxHandleFuseable.java:135) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxIterable$IterableSubscriptionConditional.slowPath(FluxIterable.java:664) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxIterable$IterableSubscriptionConditional.request(FluxIterable.java:623) ~[reactor-core-3.6.9.jar:3.6.9] at reactor.core.publisher.FluxHandleFuseable$HandleFuseableSubscriber.request(FluxHandleFuseable.java:260) ~[reactor-core-3.6.9.jar:3.6.9] 2024-09-02T15:01:27.681+08:00 DEBUG 62039 --- [boundedElastic-3] a.w.r.e.AbstractErrorWebExceptionHandler : [131a559b-102] Resolved [OutOfMemoryError: Java heap space] for HTTP GET /apis/api.console.halo.run/v1alpha1/plugins/-/bundle.js 2024-09-02T15:01:27.681+08:00 ERROR 62039 --- [boundedElastic-3] a.w.r.e.AbstractErrorWebExceptionHandler : [131a559b-102] 500 Server Error for HTTP GET "/apis/api.console.halo.run/v1alpha1/plugins/-/bundle.js?v=1725260408176" java.lang.OutOfMemoryError: Java heap space at java.base/java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:64) ~[na:na] ``` #### Does this PR introduce a user-facing change? ```release-note 优化在内存紧张时 Console 端无法加载插件资源的问题 ```pull/6679/head
parent
749c80cb96
commit
ded5b4135f
|
@ -40,6 +40,7 @@ import org.springframework.stereotype.Component;
|
|||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import org.springframework.util.FileSystemUtils;
|
||||
import org.springframework.util.StreamUtils;
|
||||
import org.springframework.web.server.ServerWebInputException;
|
||||
import reactor.core.Exceptions;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
@ -229,61 +230,73 @@ public class PluginServiceImpl implements PluginService, InitializingBean, Dispo
|
|||
@Override
|
||||
public Flux<DataBuffer> uglifyJsBundle() {
|
||||
var startedPlugins = List.copyOf(pluginManager.getStartedPlugins());
|
||||
String plugins = """
|
||||
this.enabledPlugins = [%s]
|
||||
""".formatted(startedPlugins.stream()
|
||||
.map(plugin -> """
|
||||
{
|
||||
"name": "%s",
|
||||
"version": "%s"
|
||||
var dataBufferFactory = DefaultDataBufferFactory.sharedInstance;
|
||||
var end = Mono.fromSupplier(
|
||||
() -> {
|
||||
var sb = new StringBuilder("this.enabledPlugins = [");
|
||||
var iterator = startedPlugins.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
var plugin = iterator.next();
|
||||
sb.append("""
|
||||
{"name":"%s","version":"%s"}\
|
||||
"""
|
||||
.formatted(
|
||||
plugin.getPluginId(),
|
||||
plugin.getDescriptor().getVersion()
|
||||
)
|
||||
);
|
||||
if (iterator.hasNext()) {
|
||||
sb.append(',');
|
||||
}
|
||||
}
|
||||
""".formatted(plugin.getPluginId(), plugin.getDescriptor().getVersion())
|
||||
)
|
||||
.collect(Collectors.joining(", ")));
|
||||
return Flux.fromIterable(startedPlugins)
|
||||
.mapNotNull(pluginWrapper -> {
|
||||
var pluginName = pluginWrapper.getPluginId();
|
||||
return BundleResourceUtils.getJsBundleResource(pluginManager, pluginName,
|
||||
BundleResourceUtils.JS_BUNDLE);
|
||||
})
|
||||
.flatMap(resource -> {
|
||||
try {
|
||||
// Specifying bufferSize as resource content length is
|
||||
// to append line breaks at the end of each plugin
|
||||
return DataBufferUtils.read(resource, DefaultDataBufferFactory.sharedInstance,
|
||||
(int) resource.contentLength())
|
||||
.doOnNext(dataBuffer -> {
|
||||
// add a new line after each plugin bundle to avoid syntax error
|
||||
dataBuffer.write("\n".getBytes(StandardCharsets.UTF_8));
|
||||
});
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to read plugin bundle resource", e);
|
||||
return Flux.empty();
|
||||
}
|
||||
})
|
||||
.concatWith(Flux.defer(() -> {
|
||||
var dataBuffer = DefaultDataBufferFactory.sharedInstance
|
||||
.wrap(plugins.getBytes(StandardCharsets.UTF_8));
|
||||
return Flux.just(dataBuffer);
|
||||
}));
|
||||
sb.append(']');
|
||||
return dataBufferFactory.wrap(sb.toString().getBytes(StandardCharsets.UTF_8));
|
||||
});
|
||||
var body = Flux.fromIterable(startedPlugins)
|
||||
.sort(Comparator.comparing(PluginWrapper::getPluginId))
|
||||
.concatMap(pluginWrapper -> {
|
||||
var pluginId = pluginWrapper.getPluginId();
|
||||
return Mono.<Resource>fromSupplier(
|
||||
() -> BundleResourceUtils.getJsBundleResource(
|
||||
pluginManager, pluginId, BundleResourceUtils.JS_BUNDLE
|
||||
)
|
||||
)
|
||||
.filter(Resource::isReadable)
|
||||
.flatMapMany(resource -> {
|
||||
var head = Mono.<DataBuffer>fromSupplier(
|
||||
() -> dataBufferFactory.wrap(
|
||||
("// Generated from plugin " + pluginId + "\n").getBytes()
|
||||
));
|
||||
var content = DataBufferUtils.read(
|
||||
resource, dataBufferFactory, StreamUtils.BUFFER_SIZE
|
||||
);
|
||||
var tail = Mono.fromSupplier(() -> dataBufferFactory.wrap("\n".getBytes()));
|
||||
return Flux.concat(head, content, tail);
|
||||
});
|
||||
});
|
||||
return Flux.concat(body, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<DataBuffer> uglifyCssBundle() {
|
||||
return Flux.fromIterable(pluginManager.getStartedPlugins())
|
||||
.mapNotNull(pluginWrapper -> {
|
||||
String pluginName = pluginWrapper.getPluginId();
|
||||
return BundleResourceUtils.getJsBundleResource(pluginManager, pluginName,
|
||||
BundleResourceUtils.CSS_BUNDLE);
|
||||
})
|
||||
.flatMap(resource -> {
|
||||
try {
|
||||
return DataBufferUtils.read(resource, DefaultDataBufferFactory.sharedInstance,
|
||||
(int) resource.contentLength());
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to read plugin css bundle resource", e);
|
||||
return Flux.empty();
|
||||
}
|
||||
.sort(Comparator.comparing(PluginWrapper::getPluginId))
|
||||
.concatMap(pluginWrapper -> {
|
||||
var pluginId = pluginWrapper.getPluginId();
|
||||
var dataBufferFactory = DefaultDataBufferFactory.sharedInstance;
|
||||
return Mono.<Resource>fromSupplier(() -> BundleResourceUtils.getJsBundleResource(
|
||||
pluginManager, pluginId, BundleResourceUtils.CSS_BUNDLE
|
||||
))
|
||||
.filter(Resource::isReadable)
|
||||
.flatMapMany(resource -> {
|
||||
var head = Mono.<DataBuffer>fromSupplier(() -> dataBufferFactory.wrap(
|
||||
("/* Generated from plugin " + pluginId + " */\n").getBytes()
|
||||
));
|
||||
var content = DataBufferUtils.read(
|
||||
resource, dataBufferFactory, StreamUtils.BUFFER_SIZE);
|
||||
var tail = Mono.fromSupplier(() -> dataBufferFactory.wrap("\n".getBytes()));
|
||||
return Flux.concat(head, content, tail);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue