Generate JS and CSS bundle with fixed buffer size (#6573)

#### What type of PR is this?

/kind improvement
/area core
/milestone 2.20.x

#### What this PR does / why we need it:

If we are running Halo instance in machine with small memory available, the JS/CSS bundle might  not be accessible.

This RP refactors generation of JS and CSS bundle with fixed buffer size rather than length of original resources.

```java
2024-09-02T15:01:27.667+08:00  WARN 62039 --- [boundedElastic-3] reactor.core.Exceptions                  : throwIfFatal detected a jvm fatal exception, which is thrown and logged below:

java.lang.OutOfMemoryError: Java heap space
        at java.base/java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:64) ~[na:na]
        at java.base/java.nio.ByteBuffer.allocate(ByteBuffer.java:363) ~[na:na]
        at org.springframework.core.io.buffer.DefaultDataBuffer.allocate(DefaultDataBuffer.java:234) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DefaultDataBuffer.setCapacity(DefaultDataBuffer.java:196) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DefaultDataBuffer.ensureWritable(DefaultDataBuffer.java:228) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:296) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:289) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DefaultDataBuffer.write(DefaultDataBuffer.java:43) ~[spring-core-6.1.12.jar:6.1.12]
        at run.halo.app.core.extension.service.impl.PluginServiceImpl.lambda$uglifyJsBundle$17(PluginServiceImpl.java:257) ~[classes/:na]
        at run.halo.app.core.extension.service.impl.PluginServiceImpl$$Lambda$4661/0x000000c80214e298.accept(Unknown Source) ~[na:na]
        at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onNext(FluxPeekFuseable.java:196) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.onNext(FluxUsing.java:353) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxGenerate$GenerateSubscription.next(FluxGenerate.java:178) ~[reactor-core-3.6.9.jar:3.6.9]
        at org.springframework.core.io.buffer.DataBufferUtils$ReadableByteChannelGenerator.accept(DataBufferUtils.java:1002) ~[spring-core-6.1.12.jar:6.1.12]
        at org.springframework.core.io.buffer.DataBufferUtils$ReadableByteChannelGenerator.accept(DataBufferUtils.java:974) ~[spring-core-6.1.12.jar:6.1.12]
        at reactor.core.publisher.FluxGenerate.lambda$new$1(FluxGenerate.java:58) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxGenerate$$Lambda$4155/0x000000c802069228.apply(Unknown Source) ~[na:na]
        at reactor.core.publisher.FluxGenerate$GenerateSubscription.slowPath(FluxGenerate.java:271) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxGenerate$GenerateSubscription.request(FluxGenerate.java:213) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.request(FluxUsing.java:320) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.request(FluxPeekFuseable.java:144) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxFlatMap$FlatMapInner.onSubscribe(FluxFlatMap.java:968) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxPeekFuseable$PeekFuseableSubscriber.onSubscribe(FluxPeekFuseable.java:178) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.onSubscribe(FluxUsing.java:347) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxGenerate.subscribe(FluxGenerate.java:85) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxUsing.subscribe(FluxUsing.java:102) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.Flux.subscribe(Flux.java:8848) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxFlatMap$FlatMapMain.onNext(FluxFlatMap.java:430) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxHandleFuseable$HandleFuseableSubscriber.tryOnNext(FluxHandleFuseable.java:135) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxIterable$IterableSubscriptionConditional.slowPath(FluxIterable.java:664) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxIterable$IterableSubscriptionConditional.request(FluxIterable.java:623) ~[reactor-core-3.6.9.jar:3.6.9]
        at reactor.core.publisher.FluxHandleFuseable$HandleFuseableSubscriber.request(FluxHandleFuseable.java:260) ~[reactor-core-3.6.9.jar:3.6.9]

2024-09-02T15:01:27.681+08:00 DEBUG 62039 --- [boundedElastic-3] a.w.r.e.AbstractErrorWebExceptionHandler : [131a559b-102] Resolved [OutOfMemoryError: Java heap space] for HTTP GET /apis/api.console.halo.run/v1alpha1/plugins/-/bundle.js
2024-09-02T15:01:27.681+08:00 ERROR 62039 --- [boundedElastic-3] a.w.r.e.AbstractErrorWebExceptionHandler : [131a559b-102]  500 Server Error for HTTP GET "/apis/api.console.halo.run/v1alpha1/plugins/-/bundle.js?v=1725260408176"

java.lang.OutOfMemoryError: Java heap space
        at java.base/java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:64) ~[na:na]
```

#### Does this PR introduce a user-facing change?

```release-note
优化在内存紧张时 Console 端无法加载插件资源的问题
```
pull/6679/head
John Niang 2024-09-18 16:22:50 +08:00 committed by GitHub
parent 749c80cb96
commit ded5b4135f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 62 additions and 49 deletions

View File

@ -40,6 +40,7 @@ import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import org.springframework.util.FileSystemUtils;
import org.springframework.util.StreamUtils;
import org.springframework.web.server.ServerWebInputException;
import reactor.core.Exceptions;
import reactor.core.publisher.Flux;
@ -229,61 +230,73 @@ public class PluginServiceImpl implements PluginService, InitializingBean, Dispo
@Override
public Flux<DataBuffer> uglifyJsBundle() {
var startedPlugins = List.copyOf(pluginManager.getStartedPlugins());
String plugins = """
this.enabledPlugins = [%s]
""".formatted(startedPlugins.stream()
.map(plugin -> """
{
"name": "%s",
"version": "%s"
var dataBufferFactory = DefaultDataBufferFactory.sharedInstance;
var end = Mono.fromSupplier(
() -> {
var sb = new StringBuilder("this.enabledPlugins = [");
var iterator = startedPlugins.iterator();
while (iterator.hasNext()) {
var plugin = iterator.next();
sb.append("""
{"name":"%s","version":"%s"}\
"""
.formatted(
plugin.getPluginId(),
plugin.getDescriptor().getVersion()
)
);
if (iterator.hasNext()) {
sb.append(',');
}
}
""".formatted(plugin.getPluginId(), plugin.getDescriptor().getVersion())
)
.collect(Collectors.joining(", ")));
return Flux.fromIterable(startedPlugins)
.mapNotNull(pluginWrapper -> {
var pluginName = pluginWrapper.getPluginId();
return BundleResourceUtils.getJsBundleResource(pluginManager, pluginName,
BundleResourceUtils.JS_BUNDLE);
})
.flatMap(resource -> {
try {
// Specifying bufferSize as resource content length is
// to append line breaks at the end of each plugin
return DataBufferUtils.read(resource, DefaultDataBufferFactory.sharedInstance,
(int) resource.contentLength())
.doOnNext(dataBuffer -> {
// add a new line after each plugin bundle to avoid syntax error
dataBuffer.write("\n".getBytes(StandardCharsets.UTF_8));
});
} catch (IOException e) {
log.error("Failed to read plugin bundle resource", e);
return Flux.empty();
}
})
.concatWith(Flux.defer(() -> {
var dataBuffer = DefaultDataBufferFactory.sharedInstance
.wrap(plugins.getBytes(StandardCharsets.UTF_8));
return Flux.just(dataBuffer);
}));
sb.append(']');
return dataBufferFactory.wrap(sb.toString().getBytes(StandardCharsets.UTF_8));
});
var body = Flux.fromIterable(startedPlugins)
.sort(Comparator.comparing(PluginWrapper::getPluginId))
.concatMap(pluginWrapper -> {
var pluginId = pluginWrapper.getPluginId();
return Mono.<Resource>fromSupplier(
() -> BundleResourceUtils.getJsBundleResource(
pluginManager, pluginId, BundleResourceUtils.JS_BUNDLE
)
)
.filter(Resource::isReadable)
.flatMapMany(resource -> {
var head = Mono.<DataBuffer>fromSupplier(
() -> dataBufferFactory.wrap(
("// Generated from plugin " + pluginId + "\n").getBytes()
));
var content = DataBufferUtils.read(
resource, dataBufferFactory, StreamUtils.BUFFER_SIZE
);
var tail = Mono.fromSupplier(() -> dataBufferFactory.wrap("\n".getBytes()));
return Flux.concat(head, content, tail);
});
});
return Flux.concat(body, end);
}
@Override
public Flux<DataBuffer> uglifyCssBundle() {
return Flux.fromIterable(pluginManager.getStartedPlugins())
.mapNotNull(pluginWrapper -> {
String pluginName = pluginWrapper.getPluginId();
return BundleResourceUtils.getJsBundleResource(pluginManager, pluginName,
BundleResourceUtils.CSS_BUNDLE);
})
.flatMap(resource -> {
try {
return DataBufferUtils.read(resource, DefaultDataBufferFactory.sharedInstance,
(int) resource.contentLength());
} catch (IOException e) {
log.error("Failed to read plugin css bundle resource", e);
return Flux.empty();
}
.sort(Comparator.comparing(PluginWrapper::getPluginId))
.concatMap(pluginWrapper -> {
var pluginId = pluginWrapper.getPluginId();
var dataBufferFactory = DefaultDataBufferFactory.sharedInstance;
return Mono.<Resource>fromSupplier(() -> BundleResourceUtils.getJsBundleResource(
pluginManager, pluginId, BundleResourceUtils.CSS_BUNDLE
))
.filter(Resource::isReadable)
.flatMapMany(resource -> {
var head = Mono.<DataBuffer>fromSupplier(() -> dataBufferFactory.wrap(
("/* Generated from plugin " + pluginId + " */\n").getBytes()
));
var content = DataBufferUtils.read(
resource, dataBufferFactory, StreamUtils.BUFFER_SIZE);
var tail = Mono.fromSupplier(() -> dataBufferFactory.wrap("\n".getBytes()));
return Flux.concat(head, content, tail);
});
});
}