From 9ec3088f1e390370dc57637d7efad6833bc3f9cd Mon Sep 17 00:00:00 2001 From: "chaos.su" Date: Fri, 1 May 2020 14:43:01 +0800 Subject: [PATCH] support distributed cache and customized halo home. (#754) * support distributed cache with redis * fix HOME_DIR; theme configurations auto update in distributed deployment; update redis cache configuration demo * remove redundant const of WORK_DIR Co-authored-by: John Niang --- build.gradle | 4 +- settings.gradle | 2 +- .../app/cache/AbstractStringCacheStore.java | 11 ++ .../run/halo/app/cache/LevelCacheStore.java | 12 -- .../run/halo/app/cache/RedisCacheStore.java | 143 ++++++++++++++++++ .../halo/app/config/HaloConfiguration.java | 5 +- .../app/config/properties/HaloProperties.java | 6 + .../run/halo/app/model/support/HaloConst.java | 1 - .../app/service/impl/ThemeServiceImpl.java | 18 +++ src/main/resources/application-demo.yaml | 7 +- 10 files changed, 192 insertions(+), 17 deletions(-) create mode 100644 src/main/java/run/halo/app/cache/RedisCacheStore.java diff --git a/build.gradle b/build.gradle index f703ab872..71f0568f5 100644 --- a/build.gradle +++ b/build.gradle @@ -72,6 +72,7 @@ ext { jsonVersion = "20190722" fastJsonVersion = "1.2.68" annotationsVersion = "3.0.1u2" + jedisVersion= '3.2.0' zxingVersion = "3.4.0" huaweiObsVersion = "3.19.7" } @@ -81,6 +82,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-data-jpa" implementation "org.springframework.boot:spring-boot-starter-web" implementation "org.springframework.boot:spring-boot-starter-undertow" + implementation "redis.clients:jedis:$jedisVersion" implementation "org.springframework.boot:spring-boot-starter-freemarker" implementation "com.sun.mail:jakarta.mail" @@ -140,4 +142,4 @@ dependencies { testImplementation "org.springframework.boot:spring-boot-starter-test" developmentOnly "org.springframework.boot:spring-boot-devtools" -} \ No newline at end of file +} diff --git a/settings.gradle b/settings.gradle index e8e710a50..3f230cca8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1 @@ -rootProject.name = 'halo' \ No newline at end of file +rootProject.name = 'halo' diff --git a/src/main/java/run/halo/app/cache/AbstractStringCacheStore.java b/src/main/java/run/halo/app/cache/AbstractStringCacheStore.java index bc20c13ea..22b30f592 100644 --- a/src/main/java/run/halo/app/cache/AbstractStringCacheStore.java +++ b/src/main/java/run/halo/app/cache/AbstractStringCacheStore.java @@ -18,6 +18,17 @@ import java.util.concurrent.TimeUnit; */ @Slf4j public abstract class AbstractStringCacheStore extends AbstractCacheStore { + protected Optional> jsonToCacheWrapper(String json) { + Assert.hasText(json, "json value must not be null"); + CacheWrapper cacheWrapper = null; + try { + cacheWrapper = JsonUtils.jsonToObject(json, CacheWrapper.class); + } catch (IOException e) { + e.printStackTrace(); + log.debug("erro json to wrapper value bytes: [{}]", json, e); + } + return Optional.ofNullable(cacheWrapper); + } public void putAny(String key, T value) { try { diff --git a/src/main/java/run/halo/app/cache/LevelCacheStore.java b/src/main/java/run/halo/app/cache/LevelCacheStore.java index 4306a3a2a..520de324c 100644 --- a/src/main/java/run/halo/app/cache/LevelCacheStore.java +++ b/src/main/java/run/halo/app/cache/LevelCacheStore.java @@ -116,18 +116,6 @@ public class LevelCacheStore extends AbstractStringCacheStore { return new String(bytes, Charset.defaultCharset()); } - private Optional> jsonToCacheWrapper(String json) { - Assert.hasText(json, "json value must not be null"); - CacheWrapper cacheWrapper = null; - try { - cacheWrapper = JsonUtils.jsonToObject(json, CacheWrapper.class); - } catch (IOException e) { - e.printStackTrace(); - log.debug("erro json to wrapper value bytes: [{}]", json, e); - } - return Optional.ofNullable(cacheWrapper); - } - private class CacheExpiryCleaner extends TimerTask { @Override diff --git a/src/main/java/run/halo/app/cache/RedisCacheStore.java b/src/main/java/run/halo/app/cache/RedisCacheStore.java new file mode 100644 index 000000000..b6eb7c2e5 --- /dev/null +++ b/src/main/java/run/halo/app/cache/RedisCacheStore.java @@ -0,0 +1,143 @@ +package run.halo.app.cache; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.Assert; + +import javax.annotation.PreDestroy; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.springframework.util.StringUtils; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.HostAndPort; + +import java.util.Set; +import java.util.HashSet; +import java.util.Date; + +import run.halo.app.config.properties.HaloProperties; +import run.halo.app.utils.JsonUtils; + +/** + * Redis cache store. + * + * @author chaos + */ +@Slf4j +public class RedisCacheStore extends AbstractStringCacheStore { + private volatile static JedisCluster REDIS; + protected HaloProperties haloProperties; + + /** + * Cache container. + */ + private final static ConcurrentHashMap> CACHE_CONTAINER = new ConcurrentHashMap<>(); + + /** + * Lock. + */ + private Lock lock = new ReentrantLock(); + + private void initRedis() { + JedisPoolConfig cfg = new JedisPoolConfig(); + cfg.setMaxIdle(2); + cfg.setMaxTotal(30); + cfg.setMaxWaitMillis(5000); + Set nodes = new HashSet<>(); + for (String hostPort : this.haloProperties.getCacheRedisNodes()) { + String[] temp = hostPort.split(":"); + if (temp.length > 0) { + String host = temp[0]; + int port = 6379; + if (temp.length > 1) { + try { + port = Integer.parseInt(temp[1]); + } catch (Exception ex) { + + } + } + nodes.add(new HostAndPort(host, port)); + } + } + if (nodes.isEmpty()) { + nodes.add(new HostAndPort("127.0.0.1", 6379)); + } + REDIS = new JedisCluster(nodes, 5, 20, 3, this.haloProperties.getCacheRedisPassword(), cfg); + log.info("Initialized cache redis cluster: {}", REDIS.getClusterNodes()); + } + + protected JedisCluster redis() { + if (REDIS == null) { + synchronized (RedisCacheStore.class) { + if (REDIS != null) { + return REDIS; + } + initRedis(); + return REDIS; + } + } + return REDIS; + } + + public RedisCacheStore(HaloProperties haloProperties) { + this.haloProperties = haloProperties; + initRedis(); + } + + @Override + Optional> getInternal(String key) { + Assert.hasText(key, "Cache key must not be blank"); + String v = REDIS.get(key); + return StringUtils.isEmpty(v) ? Optional.empty() : jsonToCacheWrapper(v); + } + + @Override + void putInternal(String key, CacheWrapper cacheWrapper) { + putInternalIfAbsent(key, cacheWrapper); + try { + REDIS.set(key, JsonUtils.objectToJson(cacheWrapper)); + Date ttl = cacheWrapper.getExpireAt(); + if (ttl != null) { + REDIS.pexpireAt(key, ttl.getTime()); + } + } catch (Exception e) { + log.warn("Put cache fail json2object key: [{}] value:[{}]", key, cacheWrapper); + } + } + + @Override + Boolean putInternalIfAbsent(String key, CacheWrapper cacheWrapper) { + Assert.hasText(key, "Cache key must not be blank"); + Assert.notNull(cacheWrapper, "Cache wrapper must not be null"); + try { + if (REDIS.setnx(key, JsonUtils.objectToJson(cacheWrapper)) <= 0) { + log.warn("Failed to put the cache, because the key: [{}] has been present already", key); + return false; + } + Date ttl = cacheWrapper.getExpireAt(); + if (ttl != null) { + REDIS.pexpireAt(key, ttl.getTime()); + } + return true; + } catch (JsonProcessingException e) { + log.warn("Put cache fail json2object key: [{}] value:[{}]", key, cacheWrapper); + } + log.debug("Cache key: [{}], original cache wrapper: [{}]", key, cacheWrapper); + return false; + } + + @Override + public void delete(String key) { + Assert.hasText(key, "Cache key must not be blank"); + REDIS.del(key); + log.debug("Removed key: [{}]", key); + } + + @PreDestroy + public void preDestroy() { + } +} diff --git a/src/main/java/run/halo/app/config/HaloConfiguration.java b/src/main/java/run/halo/app/config/HaloConfiguration.java index c330e4e10..42b819fb9 100644 --- a/src/main/java/run/halo/app/config/HaloConfiguration.java +++ b/src/main/java/run/halo/app/config/HaloConfiguration.java @@ -15,6 +15,7 @@ import org.springframework.web.client.RestTemplate; import run.halo.app.cache.AbstractStringCacheStore; import run.halo.app.cache.InMemoryCacheStore; import run.halo.app.cache.LevelCacheStore; +import run.halo.app.cache.RedisCacheStore; import run.halo.app.config.properties.HaloProperties; import run.halo.app.model.support.HaloConst; import run.halo.app.utils.HttpClientUtils; @@ -63,7 +64,9 @@ public class HaloConfiguration { case "level": stringCacheStore = new LevelCacheStore(); break; - + case "redis": + stringCacheStore = new RedisCacheStore(this.haloProperties); + break; case "memory": default: //memory or default diff --git a/src/main/java/run/halo/app/config/properties/HaloProperties.java b/src/main/java/run/halo/app/config/properties/HaloProperties.java index 41c726775..b011588b3 100644 --- a/src/main/java/run/halo/app/config/properties/HaloProperties.java +++ b/src/main/java/run/halo/app/config/properties/HaloProperties.java @@ -4,6 +4,7 @@ import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import run.halo.app.model.enums.Mode; +import java.util.ArrayList; import java.time.Duration; import static run.halo.app.model.support.HaloConst.*; @@ -78,4 +79,9 @@ public class HaloProperties { */ private String cache = "memory"; + private ArrayList cacheRedisNodes = new ArrayList<>(); + + private String cacheRedisPassword = ""; + + } diff --git a/src/main/java/run/halo/app/model/support/HaloConst.java b/src/main/java/run/halo/app/model/support/HaloConst.java index 8e915c6b6..7b8f3341b 100644 --- a/src/main/java/run/halo/app/model/support/HaloConst.java +++ b/src/main/java/run/halo/app/model/support/HaloConst.java @@ -143,5 +143,4 @@ public class HaloConst { * Version constant. */ public static String HALO_VERSION = null; - } diff --git a/src/main/java/run/halo/app/service/impl/ThemeServiceImpl.java b/src/main/java/run/halo/app/service/impl/ThemeServiceImpl.java index 3d2459a73..94469fd77 100644 --- a/src/main/java/run/halo/app/service/impl/ThemeServiceImpl.java +++ b/src/main/java/run/halo/app/service/impl/ThemeServiceImpl.java @@ -48,6 +48,8 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipInputStream; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import static run.halo.app.model.support.HaloConst.DEFAULT_THEME_ID; @@ -78,6 +80,11 @@ public class ThemeServiceImpl implements ThemeService { private final ApplicationEventPublisher eventPublisher; + /** + * in seconds. + */ + protected static final long ACTIVATED_THEME_SYNC_INTERVAL = 5; + /** * Activated theme id. */ @@ -103,6 +110,17 @@ public class ThemeServiceImpl implements ThemeService { themeWorkDir = Paths.get(haloProperties.getWorkDir(), THEME_FOLDER); this.eventPublisher = eventPublisher; + // check activated theme option changes every 5 seconds. + Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { + try { + String newActivatedThemeId = optionService.getByPropertyOrDefault(PrimaryProperties.THEME, String.class, DEFAULT_THEME_ID); + if (newActivatedThemeId != activatedThemeId) { + activateTheme(newActivatedThemeId); + } + } catch (Exception e) { + log.warn("theme option sync exception: {}", e); + } + }, ACTIVATED_THEME_SYNC_INTERVAL, ACTIVATED_THEME_SYNC_INTERVAL, TimeUnit.SECONDS); } @Override diff --git a/src/main/resources/application-demo.yaml b/src/main/resources/application-demo.yaml index 4458c2b44..c422669a0 100755 --- a/src/main/resources/application-demo.yaml +++ b/src/main/resources/application-demo.yaml @@ -62,4 +62,9 @@ halo: auth-enabled: true mode: demo workDir: ${user.home}/halo-demo/ - cache: level \ No newline at end of file + cache: level +# use redis as cache to support halo deployment +# cache: redis +# cache-redis-nodes: ['127.0.0.1:6380', '127.0.0.1:6379'] +# cache-redis-password: 123456 +#