github
/
halo
mirror of https://github.com/halo-dev/halo
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: add leveldb cache store impl (#494) 

add leveldb config by yaml dev
add cache impl test
pull/472/head^2
pencilso 2020-01-12 19:41:09 +08:00 committed by Ryan Wang
parent de796b35d6
commit d867f4cca3
6 changed files with 291 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -59,6 +59,7 @@ ext {
image4jVersion = '0.7zensight1'
flywayVersion = '6.1.0'
h2Version = '1.4.196'
levelDbVersion = '0.12'
}
dependencies {
@ -103,6 +104,7 @@ dependencies {
implementation "net.sf.image4j:image4j:$image4jVersion"
implementation "org.flywaydb:flyway-core:$flywayVersion"
implementation "org.iq80.leveldb:leveldb:$levelDbVersion"
runtimeOnly "com.h2database:h2:$h2Version"
runtimeOnly 'mysql:mysql-connector-java'

163
src/main/java/run/halo/app/cache/LevelCacheStore.java vendored Normal file
View File

@ -0,0 +1,163 @@
package run.halo.app.cache;
import com.fasterxml.jackson.core.JsonProcessingException;
import lombok.extern.slf4j.Slf4j;
import org.iq80.leveldb.*;
import org.iq80.leveldb.impl.Iq80DBFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import run.halo.app.config.properties.HaloProperties;
import run.halo.app.utils.JsonUtils;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.*;
/**
* level-db cache store
* Create by Pencilso on 2020/1/9 7:20 下午
*/
@Slf4j
public class LevelCacheStore extends StringCacheStore {
/**
* Cleaner schedule period. (ms)
*/
private final static long PERIOD = 60 * 1000;
private static DB leveldb;
private Timer timer;
@Autowired
private HaloProperties haloProperties;
@PostConstruct
public void init() {
if (leveldb != null) return;
try {
//work path
File folder = new File(haloProperties.getWorkDir() + ".leveldb");
DBFactory factory = new Iq80DBFactory();
Options options = new Options();
options.createIfMissing(true);
//open leveldb store folder
leveldb = factory.open(folder, options);
timer = new Timer();
timer.scheduleAtFixedRate(new CacheExpiryCleaner(), 0, PERIOD);
} catch (Exception ex) {
log.error("init leveldb error ", ex);
}
}
/**
* 销毁
*/
@PreDestroy
public void preDestroy() {
try {
leveldb.close();
timer.cancel();
} catch (IOException e) {
log.error("close leveldb error ", e);
}
}
@Override
Optional<CacheWrapper<String>> getInternal(String key) {
Assert.hasText(key, "Cache key must not be blank");
byte[] bytes = leveldb.get(stringToBytes(key));
if (bytes != null) {
String valueJson = bytesToString(bytes);
return StringUtils.isEmpty(valueJson) ? Optional.empty() : jsonToCacheWrapper(valueJson);
}
return Optional.empty();
}
@Override
void putInternal(String key, CacheWrapper<String> cacheWrapper) {
putInternalIfAbsent(key, cacheWrapper);
}
@Override
Boolean putInternalIfAbsent(String key, CacheWrapper<String> cacheWrapper) {
Assert.hasText(key, "Cache key must not be blank");
Assert.notNull(cacheWrapper, "Cache wrapper must not be null");
try {
leveldb.put(
stringToBytes(key),
stringToBytes(JsonUtils.objectToJson(cacheWrapper))
);
return true;
} catch (JsonProcessingException e) {
log.warn("Put cache fail json2object key: [{}] value:[{}]", key, cacheWrapper);
}
log.debug("Cache key: [{}], original cache wrapper: [{}]", key, cacheWrapper);
return false;
}
@Override
public void delete(String key) {
leveldb.delete(stringToBytes(key));
log.debug("cache remove key: [{}]", key);
}
private byte[] stringToBytes(String str) {
return str.getBytes(Charset.defaultCharset());
}
private String bytesToString(byte[] bytes) {
return new String(bytes, Charset.defaultCharset());
}
private Optional<CacheWrapper<String>> jsonToCacheWrapper(String json) {
Assert.hasText(json, "json value must not be null");
CacheWrapper<String> cacheWrapper = null;
try {
cacheWrapper = JsonUtils.jsonToObject(json, CacheWrapper.class);
} catch (IOException e) {
e.printStackTrace();
log.debug("erro json to wrapper value bytes: [{}]", json, e);
}
return Optional.ofNullable(cacheWrapper);
}
private class CacheExpiryCleaner extends TimerTask {
@Override
public void run() {
//batch
WriteBatch writeBatch = leveldb.createWriteBatch();
DBIterator iterator = leveldb.iterator();
long currentTimeMillis = System.currentTimeMillis();
while (iterator.hasNext()) {
Map.Entry<byte[], byte[]> next = iterator.next();
if (next.getKey() == null || next.getValue() == null) {
continue;
}
String valueJson = bytesToString(next.getValue());
Optional<CacheWrapper<String>> stringCacheWrapper = StringUtils.isEmpty(valueJson) ? Optional.empty() : jsonToCacheWrapper(valueJson);
if (stringCacheWrapper.isPresent()) {
//get expireat time
long expireAtTime = stringCacheWrapper.map(CacheWrapper::getExpireAt)
.map(Date::getTime)
.orElse(0L);
//if expire
if (expireAtTime != 0 && currentTimeMillis > expireAtTime) {
writeBatch.delete(next.getKey());
log.debug("deleted the cache: [{}] for expiration", bytesToString(next.getKey()));
}
}
}
leveldb.write(writeBatch);
}
}
}

18
src/main/java/run/halo/app/config/HaloConfiguration.java
View File

@ -14,6 +14,7 @@ import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
import org.springframework.web.client.RestTemplate;
import run.halo.app.cache.InMemoryCacheStore;
import run.halo.app.cache.LevelCacheStore;
import run.halo.app.cache.StringCacheStore;
import run.halo.app.config.properties.HaloProperties;
import run.halo.app.filter.CorsFilter;
@ -63,7 +64,22 @@ public class HaloConfiguration {
@Bean
@ConditionalOnMissingBean
public StringCacheStore stringCacheStore() {
return new InMemoryCacheStore();
StringCacheStore stringCacheStore;
switch (haloProperties.getCache()) {
case "level":
stringCacheStore = new LevelCacheStore();
break;
case "memory":
default:
//memory or default
stringCacheStore = new InMemoryCacheStore();
break;
}
log.info("halo cache store load impl : [{}]", stringCacheStore.getClass());
return stringCacheStore;
}
/**

8
src/main/java/run/halo/app/config/properties/HaloProperties.java
View File

@ -61,6 +61,14 @@ public class HaloProperties {
*/
private Duration downloadTimeout = Duration.ofSeconds(30);
/**
* cache store impl
* memory
* level
*/
private String cache = "memory";
public HaloProperties() throws IOException {
// Create work directory if not exist
Files.createDirectories(Paths.get(workDir));

3
src/main/resources/application-dev.yaml
View File

@ -63,4 +63,5 @@ halo:
doc-disabled: false
production-env: false
auth-enabled: true
workDir: ${user.home}/halo-dev/
workDir: ${user.home}/halo-dev/
cache: level

99
src/test/java/run/halo/app/cache/CacheStoreTest.java vendored Normal file
View File

@ -0,0 +1,99 @@
package run.halo.app.cache;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.*;
/**
* CacheStoreTest.
*
* @author johnniang
* @date 3/28/19
*/
@RunWith(SpringRunner.class)
@SpringBootTest
public class CacheStoreTest {
@Autowired
private StringCacheStore cacheStore;
@Test(expected = IllegalArgumentException.class)
public void putNullValueTest() {
String key = "test_key";
cacheStore.put(key, null);
}
@Test(expected = IllegalArgumentException.class)
public void putNullKeyTest() {
String value = "test_value";
cacheStore.put(null, value);
}
@Test(expected = IllegalArgumentException.class)
public void getByNullKeyTest() {
cacheStore.get(null);
}
@Test
public void getNullTest() {
String key = "test_key";
Optional<String> valueOptional = cacheStore.get(key);
assertFalse(valueOptional.isPresent());
}
@Test
public void expirationTest() throws InterruptedException {
String key = "test_key";
String value = "test_value";
cacheStore.put(key, value, 500, TimeUnit.MILLISECONDS);
Optional<String> valueOptional = cacheStore.get(key);
assertTrue(valueOptional.isPresent());
assertThat(valueOptional.get(), equalTo(value));
TimeUnit.SECONDS.sleep(1L);
valueOptional = cacheStore.get(key);
assertFalse(valueOptional.isPresent());
}
@Test
public void deleteTest() {
String key = "test_key";
String value = "test_value";
// Put the cache
cacheStore.put(key, value);
// Get the caceh
Optional<String> valueOptional = cacheStore.get(key);
// Assert
assertTrue(valueOptional.isPresent());
assertThat(valueOptional.get(), equalTo(value));
// Delete the cache
cacheStore.delete(key);
// Get the cache again
valueOptional = cacheStore.get(key);
// Assertion
assertFalse(valueOptional.isPresent());
}
}