From 68aa5db66bf28b3b6754a17d67dc260b766e117b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E7=B2=BE=E5=8D=8E?= <842761733@qq.com> Date: Mon, 22 Apr 2019 11:40:13 +0800 Subject: [PATCH] =?UTF-8?q?RocksDB=E7=BC=93=E5=AD=98=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E6=9B=B4=E6=8D=A2=E9=BB=98=E8=AE=A4=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E4=B8=BARocksDB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jodconverter-web/pom.xml | 5 + .../src/main/conf/application.properties | 3 +- .../cache/impl/CacheServiceJDKImpl.java | 3 +- .../cache/impl/CacheServiceRocksDBImpl.java | 170 ++++++++++++++++++ 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRocksDBImpl.java diff --git a/jodconverter-web/pom.xml b/jodconverter-web/pom.xml index a131dc29..f51be380 100644 --- a/jodconverter-web/pom.xml +++ b/jodconverter-web/pom.xml @@ -156,6 +156,11 @@ concurrentlinkedhashmap-lru 1.4.2 + + org.rocksdb + rocksdbjni + 5.17.2 + diff --git a/jodconverter-web/src/main/conf/application.properties b/jodconverter-web/src/main/conf/application.properties index a774c5d2..53d18fd5 100644 --- a/jodconverter-web/src/main/conf/application.properties +++ b/jodconverter-web/src/main/conf/application.properties @@ -21,10 +21,11 @@ spring.http.multipart.max-file-size=100MB #openoffice home路径 #office.home = C:\\Program Files (x86)\\OpenOffice 4 -#缓存实现类型，不配默认为JDK实现，可配置为redis实现（需要配置spring.redisson.address等参数） +#缓存实现类型，不配默认为内嵌RocksDB实现，可配置为redis(type = redis)实现（需要配置spring.redisson.address等参数）和 JDK 内置对象实现（type = jdk）, #cache.type = redis #redis连接 #spring.redisson.address = 192.168.1.204:6379 +#spring.redisson.password = xxx #######################################可在运行时动态配置####################################### #文本类型，默认如下，可自定义添加 diff --git a/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java index 01320859..8d48a257 100644 --- a/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java +++ b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java @@ -3,6 +3,7 @@ package cn.keking.service.cache.impl; import cn.keking.service.cache.CacheService; import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import com.googlecode.concurrentlinkedhashmap.Weighers; +import org.rocksdb.RocksDB; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -17,7 +18,7 @@ import java.util.concurrent.BlockingQueue; * @description */ @Service -@ConditionalOnExpression("'${cache.type:default}'.equals('default')") +@ConditionalOnExpression("'${cache.type:default}'.equals('jdk')") public class CacheServiceJDKImpl implements CacheService { private Map pdfCache; diff --git a/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRocksDBImpl.java b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRocksDBImpl.java new file mode 100644 index 00000000..5e84cfb4 --- /dev/null +++ b/jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRocksDBImpl.java @@ -0,0 +1,170 @@ +package cn.keking.service.cache.impl; + +import cn.keking.service.cache.CacheService; +import org.artofsolving.jodconverter.office.OfficeUtils; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** + * @auther: chenjh + * @time: 2019/4/22 11:02 + * @description + */ +@ConditionalOnExpression("'${cache.type:default}'.equals('default')") +@Service +public class CacheServiceRocksDBImpl implements CacheService { + + static { + RocksDB.loadLibrary(); + } + + private static final String DB_PATH = OfficeUtils.getHomePath() + File.separator + "cache"; + + private static final int QUEUE_SIZE = 500000; + + private static final Logger LOGGER = LoggerFactory.getLogger(CacheServiceRocksDBImpl.class); + + private BlockingQueue blockingQueue = new ArrayBlockingQueue(QUEUE_SIZE); + + private RocksDB db; + + { + try { + db = RocksDB.open(DB_PATH); + if (db.get(REDIS_FILE_PREVIEW_PDF_KEY.getBytes()) == null) { + Map initPDFCache = new HashMap<>(); + db.put(REDIS_FILE_PREVIEW_PDF_KEY.getBytes(), toByteArray(initPDFCache)); + } + if (db.get(REDIS_FILE_PREVIEW_IMGS_KEY.getBytes()) == null) { + Map> initIMGCache = new HashMap<>(); + db.put(REDIS_FILE_PREVIEW_IMGS_KEY.getBytes(), toByteArray(initIMGCache)); + } + } catch (RocksDBException | IOException e) { + LOGGER.error("Uable to init RocksDB" + e); + } + } + + + @Override + public void initPDFCachePool(Integer capacity) { + + } + + @Override + public void initIMGCachePool(Integer capacity) { + + } + + @Override + public void putPDFCache(String key, String value) { + try { + Map pdfCacheItem = new HashMap<>(); + pdfCacheItem.put(key, value); + db.put(REDIS_FILE_PREVIEW_PDF_KEY.getBytes(), toByteArray(pdfCacheItem)); + } catch (RocksDBException | IOException e) { + LOGGER.error("Put into RocksDB Exception" + e); + } + } + + @Override + public void putImgCache(String key, List value) { + try { + Map> imgCacheItem = new HashMap<>(); + imgCacheItem.put(key, value); + db.put(REDIS_FILE_PREVIEW_PDF_KEY.getBytes(), toByteArray(imgCacheItem)); + } catch (RocksDBException | IOException e) { + LOGGER.error("Put into RocksDB Exception" + e); + } + } + + @Override + public Map getPDFCache() { + Map result = new HashMap<>(); + try{ + result = (Map) toObject(db.get(REDIS_FILE_PREVIEW_PDF_KEY.getBytes())); + } catch (RocksDBException | IOException | ClassNotFoundException e) { + LOGGER.error("Get from RocksDB Exception" + e); + } + return result; + } + + @Override + public String getPDFCache(String key) { + String result = ""; + try{ + Map map = (Map) toObject(db.get(REDIS_FILE_PREVIEW_PDF_KEY.getBytes())); + result = map.get(key); + } catch (RocksDBException | IOException | ClassNotFoundException e) { + LOGGER.error("Get from RocksDB Exception" + e); + } + return result; + } + + @Override + public Map> getImgCache() { + Map> result = new HashMap<>(); + try{ + result = (Map>) toObject(db.get(REDIS_FILE_PREVIEW_IMGS_KEY.getBytes())); + } catch (RocksDBException | IOException | ClassNotFoundException e) { + LOGGER.error("Get from RocksDB Exception" + e); + } + return result; + } + + @Override + public List getImgCache(String key) { + List result = new ArrayList<>(); + Map> map; + try{ + map = (Map>) toObject(db.get(REDIS_FILE_PREVIEW_IMGS_KEY.getBytes())); + result = map.get(key); + } catch (RocksDBException | IOException | ClassNotFoundException e) { + LOGGER.error("Get from RocksDB Exception" + e); + } + return result; + } + + @Override + public void addQueueTask(String url) { + blockingQueue.add(url); + } + + @Override + public String takeQueueTask() throws InterruptedException { + return String.valueOf(blockingQueue.take()); + } + + private byte[] toByteArray (Object obj) throws IOException { + byte[] bytes = null; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(obj); + oos.flush(); + bytes = bos.toByteArray (); + oos.close(); + bos.close(); + return bytes; + } + + private Object toObject (byte[] bytes) throws IOException, ClassNotFoundException { + Object obj = null; + ByteArrayInputStream bis = new ByteArrayInputStream (bytes); + ObjectInputStream ois = new ObjectInputStream (bis); + obj = ois.readObject(); + ois.close(); + bis.close(); + return obj; + } +}