From 37762cf034c8fd3a70afa08eb694a48b97e094c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E7=B2=BE=E5=8D=8E?= <842761733@qq.com> Date: Wed, 19 Jun 2019 14:18:09 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81FTP=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=E4=BD=9C=E4=B8=BA=E9=A2=84=E8=A7=88=E6=BA=90?= =?UTF-8?q?url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jodconverter-web/pom.xml | 6 ++ .../src/main/conf/application.properties | 7 +++ .../cn/keking/config/ConfigConstants.java | 27 +++++++++ .../keking/config/ConfigRefreshComponent.java | 20 +++++-- .../service/impl/CompressFilePreviewImpl.java | 3 +- .../service/impl/OfficeFilePreviewImpl.java | 3 +- .../service/impl/PdfFilePreviewImpl.java | 3 +- .../java/cn/keking/utils/DownloadUtils.java | 52 +++++++++++------ .../main/java/cn/keking/utils/FileUtils.java | 6 +- .../main/java/cn/keking/utils/FtpUtils.java | 57 +++++++++++++++++++ .../java/cn/keking/utils/SimTextUtil.java | 9 ++- 11 files changed, 161 insertions(+), 32 deletions(-) create mode 100644 jodconverter-web/src/main/java/cn/keking/utils/FtpUtils.java diff --git a/jodconverter-web/pom.xml b/jodconverter-web/pom.xml index de2b4f19..265e2e19 100644 --- a/jodconverter-web/pom.xml +++ b/jodconverter-web/pom.xml @@ -147,6 +147,12 @@ commons-cli 1.2 + + + commons-net + commons-net + 3.6 + com.thoughtworks.xstream xstream diff --git a/jodconverter-web/src/main/conf/application.properties b/jodconverter-web/src/main/conf/application.properties index 7929870a..4ab7175b 100644 --- a/jodconverter-web/src/main/conf/application.properties +++ b/jodconverter-web/src/main/conf/application.properties @@ -39,3 +39,10 @@ cache.clean = true #converted.file.charset = GBK #office类型文档(word ppt)样式，默认为图片(image)，可配置为pdf（预览时也有按钮切换） #office.preview.type = pdf + +#预览源为FTP时 FTP用户名，可在ftp url后面加参数ftp.username=ftpuser指定，不指定默认用配置的 +ftp.username = ftpuser +#预览源为FTP时 FTP密码，可在ftp url后面加参数ftp.password=123456指定，不指定默认用配置的 +ftp.password = 123456 +#预览源为FTP时, FTP连接默认ControlEncoding(根据FTP服务器操作系统选择，Linux一般为UTF-8，Windows一般为GBK)，可在ftp url后面加参数ftp.control.encoding=UTF-8指定，不指定默认用配置的 +ftp.control.encoding = UTF-8 diff --git a/jodconverter-web/src/main/java/cn/keking/config/ConfigConstants.java b/jodconverter-web/src/main/java/cn/keking/config/ConfigConstants.java index 44d3ec3d..934a710e 100644 --- a/jodconverter-web/src/main/java/cn/keking/config/ConfigConstants.java +++ b/jodconverter-web/src/main/java/cn/keking/config/ConfigConstants.java @@ -18,6 +18,9 @@ public class ConfigConstants { private static String[] media = {}; private static String convertedFileCharset; private static String officePreviewType; + private static String ftpUsername; + private static String ftpPassword; + private static String ftpControlEncoding; private static String fileDir = OfficeUtils.getHomePath() + File.separator + "file" + File.separator; public static String[] getSimText() { @@ -52,6 +55,30 @@ public class ConfigConstants { ConfigConstants.officePreviewType = officePreviewType; } + public static String getFtpUsername() { + return ftpUsername; + } + + public static void setFtpUsername(String ftpUsername) { + ConfigConstants.ftpUsername = ftpUsername; + } + + public static String getFtpPassword() { + return ftpPassword; + } + + public static String getFtpControlEncoding() { + return ftpControlEncoding; + } + + public static void setFtpControlEncoding(String ftpControlEncoding) { + ConfigConstants.ftpControlEncoding = ftpControlEncoding; + } + + public static void setFtpPassword(String ftpPassword) { + ConfigConstants.ftpPassword = ftpPassword; + } + public static String getFileDir() { return fileDir; } diff --git a/jodconverter-web/src/main/java/cn/keking/config/ConfigRefreshComponent.java b/jodconverter-web/src/main/java/cn/keking/config/ConfigRefreshComponent.java index eb2ca2ba..32efe2ed 100644 --- a/jodconverter-web/src/main/java/cn/keking/config/ConfigRefreshComponent.java +++ b/jodconverter-web/src/main/java/cn/keking/config/ConfigRefreshComponent.java @@ -24,6 +24,10 @@ public class ConfigRefreshComponent { public static final String DEFAULT_TXT_TYPE = "txt,html,xml,properties,md,java,py,c,cpp,sql"; public static final String DEFAULT_MEDIA_TYPE = "mp3,wav,mp4,flv"; + public static final String DEFAULT_CONVERTER_CHARSET = System.getProperty("sun.jnu.encoding"); + public static final String DEFAULT_FTP_USERNAME = null; + public static final String DEFAULT_FTP_PASSWORD = null; + public static final String DEFAULT_FTP_CONTROL_ENCODING = "UTF-8"; @PostConstruct @@ -37,13 +41,15 @@ public class ConfigRefreshComponent { public void run() { try { Properties properties = new Properties(); - Properties sysProperties = System.getProperties(); String text; String media; - String convertedFileCharset = sysProperties.getProperty("sun.jnu.encoding"); String[] textArray; String[] mediaArray; + String convertedFileCharset; String officePreviewType; + String ftpUsername; + String ftpPassword; + String ftpControlEncoding; String configFilePath = OfficeUtils.getCustomizedConfigPath(); while (true) { FileReader fileReader = new FileReader(configFilePath); @@ -51,20 +57,26 @@ public class ConfigRefreshComponent { properties.load(bufferedReader); text = properties.getProperty("simText", DEFAULT_TXT_TYPE); media = properties.getProperty("media", DEFAULT_MEDIA_TYPE); - convertedFileCharset = properties.getProperty("converted.file.charset", convertedFileCharset); + convertedFileCharset = properties.getProperty("converted.file.charset", DEFAULT_CONVERTER_CHARSET); officePreviewType = properties.getProperty("office.preview.type", OfficeFilePreviewImpl.OFFICE_PREVIEW_TYPE_IMAGE); + ftpUsername = properties.getProperty("ftp.username", DEFAULT_FTP_USERNAME); + ftpPassword = properties.getProperty("ftp.password", DEFAULT_FTP_PASSWORD); + ftpControlEncoding = properties.getProperty("ftp.control.encoding", DEFAULT_FTP_CONTROL_ENCODING); textArray = text.split(","); mediaArray = media.split(","); ConfigConstants.setSimText(textArray); ConfigConstants.setMedia(mediaArray); ConfigConstants.setConvertedFileCharset(convertedFileCharset); ConfigConstants.setOfficePreviewType(officePreviewType); + ConfigConstants.setFtpUsername(ftpUsername); + ConfigConstants.setFtpPassword(ftpPassword); + ConfigConstants.setFtpControlEncoding(ftpControlEncoding); bufferedReader.close(); fileReader.close(); Thread.sleep(1000L); } } catch (IOException | InterruptedException e) { - LOGGER.error("读取配置文件异常：{}", e); + LOGGER.error("读取配置文件异常", e); } } } diff --git a/jodconverter-web/src/main/java/cn/keking/service/impl/CompressFilePreviewImpl.java b/jodconverter-web/src/main/java/cn/keking/service/impl/CompressFilePreviewImpl.java index c1297661..daef2df1 100644 --- a/jodconverter-web/src/main/java/cn/keking/service/impl/CompressFilePreviewImpl.java +++ b/jodconverter-web/src/main/java/cn/keking/service/impl/CompressFilePreviewImpl.java @@ -30,12 +30,11 @@ public class CompressFilePreviewImpl implements FilePreview{ @Override public String filePreviewHandle(String url, Model model, FileAttribute fileAttribute) { String fileName=fileAttribute.getName(); - String decodedUrl=fileAttribute.getDecodedUrl(); String suffix=fileAttribute.getSuffix(); String fileTree = null; // 判断文件名是否存在(redis缓存读取) if (!StringUtils.hasText(fileUtils.getConvertedFile(fileName))) { - ReturnResponse response = downloadUtils.downLoad(decodedUrl, suffix, fileName); + ReturnResponse response = downloadUtils.downLoad(fileAttribute, fileName); if (0 != response.getCode()) { model.addAttribute("fileType", suffix); model.addAttribute("msg", response.getMsg()); diff --git a/jodconverter-web/src/main/java/cn/keking/service/impl/OfficeFilePreviewImpl.java b/jodconverter-web/src/main/java/cn/keking/service/impl/OfficeFilePreviewImpl.java index 57543637..2df17311 100644 --- a/jodconverter-web/src/main/java/cn/keking/service/impl/OfficeFilePreviewImpl.java +++ b/jodconverter-web/src/main/java/cn/keking/service/impl/OfficeFilePreviewImpl.java @@ -48,7 +48,6 @@ public class OfficeFilePreviewImpl implements FilePreview { String originUrl = (String) model.asMap().get("originUrl"); String suffix=fileAttribute.getSuffix(); String fileName=fileAttribute.getName(); - String decodedUrl=fileAttribute.getDecodedUrl(); boolean isHtml = suffix.equalsIgnoreCase("xls") || suffix.equalsIgnoreCase("xlsx"); String pdfName = fileName.substring(0, fileName.lastIndexOf(".") + 1) + (isHtml ? "html" : "pdf"); String outFilePath = fileDir + pdfName; @@ -56,7 +55,7 @@ public class OfficeFilePreviewImpl implements FilePreview { if (!fileUtils.listConvertedFiles().containsKey(pdfName)) { String filePath = fileDir + fileName; if (!new File(filePath).exists()) { - ReturnResponse response = downloadUtils.downLoad(decodedUrl, suffix, null); + ReturnResponse response = downloadUtils.downLoad(fileAttribute, null); if (0 != response.getCode()) { model.addAttribute("fileType", suffix); model.addAttribute("msg", response.getMsg()); diff --git a/jodconverter-web/src/main/java/cn/keking/service/impl/PdfFilePreviewImpl.java b/jodconverter-web/src/main/java/cn/keking/service/impl/PdfFilePreviewImpl.java index 42765db1..dba00d15 100644 --- a/jodconverter-web/src/main/java/cn/keking/service/impl/PdfFilePreviewImpl.java +++ b/jodconverter-web/src/main/java/cn/keking/service/impl/PdfFilePreviewImpl.java @@ -35,7 +35,6 @@ public class PdfFilePreviewImpl implements FilePreview{ @Override public String filePreviewHandle(String url, Model model, FileAttribute fileAttribute) { - String decodedUrl=fileAttribute.getDecodedUrl(); String suffix=fileAttribute.getSuffix(); String fileName=fileAttribute.getName(); String officePreviewType = model.asMap().get("officePreviewType") == null ? ConfigConstants.getOfficePreviewType() : model.asMap().get("officePreviewType").toString(); @@ -46,7 +45,7 @@ public class PdfFilePreviewImpl implements FilePreview{ if (OfficeFilePreviewImpl.OFFICE_PREVIEW_TYPE_IMAGE.equals(officePreviewType) || OfficeFilePreviewImpl.OFFICE_PREVIEW_TYPE_ALLIMAGES.equals(officePreviewType)) { //当文件不存在时，就去下载 if (!new File(outFilePath).exists()) { - ReturnResponse response = downloadUtils.downLoad(decodedUrl, suffix, fileName); + ReturnResponse response = downloadUtils.downLoad(fileAttribute, fileName); if (0 != response.getCode()) { model.addAttribute("fileType", suffix); model.addAttribute("msg", response.getMsg()); diff --git a/jodconverter-web/src/main/java/cn/keking/utils/DownloadUtils.java b/jodconverter-web/src/main/java/cn/keking/utils/DownloadUtils.java index 5ff8dc6f..1615fc30 100644 --- a/jodconverter-web/src/main/java/cn/keking/utils/DownloadUtils.java +++ b/jodconverter-web/src/main/java/cn/keking/utils/DownloadUtils.java @@ -1,7 +1,11 @@ package cn.keking.utils; import cn.keking.config.ConfigConstants; +import cn.keking.model.FileAttribute; import cn.keking.model.ReturnResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.*; import java.net.*; @@ -13,7 +17,16 @@ import java.util.UUID; @Component public class DownloadUtils { - String fileDir = ConfigConstants.getFileDir(); + private static final Logger LOGGER = LoggerFactory.getLogger(DownloadUtils.class); + + private String fileDir = ConfigConstants.getFileDir(); + + @Autowired + private FileUtils fileUtils; + + private static final String URL_PARAM_FTP_USERNAME = "ftp.username"; + private static final String URL_PARAM_FTP_PASSWORD = "ftp.password"; + private static final String URL_PARAM_FTP_CONTROL_ENCODING = "ftp.control.encoding"; /** * 一开始测试的时候发现有些文件没有下载下来，而有些可以；当时也是郁闷了好一阵，但是最终还是不得解 @@ -21,11 +34,12 @@ public class DownloadUtils { * 应该是转义出了问题，url转义中会把+号当成空格来计算，所以才会出现这种情况，遂想要通过整体替换空格为加号，因为url * 中的参数部分是不会出现空格的，但是文件名中就不好确定了，所以只对url参数部分做替换 * 注: 针对URLEncoder.encode(s,charset)会将空格转成+的情况需要做下面的替换工作 - * @param urlAddress - * @param type + * @param fileAttribute * @return */ - public ReturnResponse downLoad(String urlAddress, String type, String fileName) { + public ReturnResponse downLoad(FileAttribute fileAttribute, String fileName) { + String urlAddress = fileAttribute.getDecodedUrl(); + String type = fileAttribute.getSuffix(); ReturnResponse response = new ReturnResponse<>(0, "下载成功!!!", ""); URL url = null; try { @@ -49,17 +63,24 @@ public class DownloadUtils { dirFile.mkdirs(); } try { - URLConnection connection = url.openConnection(); - InputStream in = connection.getInputStream(); + if ("ftp".equals(url.getProtocol())) { + String ftpUsername = fileUtils.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_USERNAME); + String ftpPassword = fileUtils.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_PASSWORD); + String ftpControlEncoding = fileUtils.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_CONTROL_ENCODING); + FtpUtils.download(fileAttribute.getUrl(), realPath, ftpUsername, ftpPassword, ftpControlEncoding); + } else { + URLConnection connection = url.openConnection(); + InputStream in = connection.getInputStream(); - FileOutputStream os = new FileOutputStream(realPath); - byte[] buffer = new byte[4 * 1024]; - int read; - while ((read = in.read(buffer)) > 0) { - os.write(buffer, 0, read); + FileOutputStream os = new FileOutputStream(realPath); + byte[] buffer = new byte[4 * 1024]; + int read; + while ((read = in.read(buffer)) > 0) { + os.write(buffer, 0, read); + } + os.close(); + in.close(); } - os.close(); - in.close(); response.setContent(realPath); // 同样针对类txt文件，如果成功msg包含的是转换后的文件名 response.setMsg(fileName); @@ -68,15 +89,14 @@ public class DownloadUtils { if("txt".equals(type)){ convertTextPlainFileCharsetToUtf8(realPath); } - return response; } catch (IOException e) { - e.printStackTrace(); + LOGGER.error("文件下载失败", e); response.setCode(1); response.setContent(null); if (e instanceof FileNotFoundException) { response.setMsg("文件不存在!!!"); - }else { + } else { response.setMsg(e.getMessage()); } return response; diff --git a/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java b/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java index a3ff4765..0a525dec 100644 --- a/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java +++ b/jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java @@ -285,9 +285,9 @@ public class FileUtils { * @param name * @return */ - private String getUrlParameterReg(String url, String name) { + public String getUrlParameterReg(String url, String name) { Map mapRequest = new HashMap(); - String strUrlParam = TruncateUrlPage(url); + String strUrlParam = truncateUrlPage(url); if (strUrlParam == null) { return ""; } @@ -312,7 +312,7 @@ public class FileUtils { * @param strURL url地址 * @return url请求参数部分 */ - private static String TruncateUrlPage(String strURL) { + private String truncateUrlPage(String strURL) { String strAllParam = null; strURL = strURL.trim(); String[] arrSplit = strURL.split("[?]"); diff --git a/jodconverter-web/src/main/java/cn/keking/utils/FtpUtils.java b/jodconverter-web/src/main/java/cn/keking/utils/FtpUtils.java new file mode 100644 index 00000000..91f45de3 --- /dev/null +++ b/jodconverter-web/src/main/java/cn/keking/utils/FtpUtils.java @@ -0,0 +1,57 @@ +package cn.keking.utils; + +import cn.keking.config.ConfigConstants; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPReply; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URL; + +/** + * @auther: chenjh + * @since: 2019/6/18 14:36 + */ +public class FtpUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(FtpUtils.class); + + public static FTPClient connect(String host, int port, String username, String password, String controlEncoding) throws IOException { + FTPClient ftpClient = new FTPClient(); + ftpClient.connect(host, port); + if (!StringUtils.isEmpty(username) && !StringUtils.isEmpty(password)) { + ftpClient.login(username, password); + } + int reply = ftpClient.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftpClient.disconnect(); + } + ftpClient.setControlEncoding(controlEncoding); + ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE); + return ftpClient; + } + + public static void download(String ftpUrl, String localFilePath, String ftpUsername, String ftpPassword, String ftpControlEncoding) throws IOException { + String username = StringUtils.isEmpty(ftpUsername) ? ConfigConstants.getFtpUsername() : ftpUsername; + String password = StringUtils.isEmpty(ftpPassword) ? ConfigConstants.getFtpPassword() : ftpPassword; + String controlEncoding = StringUtils.isEmpty(ftpControlEncoding) ? ConfigConstants.getFtpControlEncoding() : ftpControlEncoding; + URL url = new URL(ftpUrl); + String host = url.getHost(); + int port = (url.getPort() == -1) ? url.getDefaultPort() : url.getPort(); + String remoteFilePath = url.getPath(); + LOGGER.debug("FTP connection url:{}, username:{}, password:{}, controlEncoding:{}, localFilePath:{}", ftpUrl, username, password, controlEncoding, localFilePath); + FTPClient ftpClient = connect(host, port, username, password, controlEncoding); + OutputStream outputStream = new FileOutputStream(localFilePath); + ftpClient.enterLocalPassiveMode(); + boolean downloadResult = ftpClient.retrieveFile(new String(remoteFilePath.getBytes(controlEncoding), "ISO-8859-1"), outputStream); + LOGGER.debug("FTP download result {}", downloadResult); + outputStream.flush(); + outputStream.close(); + ftpClient.logout(); + ftpClient.disconnect(); + } +} diff --git a/jodconverter-web/src/main/java/cn/keking/utils/SimTextUtil.java b/jodconverter-web/src/main/java/cn/keking/utils/SimTextUtil.java index 44c299c7..e7bfde5b 100644 --- a/jodconverter-web/src/main/java/cn/keking/utils/SimTextUtil.java +++ b/jodconverter-web/src/main/java/cn/keking/utils/SimTextUtil.java @@ -1,6 +1,7 @@ package cn.keking.utils; import cn.keking.config.ConfigConstants; +import cn.keking.model.FileAttribute; import cn.keking.model.ReturnResponse; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -12,12 +13,14 @@ import org.springframework.stereotype.Component; */ @Component public class SimTextUtil { - String fileDir = ConfigConstants.getFileDir(); @Autowired - DownloadUtils downloadUtils; + private FileUtils fileUtils; + @Autowired + private DownloadUtils downloadUtils; public ReturnResponse readSimText(String url, String fileName){ - ReturnResponse response = downloadUtils.downLoad(url, "txt", fileName); + FileAttribute fileAttribute = fileUtils.getFileAttribute(url); + ReturnResponse response = downloadUtils.downLoad(fileAttribute, fileName); return response; } }