25.09.08
parent
f5f1c4e0b0
commit
24df67a6a2
|
@ -1,4 +1 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from .favicon_async import FaviconAsync
|
|
||||||
from .favicon_service_async import FaviconServiceAsync
|
|
||||||
|
|
|
@ -97,7 +97,7 @@ class FaviconAsync(favicon.Favicon):
|
||||||
元组(内容, 内容类型)
|
元组(内容, 内容类型)
|
||||||
"""
|
"""
|
||||||
global _aiohttp_client
|
global _aiohttp_client
|
||||||
logger.info('发送异步请求: %s', url)
|
logger.debug('发送异步请求: %s', url)
|
||||||
|
|
||||||
# 初始化aiohttp客户端会话
|
# 初始化aiohttp客户端会话
|
||||||
if _aiohttp_client is None:
|
if _aiohttp_client is None:
|
||||||
|
|
|
@ -27,12 +27,6 @@ class FaviconServiceAsync(favicon_service.FaviconService):
|
||||||
icon_content = None
|
icon_content = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if entity.domain in self.domain_list:
|
|
||||||
self._queue_pull(True, self.total_queue)
|
|
||||||
return _cached or setting.default_icon_file
|
|
||||||
else:
|
|
||||||
self.domain_list.append(entity.domain)
|
|
||||||
|
|
||||||
# 尝试从网站异步获取HTML内容
|
# 尝试从网站异步获取HTML内容
|
||||||
html_content = await entity.async_req_get()
|
html_content = await entity.async_req_get()
|
||||||
if html_content:
|
if html_content:
|
||||||
|
@ -60,7 +54,7 @@ class FaviconServiceAsync(favicon_service.FaviconService):
|
||||||
|
|
||||||
strategy_url, strategy_name = strategy()
|
strategy_url, strategy_name = strategy()
|
||||||
if strategy_url is not None:
|
if strategy_url is not None:
|
||||||
logger.info(f"-> 异步尝试从 {strategy_name} 获取图标")
|
logger.debug(f"-> 异步尝试从 {strategy_name} 获取图标")
|
||||||
icon_content, icon_type = await entity.async_get_icon_file(strategy_url, strategy_url == '')
|
icon_content, icon_type = await entity.async_get_icon_file(strategy_url, strategy_url == '')
|
||||||
|
|
||||||
# 图标获取失败,或图标不是支持的图片格式,写入默认图标
|
# 图标获取失败,或图标不是支持的图片格式,写入默认图标
|
||||||
|
@ -90,8 +84,6 @@ class FaviconServiceAsync(favicon_service.FaviconService):
|
||||||
logger.error(f"异步获取图标时发生错误 {entity.domain}: {e}")
|
logger.error(f"异步获取图标时发生错误 {entity.domain}: {e}")
|
||||||
return _cached or setting.default_icon_file
|
return _cached or setting.default_icon_file
|
||||||
finally:
|
finally:
|
||||||
if entity.domain in self.domain_list:
|
|
||||||
self.domain_list.remove(entity.domain)
|
|
||||||
# 任务完成,从两个队列中移出元素
|
# 任务完成,从两个队列中移出元素
|
||||||
self._queue_pull(True, self.total_queue)
|
self._queue_pull(True, self.total_queue)
|
||||||
|
|
||||||
|
|
|
@ -255,7 +255,7 @@ class Favicon:
|
||||||
Returns:
|
Returns:
|
||||||
元组(内容, 内容类型)
|
元组(内容, 内容类型)
|
||||||
"""
|
"""
|
||||||
logger.info('发送请求: %s', url)
|
logger.debug('发送请求: %s', url)
|
||||||
|
|
||||||
retry_count = 0
|
retry_count = 0
|
||||||
while retry_count <= retries:
|
while retry_count <= retries:
|
||||||
|
|
|
@ -40,12 +40,8 @@ class FaviconService:
|
||||||
self.url_count = 0
|
self.url_count = 0
|
||||||
self.request_icon_count = 0
|
self.request_icon_count = 0
|
||||||
self.request_cache_count = 0
|
self.request_cache_count = 0
|
||||||
self.domain_list: List[str] = list()
|
|
||||||
|
|
||||||
# 初始化队列
|
# 初始化队列
|
||||||
# 实时处理的任务数量
|
|
||||||
# self.icon_queue = Queue()
|
|
||||||
# 所有正在处理的任务数量
|
|
||||||
self.total_queue = Queue()
|
self.total_queue = Queue()
|
||||||
|
|
||||||
# # 队列阈值常量配置
|
# # 队列阈值常量配置
|
||||||
|
@ -238,7 +234,7 @@ class FaviconService:
|
||||||
self._get_link_rel(html_links, entity, ''))
|
self._get_link_rel(html_links, entity, ''))
|
||||||
|
|
||||||
if icon_url:
|
if icon_url:
|
||||||
logger.info(f"-> 从HTML获取图标URL: {icon_url}")
|
logger.debug(f"-> 从HTML获取图标URL: {icon_url}")
|
||||||
|
|
||||||
return icon_url
|
return icon_url
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -270,12 +266,6 @@ class FaviconService:
|
||||||
icon_content = None
|
icon_content = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if entity.domain in self.domain_list:
|
|
||||||
self._queue_pull(True, self.total_queue)
|
|
||||||
return _cached or setting.default_icon_file
|
|
||||||
else:
|
|
||||||
self.domain_list.append(entity.domain)
|
|
||||||
|
|
||||||
# 尝试从网站获取HTML内容
|
# 尝试从网站获取HTML内容
|
||||||
html_content = entity.req_get()
|
html_content = entity.req_get()
|
||||||
if html_content:
|
if html_content:
|
||||||
|
@ -305,7 +295,7 @@ class FaviconService:
|
||||||
|
|
||||||
strategy_url, strategy_name = strategy()
|
strategy_url, strategy_name = strategy()
|
||||||
if strategy_url is not None:
|
if strategy_url is not None:
|
||||||
logger.info(f"-> 尝试从 {strategy_name} 获取图标")
|
logger.debug(f"-> 尝试从 {strategy_name} 获取图标")
|
||||||
icon_content, icon_type = entity.get_icon_file(strategy_url, strategy_url == '')
|
icon_content, icon_type = entity.get_icon_file(strategy_url, strategy_url == '')
|
||||||
|
|
||||||
# 图标获取失败,或图标不是支持的图片格式,写入默认图标
|
# 图标获取失败,或图标不是支持的图片格式,写入默认图标
|
||||||
|
@ -335,8 +325,6 @@ class FaviconService:
|
||||||
logger.error(f"获取图标时发生错误 {entity.domain}: {e}")
|
logger.error(f"获取图标时发生错误 {entity.domain}: {e}")
|
||||||
return _cached or setting.default_icon_file
|
return _cached or setting.default_icon_file
|
||||||
finally:
|
finally:
|
||||||
if entity.domain in self.domain_list:
|
|
||||||
self.domain_list.remove(entity.domain)
|
|
||||||
# 任务完成,从两个队列中移出元素
|
# 任务完成,从两个队列中移出元素
|
||||||
self._queue_pull(True, self.total_queue)
|
self._queue_pull(True, self.total_queue)
|
||||||
|
|
||||||
|
@ -347,7 +335,6 @@ class FaviconService:
|
||||||
'request_icon_count': self.request_icon_count,
|
'request_icon_count': self.request_icon_count,
|
||||||
'request_cache_count': self.request_cache_count,
|
'request_cache_count': self.request_cache_count,
|
||||||
'queue_size': self.total_queue.qsize(),
|
'queue_size': self.total_queue.qsize(),
|
||||||
'domain_list': len(self.domain_list),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def get_favicon_handler(
|
def get_favicon_handler(
|
||||||
|
|
|
@ -80,7 +80,7 @@ class FileUtil:
|
||||||
if not FileUtil._validate_path(path):
|
if not FileUtil._validate_path(path):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
logger.info(f"开始遍历目录: {path}, 递归: {recursive}, 最小文件大小: {min_size}字节")
|
logger.debug(f"开始遍历目录: {path}, 递归: {recursive}, 最小文件大小: {min_size}字节")
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
if recursive:
|
if recursive:
|
||||||
|
@ -97,7 +97,7 @@ class FileUtil:
|
||||||
continue
|
continue
|
||||||
FileUtil._process_file(path, filename, min_size, include_size, result)
|
FileUtil._process_file(path, filename, min_size, include_size, result)
|
||||||
|
|
||||||
logger.info(f"目录遍历完成: {path}, 找到文件数: {len(result)}")
|
logger.debug(f"目录遍历完成: {path}, 找到文件数: {len(result)}")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -124,7 +124,7 @@ class FileUtil:
|
||||||
if not FileUtil._validate_path(path):
|
if not FileUtil._validate_path(path):
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
logger.info(f"开始构建文件字典: {path}")
|
logger.debug(f"开始构建文件字典: {path}")
|
||||||
file_dict = {}
|
file_dict = {}
|
||||||
|
|
||||||
for root, _, files in os.walk(path):
|
for root, _, files in os.walk(path):
|
||||||
|
@ -148,7 +148,7 @@ class FileUtil:
|
||||||
if not recursive:
|
if not recursive:
|
||||||
break
|
break
|
||||||
|
|
||||||
logger.info(f"文件字典构建完成: {path}, 文件数: {len(file_dict)}")
|
logger.debug(f"文件字典构建完成: {path}, 文件数: {len(file_dict)}")
|
||||||
return file_dict
|
return file_dict
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
@ -6,7 +6,6 @@ import threading
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Optional
|
||||||
|
|
||||||
# 配置日志
|
# 配置日志
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue