jinql
/
favicon-api-v3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

25.09.06

master
jinql 2025-09-06 20:45:47 +08:00
parent a92dd232ee
commit 3102ce8b8e
4 changed files with 29 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
favicon_app/models/favicon.py
View File

@ -360,5 +360,5 @@ _pattern_domain = re.compile(
def failed_url_cache(_domain: str, _time: int): def failed_url_cache(_domain: str, _time: int):
if _domain: if _domain:
_current_time = int(time.time()) _current_time = int(time.time())
if (not failed_urls[_domain]) or (_current_time <= failed_urls[_domain]): if (not failed_urls.get(_domain)) or (_current_time <= failed_urls.get(_domain)):
failed_urls[_domain] = _current_time + _time failed_urls[_domain] = _current_time + _time

2
favicon_app/routes/favicon_routes.py
View File

@ -63,7 +63,7 @@ async def get_referrer(unique: Optional[str] = Query(None)):
try: try:
content = FileUtil.read_file(path, mode='r') or 'None' content = FileUtil.read_file(path, mode='r') or 'None'
if unique in ['true', '1', 'True']: if unique in ['true', '1']:
lines = [line.strip() for line in content.split('\n') if line.strip()] lines = [line.strip() for line in content.split('\n') if line.strip()]
unique_lines = list(set(lines)) unique_lines = list(set(lines))
unique_content = '\n'.join(unique_lines) unique_content = '\n'.join(unique_lines)

45
favicon_app/routes/favicon_service.py
View File

@ -7,7 +7,6 @@ import random
import re import re
import time import time
from queue import Queue from queue import Queue
from threading import Lock
from typing import Optional, Tuple, Dict, List from typing import Optional, Tuple, Dict, List
import bs4 import bs4
@ -350,6 +349,7 @@ class FaviconService:
'request_cache_count': self.request_cache_count, 'request_cache_count': self.request_cache_count,
'queue_size': self.icon_queue.qsize(), 'queue_size': self.icon_queue.qsize(),
'total_queue_size': self.total_queue.qsize(), 'total_queue_size': self.total_queue.qsize(),
'domain_list': self.domain_list,
} }
async def get_favicon_handler( async def get_favicon_handler(
@ -362,7 +362,7 @@ class FaviconService:
) -> dict[str, str] | Response: ) -> dict[str, str] | Response:
"""处理获取图标的请求""" """处理获取图标的请求"""
logger.info(f"队列大小：{self.icon_queue.qsize()} | {self.total_queue.qsize()} | {len(favicon.failed_urls)}") logger.info(f"队列大小 icon/total/failed：{self.icon_queue.qsize()} | {self.total_queue.qsize()} | {len(favicon.failed_urls)}")
self.url_count += 1 self.url_count += 1
@ -380,18 +380,18 @@ class FaviconService:
# 检查内存缓存中的失败URL # 检查内存缓存中的失败URL
if entity.domain in favicon.failed_urls: if entity.domain in favicon.failed_urls:
_expire_time = favicon.failed_urls[entity.domain] _expire_time = favicon.failed_urls.get(entity.domain)
if int(time.time()) <= _expire_time: if int(time.time()) <= _expire_time:
return self.get_default(self.time_of_7_days) return self.get_default(self.time_of_7_days)
else: else:
del favicon.failed_urls[entity.domain] del favicon.failed_urls[entity.domain]
# 检查缓存 # 检查缓存
_cached, cached_icon = self._get_cache_icon(entity.domain_md5, refresh=refresh in ['true', '1', 'True']) _cached, cached_icon = self._get_cache_icon(entity.domain_md5, refresh=refresh in ['true', '1'])
if cached_icon: if _cached or cached_icon:
# 使用缓存图标 # 使用缓存图标
icon_content = cached_icon icon_content = cached_icon if cached_icon else _cached
self.request_cache_count += 1 self.request_cache_count += 1
# 确定内容类型和缓存时间 # 确定内容类型和缓存时间
@ -412,38 +412,32 @@ class FaviconService:
media_type=content_type if content_type else "image/x-icon", media_type=content_type if content_type else "image/x-icon",
headers=self._get_header(content_type, cache_time)) headers=self._get_header(content_type, cache_time))
else: else:
_referrer = request.headers.get('referrer') or request.headers.get('referer') # 开始图标处理，加入两个队列
self.icon_queue.put(entity.domain)
self.total_queue.put(entity.domain)
# 检查sync参数 # 检查sync参数
is_sync = sync in ['true', '1', 'True'] is_sync = sync in ['true', '1']
if (not is_sync) or (_referrer and not _referrer.startswith('https://api.xinac.net')): if (not is_sync) or (not check_referer(request)):
# 返回默认图片并加入后台队列 # 返回默认图片并加入后台队列
logger.info(f"返回默认图片并加入后台队列: {entity.domain}") logger.info(f"返回默认图片并加入后台队列: {entity.domain}")
# 开始图标处理，加入两个队列
self.icon_queue.put(entity.domain)
self.total_queue.put(entity.domain)
bg_tasks.add_task(self.get_icon_sync, entity, _cached) bg_tasks.add_task(self.get_icon_sync, entity, _cached)
return self.get_default() return self.get_default(0)
else: else:
# 没有缓存，实时处理，检查队列大小 # 没有缓存，实时处理，检查队列大小
queue_size = self.icon_queue.qsize() queue_size = self.icon_queue.qsize()
if queue_size >= self.MAX_QUEUE_SIZE: if queue_size >= self.MAX_QUEUE_SIZE:
# 加入后台队列并返回默认图片 # 加入后台队列并返回默认图片
logger.info(f"队列大小({queue_size})>={self.MAX_QUEUE_SIZE}，返回默认图片并加入后台队列: {entity.domain}") logger.info(f"队列大小({queue_size})>={self.MAX_QUEUE_SIZE}，返回默认图片并加入后台队列: {entity.domain}")
# 开始图标处理，加入两个队列
self.icon_queue.put(entity.domain)
self.total_queue.put(entity.domain)
bg_tasks.add_task(self.get_icon_sync, entity, _cached) bg_tasks.add_task(self.get_icon_sync, entity, _cached)
return self.get_default() return self.get_default(0)
else: else:
# 队列<MAX_QUEUE_SIZE，实时处理 # 队列<MAX_QUEUE_SIZE，实时处理
logger.info(f"队列大小({queue_size})<{self.MAX_QUEUE_SIZE}，实时处理: {entity.domain}") logger.info(f"队列大小({queue_size})<{self.MAX_QUEUE_SIZE}，实时处理: {entity.domain}")
# 开始图标处理，加入两个队列
self.icon_queue.put(entity.domain)
self.total_queue.put(entity.domain)
icon_content = self.get_icon_sync(entity, _cached) icon_content = self.get_icon_sync(entity, _cached)
if not icon_content: if not icon_content:
# 获取失败，返回默认图标，不缓存 # 获取失败，返回默认图标
return self.get_default() return self.get_default()
# 确定内容类型和缓存时间 # 确定内容类型和缓存时间
@ -467,3 +461,12 @@ class FaviconService:
return Response(content=default_icon_content, return Response(content=default_icon_content,
media_type="image/png", media_type="image/png",
headers=self._get_header("image/png", cache_time)) headers=self._get_header("image/png", cache_time))
def check_referer(request: Request) -> bool:
_referer = request.headers.get('referrer') or request.headers.get('referer')
if _referer:
return (_referer.startswith('https://api.xinac.net')
or _referer.startswith('http://127.0.0.1')
or _referer.startswith('http://localhost'))
return False

6
main.py
View File

@ -30,9 +30,9 @@ app.include_router(favicon_router)
@app.middleware("http") @app.middleware("http")
async def log_referer(request: Request, call_next): async def log_referer(request: Request, call_next):
_referrer = request.headers.get('referrer') or request.headers.get('referer') _referer = request.headers.get('referrer') or request.headers.get('referer')
if _referrer: if _referer:
FileUtil.write_file(referer_log_file, '%s\n' % _referrer, mode='a') FileUtil.write_file(referer_log_file, '%s\n' % _referer, mode='a')
response = await call_next(request) response = await call_next(request)
return response return response