sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)
同步内容: - knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback - services/feedback.py: FAQ max_tokens 200→512 - services/session.py: 消息排序增加 id DESC 次级排序 - knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块 - knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略 - knowledge/document.py: 文档删除时清理孤儿文件 - api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援) - core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑 - cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
140
knowledge/image_cleanup.py
Normal file
140
knowledge/image_cleanup.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
图片/VLM缓存孤儿文件清理模块
|
||||
|
||||
提供可被 document.py / collection.py 调用的清理函数,
|
||||
也可被 cleanup_orphans.py 独立脚本使用。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGES_DIR = Path(".data/images")
|
||||
VLM_CACHE_DIR = Path(".data/cache/vlm")
|
||||
|
||||
|
||||
def compute_file_hash(file_path: str) -> str:
|
||||
"""计算文件 MD5"""
|
||||
with open(file_path, 'rb') as f:
|
||||
return hashlib.md5(f.read()).hexdigest()
|
||||
|
||||
|
||||
def collect_referenced_images(manager, collections=None) -> set:
|
||||
"""
|
||||
从 ChromaDB 收集所有被引用的图片文件名。
|
||||
|
||||
Args:
|
||||
manager: KnowledgeBaseManager 实例
|
||||
collections: 限定知识库列表,None 表示全部
|
||||
|
||||
Returns:
|
||||
set of image filenames (e.g., {"185a7a75d246.png", ...})
|
||||
"""
|
||||
referenced = set()
|
||||
|
||||
if collections:
|
||||
kb_names = collections
|
||||
else:
|
||||
kb_names = [c.name if hasattr(c, 'name') else str(c)
|
||||
for c in manager.list_collections()]
|
||||
|
||||
for kb_name in kb_names:
|
||||
try:
|
||||
col = manager.get_collection(kb_name)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
results = col.get(include=['metadatas'])
|
||||
if not results['ids']:
|
||||
continue
|
||||
|
||||
for meta in results['metadatas']:
|
||||
image_path = meta.get('image_path', '')
|
||||
if image_path:
|
||||
referenced.add(os.path.basename(image_path))
|
||||
|
||||
return referenced
|
||||
|
||||
|
||||
def cleanup_image_orphans(manager, collections=None, dry_run=False) -> dict:
|
||||
"""
|
||||
清理不再被任何 ChromaDB 切片引用的图片和 VLM 缓存文件。
|
||||
|
||||
Args:
|
||||
manager: KnowledgeBaseManager 实例
|
||||
collections: 限定知识库列表,None 表示全部
|
||||
dry_run: True 时只返回孤儿列表不实际删除
|
||||
|
||||
Returns:
|
||||
{
|
||||
'orphan_images': [(filepath, filename, size_bytes)],
|
||||
'orphan_caches': [(filepath, filename, size_bytes)],
|
||||
'deleted_images': int,
|
||||
'deleted_caches': int,
|
||||
'freed_bytes': int
|
||||
}
|
||||
"""
|
||||
result = {
|
||||
'orphan_images': [],
|
||||
'orphan_caches': [],
|
||||
'deleted_images': 0,
|
||||
'deleted_caches': 0,
|
||||
'freed_bytes': 0
|
||||
}
|
||||
|
||||
# 1. 收集引用
|
||||
referenced = collect_referenced_images(manager, collections)
|
||||
|
||||
# 2. 查找孤儿图片
|
||||
if IMAGES_DIR.exists():
|
||||
for f in IMAGES_DIR.iterdir():
|
||||
if f.is_file() and f.name not in referenced:
|
||||
result['orphan_images'].append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
# 3. 查找孤儿 VLM 缓存(图片已删除则缓存也应是孤儿)
|
||||
if VLM_CACHE_DIR.exists():
|
||||
referenced_hashes = set()
|
||||
for filename in referenced:
|
||||
full_path = IMAGES_DIR / filename
|
||||
if full_path.exists():
|
||||
try:
|
||||
img_hash = compute_file_hash(str(full_path))
|
||||
referenced_hashes.add(img_hash)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for f in VLM_CACHE_DIR.iterdir():
|
||||
if f.is_file() and f.suffix == '.txt':
|
||||
cache_hash = f.stem
|
||||
if cache_hash not in referenced_hashes:
|
||||
result['orphan_caches'].append((str(f), f.name, f.stat().st_size))
|
||||
|
||||
# 4. 删除
|
||||
if not dry_run:
|
||||
for filepath, filename, size in result['orphan_images']:
|
||||
try:
|
||||
os.remove(filepath)
|
||||
result['deleted_images'] += 1
|
||||
result['freed_bytes'] += size
|
||||
except OSError as e:
|
||||
logger.warning(f"删除孤儿图片失败: {filename} - {e}")
|
||||
|
||||
for filepath, filename, size in result['orphan_caches']:
|
||||
try:
|
||||
os.remove(filepath)
|
||||
result['deleted_caches'] += 1
|
||||
result['freed_bytes'] += size
|
||||
except OSError as e:
|
||||
logger.warning(f"删除孤儿缓存失败: {filename} - {e}")
|
||||
|
||||
if result['deleted_images'] or result['deleted_caches']:
|
||||
logger.info(
|
||||
f"清理孤儿: {result['deleted_images']} 图片 + "
|
||||
f"{result['deleted_caches']} 缓存, "
|
||||
f"释放 {result['freed_bytes']/1024:.1f} KB"
|
||||
)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user