sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)

同步内容:
- knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback
- services/feedback.py: FAQ max_tokens 200→512
- services/session.py: 消息排序增加 id DESC 次级排序
- knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块
- knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略
- knowledge/document.py: 文档删除时清理孤儿文件
- api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援)
- core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑
- cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
lacerate551
2026-06-21 14:49:46 +08:00
parent d6d27b70f3
commit 258be54df7
9 changed files with 553 additions and 26 deletions

140
knowledge/image_cleanup.py Normal file
View File

@@ -0,0 +1,140 @@
"""
图片/VLM缓存孤儿文件清理模块
提供可被 document.py / collection.py 调用的清理函数,
也可被 cleanup_orphans.py 独立脚本使用。
"""
import hashlib
import logging
import os
from pathlib import Path
logger = logging.getLogger(__name__)
IMAGES_DIR = Path(".data/images")
VLM_CACHE_DIR = Path(".data/cache/vlm")
def compute_file_hash(file_path: str) -> str:
"""计算文件 MD5"""
with open(file_path, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
def collect_referenced_images(manager, collections=None) -> set:
"""
从 ChromaDB 收集所有被引用的图片文件名。
Args:
manager: KnowledgeBaseManager 实例
collections: 限定知识库列表None 表示全部
Returns:
set of image filenames (e.g., {"185a7a75d246.png", ...})
"""
referenced = set()
if collections:
kb_names = collections
else:
kb_names = [c.name if hasattr(c, 'name') else str(c)
for c in manager.list_collections()]
for kb_name in kb_names:
try:
col = manager.get_collection(kb_name)
except Exception:
continue
results = col.get(include=['metadatas'])
if not results['ids']:
continue
for meta in results['metadatas']:
image_path = meta.get('image_path', '')
if image_path:
referenced.add(os.path.basename(image_path))
return referenced
def cleanup_image_orphans(manager, collections=None, dry_run=False) -> dict:
"""
清理不再被任何 ChromaDB 切片引用的图片和 VLM 缓存文件。
Args:
manager: KnowledgeBaseManager 实例
collections: 限定知识库列表None 表示全部
dry_run: True 时只返回孤儿列表不实际删除
Returns:
{
'orphan_images': [(filepath, filename, size_bytes)],
'orphan_caches': [(filepath, filename, size_bytes)],
'deleted_images': int,
'deleted_caches': int,
'freed_bytes': int
}
"""
result = {
'orphan_images': [],
'orphan_caches': [],
'deleted_images': 0,
'deleted_caches': 0,
'freed_bytes': 0
}
# 1. 收集引用
referenced = collect_referenced_images(manager, collections)
# 2. 查找孤儿图片
if IMAGES_DIR.exists():
for f in IMAGES_DIR.iterdir():
if f.is_file() and f.name not in referenced:
result['orphan_images'].append((str(f), f.name, f.stat().st_size))
# 3. 查找孤儿 VLM 缓存(图片已删除则缓存也应是孤儿)
if VLM_CACHE_DIR.exists():
referenced_hashes = set()
for filename in referenced:
full_path = IMAGES_DIR / filename
if full_path.exists():
try:
img_hash = compute_file_hash(str(full_path))
referenced_hashes.add(img_hash)
except Exception:
pass
for f in VLM_CACHE_DIR.iterdir():
if f.is_file() and f.suffix == '.txt':
cache_hash = f.stem
if cache_hash not in referenced_hashes:
result['orphan_caches'].append((str(f), f.name, f.stat().st_size))
# 4. 删除
if not dry_run:
for filepath, filename, size in result['orphan_images']:
try:
os.remove(filepath)
result['deleted_images'] += 1
result['freed_bytes'] += size
except OSError as e:
logger.warning(f"删除孤儿图片失败: {filename} - {e}")
for filepath, filename, size in result['orphan_caches']:
try:
os.remove(filepath)
result['deleted_caches'] += 1
result['freed_bytes'] += size
except OSError as e:
logger.warning(f"删除孤儿缓存失败: {filename} - {e}")
if result['deleted_images'] or result['deleted_caches']:
logger.info(
f"清理孤儿: {result['deleted_images']} 图片 + "
f"{result['deleted_caches']} 缓存, "
f"释放 {result['freed_bytes']/1024:.1f} KB"
)
return result