sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)

同步内容:
- knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback
- services/feedback.py: FAQ max_tokens 200→512
- services/session.py: 消息排序增加 id DESC 次级排序
- knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块
- knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略
- knowledge/document.py: 文档删除时清理孤儿文件
- api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援)
- core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑
- cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
lacerate551
2026-06-21 14:49:46 +08:00
parent d6d27b70f3
commit 258be54df7
9 changed files with 553 additions and 26 deletions

View File

@@ -331,6 +331,21 @@ class CollectionMixin:
except Exception as e:
logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
# 注意:此时 ChromaDB collection 已删除cleanup_image_orphans 会扫描
# 所有剩余 collection仅该 collection 引用的图片会被识别为孤儿
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self)
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存, "
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
if kb_name in self._metadata.get("collections", {}):
del self._metadata["collections"][kb_name]
self._save_metadata()
@@ -361,8 +376,40 @@ class CollectionMixin:
self._metadata = self._load_metadata()
result = []
# 以元数据为唯一真相源,不再扫描磁盘自动补充
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
# 扫描 base_path 下的所有子目录作为向量库
# 每个向量库使用独立目录base_path/my_ky, base_path/public_kb 等
actual_collections = []
try:
if os.path.exists(self.base_path):
for item in os.listdir(self.base_path):
item_path = os.path.join(self.base_path, item)
if os.path.isdir(item_path) and not item.startswith('.'):
# 检查是否包含 chroma.sqlite3有效的向量库目录
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
actual_collections.append(item)
except Exception as e:
logger.warning(f"扫描向量库目录失败: {e}")
# 如果扫描失败,回退到元数据中的集合列表
if not actual_collections:
actual_collections = list(self._metadata.get("collections", {}).keys())
for name in actual_collections:
if name not in self._metadata.get("collections", {}):
if "collections" not in self._metadata:
self._metadata["collections"] = {}
self._metadata["collections"][name] = {
"display_name": name,
"department": "",
"description": "",
"created_at": datetime.now().isoformat()
}
logger.info(f"自动补充向量库元数据: {name}")
self._save_metadata()
stale_collections = []
for name, info in self._metadata.get("collections", {}).items():
try:
collection = self.get_collection(name)
@@ -375,18 +422,17 @@ class CollectionMixin:
description=info.get("description", "")
))
except Exception as e:
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
logger.warning(
f"跳过异常向量库 '{name}': {e}"
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
)
result.append(CollectionInfo(
name=name,
display_name=info.get("display_name", name),
document_count=0,
created_at=info.get("created_at", ""),
department=info.get("department", ""),
description=info.get("description", "")
))
stale_collections.append(name)
# 清理元数据中指向已失效集合的条目
if stale_collections:
for name in stale_collections:
self._metadata.get("collections", {}).pop(name, None)
logger.info(f"清理失效向量库元数据: {name}")
self._save_metadata()
return result