sync(server-release): 从 main 同步逻辑改动(不含 API 格式变更)
同步内容: - knowledge/manager.py: VLM max_tokens 512→2048 + reasoning_content fallback - services/feedback.py: FAQ max_tokens 200→512 - services/session.py: 消息排序增加 id DESC 次级排序 - knowledge/image_cleanup.py: 新增图片/VLM缓存孤儿文件清理模块 - knowledge/collection.py: 集合删除时清理孤儿文件 + list_collections 磁盘扫描策略 - knowledge/document.py: 文档删除时清理孤儿文件 - api/chat_routes.py: 新增 _rescue_bm25_divergence 函数(BM25-CE分歧救援) - core/intent_analyzer.py: 使用 get_intent_client 专用客户端 + 移除短追问缓存跳过逻辑 - cleanup_orphans.py: 独立孤儿文件清理脚本
This commit is contained in:
@@ -331,6 +331,21 @@ class CollectionMixin:
|
||||
except Exception as e:
|
||||
logger.warning(f"清理版本记录失败: {e}")
|
||||
|
||||
# 清理不再被引用的图片和 VLM 缓存文件
|
||||
# 注意:此时 ChromaDB collection 已删除,cleanup_image_orphans 会扫描
|
||||
# 所有剩余 collection,仅该 collection 引用的图片会被识别为孤儿
|
||||
try:
|
||||
from knowledge.image_cleanup import cleanup_image_orphans
|
||||
cleanup_result = cleanup_image_orphans(self)
|
||||
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||
logger.info(
|
||||
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||
f"{cleanup_result['deleted_caches']} VLM缓存, "
|
||||
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理孤儿文件失败: {e}")
|
||||
|
||||
if kb_name in self._metadata.get("collections", {}):
|
||||
del self._metadata["collections"][kb_name]
|
||||
self._save_metadata()
|
||||
@@ -361,8 +376,40 @@ class CollectionMixin:
|
||||
self._metadata = self._load_metadata()
|
||||
result = []
|
||||
|
||||
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
||||
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
||||
# 扫描 base_path 下的所有子目录作为向量库
|
||||
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
||||
actual_collections = []
|
||||
try:
|
||||
if os.path.exists(self.base_path):
|
||||
for item in os.listdir(self.base_path):
|
||||
item_path = os.path.join(self.base_path, item)
|
||||
if os.path.isdir(item_path) and not item.startswith('.'):
|
||||
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
||||
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
||||
actual_collections.append(item)
|
||||
except Exception as e:
|
||||
logger.warning(f"扫描向量库目录失败: {e}")
|
||||
|
||||
# 如果扫描失败,回退到元数据中的集合列表
|
||||
if not actual_collections:
|
||||
actual_collections = list(self._metadata.get("collections", {}).keys())
|
||||
|
||||
for name in actual_collections:
|
||||
if name not in self._metadata.get("collections", {}):
|
||||
if "collections" not in self._metadata:
|
||||
self._metadata["collections"] = {}
|
||||
self._metadata["collections"][name] = {
|
||||
"display_name": name,
|
||||
"department": "",
|
||||
"description": "",
|
||||
"created_at": datetime.now().isoformat()
|
||||
}
|
||||
logger.info(f"自动补充向量库元数据: {name}")
|
||||
|
||||
self._save_metadata()
|
||||
|
||||
stale_collections = []
|
||||
|
||||
for name, info in self._metadata.get("collections", {}).items():
|
||||
try:
|
||||
collection = self.get_collection(name)
|
||||
@@ -375,18 +422,17 @@ class CollectionMixin:
|
||||
description=info.get("description", "")
|
||||
))
|
||||
except Exception as e:
|
||||
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
||||
logger.warning(
|
||||
f"跳过异常向量库 '{name}': {e}"
|
||||
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
|
||||
)
|
||||
result.append(CollectionInfo(
|
||||
name=name,
|
||||
display_name=info.get("display_name", name),
|
||||
document_count=0,
|
||||
created_at=info.get("created_at", ""),
|
||||
department=info.get("department", ""),
|
||||
description=info.get("description", "")
|
||||
))
|
||||
stale_collections.append(name)
|
||||
|
||||
# 清理元数据中指向已失效集合的条目
|
||||
if stale_collections:
|
||||
for name in stale_collections:
|
||||
self._metadata.get("collections", {}).pop(name, None)
|
||||
logger.info(f"清理失效向量库元数据: {name}")
|
||||
self._save_metadata()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
Reference in New Issue
Block a user