fix(knowledge): 全量修复多 worker 元数据竞态问题
1. collection.py: create/delete/update/exists 方法开头加 _load_metadata() 2. list_collections: 去掉磁盘扫描自动补充和 stale 清理逻辑, 改为以元数据为唯一真相源,异常集合只跳过不删不改 3. manager.py: _load/_save_metadata 加 fcntl 文件锁防并发写覆盖, Windows 兼容(fcntl try/import 包裹)
This commit is contained in:
@@ -134,6 +134,9 @@ class CollectionMixin:
|
||||
"""
|
||||
from .base import BM25Index
|
||||
|
||||
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||
self._metadata = self._load_metadata()
|
||||
|
||||
if not kb_name or not kb_name.replace('_', '').isalnum():
|
||||
return False, "向量库名称只能包含字母、数字和下划线"
|
||||
|
||||
@@ -190,6 +193,9 @@ class CollectionMixin:
|
||||
Returns:
|
||||
更新成功返回 True,向量库不存在返回 False
|
||||
"""
|
||||
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||
self._metadata = self._load_metadata()
|
||||
|
||||
collections = self._metadata.get("collections", {})
|
||||
if kb_name not in collections:
|
||||
return False
|
||||
@@ -228,6 +234,9 @@ class CollectionMixin:
|
||||
"""
|
||||
import shutil
|
||||
|
||||
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||
self._metadata = self._load_metadata()
|
||||
|
||||
if kb_name == PUBLIC_KB_NAME:
|
||||
return False, "公开知识库不能删除"
|
||||
|
||||
@@ -352,40 +361,8 @@ class CollectionMixin:
|
||||
self._metadata = self._load_metadata()
|
||||
result = []
|
||||
|
||||
# 扫描 base_path 下的所有子目录作为向量库
|
||||
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
||||
actual_collections = []
|
||||
try:
|
||||
if os.path.exists(self.base_path):
|
||||
for item in os.listdir(self.base_path):
|
||||
item_path = os.path.join(self.base_path, item)
|
||||
if os.path.isdir(item_path) and not item.startswith('.'):
|
||||
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
||||
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
||||
actual_collections.append(item)
|
||||
except Exception as e:
|
||||
logger.warning(f"扫描向量库目录失败: {e}")
|
||||
|
||||
# 如果扫描失败,回退到元数据中的集合列表
|
||||
if not actual_collections:
|
||||
actual_collections = list(self._metadata.get("collections", {}).keys())
|
||||
|
||||
for name in actual_collections:
|
||||
if name not in self._metadata.get("collections", {}):
|
||||
if "collections" not in self._metadata:
|
||||
self._metadata["collections"] = {}
|
||||
self._metadata["collections"][name] = {
|
||||
"display_name": name,
|
||||
"department": "",
|
||||
"description": "",
|
||||
"created_at": datetime.now().isoformat()
|
||||
}
|
||||
logger.info(f"自动补充向量库元数据: {name}")
|
||||
|
||||
self._save_metadata()
|
||||
|
||||
stale_collections = []
|
||||
|
||||
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
||||
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
||||
for name, info in self._metadata.get("collections", {}).items():
|
||||
try:
|
||||
collection = self.get_collection(name)
|
||||
@@ -398,17 +375,18 @@ class CollectionMixin:
|
||||
description=info.get("description", "")
|
||||
))
|
||||
except Exception as e:
|
||||
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
||||
logger.warning(
|
||||
f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失"
|
||||
f"跳过异常向量库 '{name}': {e}"
|
||||
)
|
||||
stale_collections.append(name)
|
||||
|
||||
# 清理元数据中指向已失效集合的条目
|
||||
if stale_collections:
|
||||
for name in stale_collections:
|
||||
self._metadata.get("collections", {}).pop(name, None)
|
||||
logger.info(f"清理失效向量库元数据: {name}")
|
||||
self._save_metadata()
|
||||
result.append(CollectionInfo(
|
||||
name=name,
|
||||
display_name=info.get("display_name", name),
|
||||
document_count=0,
|
||||
created_at=info.get("created_at", ""),
|
||||
department=info.get("department", ""),
|
||||
description=info.get("description", "")
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
@@ -422,4 +400,6 @@ class CollectionMixin:
|
||||
Returns:
|
||||
存在返回 True,不存在返回 False
|
||||
"""
|
||||
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||
self._metadata = self._load_metadata()
|
||||
return kb_name in self._metadata.get("collections", {})
|
||||
|
||||
Reference in New Issue
Block a user