diff --git a/knowledge/collection.py b/knowledge/collection.py index 26b3ee1..53566e2 100644 --- a/knowledge/collection.py +++ b/knowledge/collection.py @@ -134,6 +134,9 @@ class CollectionMixin: """ from .base import BM25Index + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + if not kb_name or not kb_name.replace('_', '').isalnum(): return False, "向量库名称只能包含字母、数字和下划线" @@ -190,6 +193,9 @@ class CollectionMixin: Returns: 更新成功返回 True,向量库不存在返回 False """ + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + collections = self._metadata.get("collections", {}) if kb_name not in collections: return False @@ -228,6 +234,9 @@ class CollectionMixin: """ import shutil + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + if kb_name == PUBLIC_KB_NAME: return False, "公开知识库不能删除" @@ -352,40 +361,8 @@ class CollectionMixin: self._metadata = self._load_metadata() result = [] - # 扫描 base_path 下的所有子目录作为向量库 - # 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等 - actual_collections = [] - try: - if os.path.exists(self.base_path): - for item in os.listdir(self.base_path): - item_path = os.path.join(self.base_path, item) - if os.path.isdir(item_path) and not item.startswith('.'): - # 检查是否包含 chroma.sqlite3(有效的向量库目录) - if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')): - actual_collections.append(item) - except Exception as e: - logger.warning(f"扫描向量库目录失败: {e}") - - # 如果扫描失败,回退到元数据中的集合列表 - if not actual_collections: - actual_collections = list(self._metadata.get("collections", {}).keys()) - - for name in actual_collections: - if name not in self._metadata.get("collections", {}): - if "collections" not in self._metadata: - self._metadata["collections"] = {} - self._metadata["collections"][name] = { - "display_name": name, - "department": "", - "description": "", - "created_at": datetime.now().isoformat() - } - logger.info(f"自动补充向量库元数据: {name}") - - self._save_metadata() - - stale_collections = [] - + # 以元数据为唯一真相源,不再扫描磁盘自动补充 + # (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作) for name, info in self._metadata.get("collections", {}).items(): try: collection = self.get_collection(name) @@ -398,17 +375,18 @@ class CollectionMixin: description=info.get("description", "") )) except Exception as e: + # 只跳过不修改元数据(可能是其他 worker 刚创建的集合) logger.warning( - f"跳过异常向量库 '{name}': {e},可能是 ChromaDB 数据目录已丢失" + f"跳过异常向量库 '{name}': {e}" ) - stale_collections.append(name) - - # 清理元数据中指向已失效集合的条目 - if stale_collections: - for name in stale_collections: - self._metadata.get("collections", {}).pop(name, None) - logger.info(f"清理失效向量库元数据: {name}") - self._save_metadata() + result.append(CollectionInfo( + name=name, + display_name=info.get("display_name", name), + document_count=0, + created_at=info.get("created_at", ""), + department=info.get("department", ""), + description=info.get("description", "") + )) return result @@ -422,4 +400,6 @@ class CollectionMixin: Returns: 存在返回 True,不存在返回 False """ + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() return kb_name in self._metadata.get("collections", {}) diff --git a/knowledge/manager.py b/knowledge/manager.py index 31a8e86..346698c 100644 --- a/knowledge/manager.py +++ b/knowledge/manager.py @@ -29,6 +29,11 @@ import os import json import threading +try: + import fcntl + _HAS_FCNTL = True +except ImportError: + _HAS_FCNTL = False from typing import List, Dict, Optional, Tuple from pathlib import Path import logging @@ -134,22 +139,36 @@ class KnowledgeBaseManager( logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}") def _load_metadata(self) -> dict: - """加载元数据""" + """加载元数据(带文件锁)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) if os.path.exists(metadata_path): try: with open(metadata_path, 'r', encoding='utf-8') as f: - return json.load(f) + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) + try: + return json.load(f) + finally: + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) except Exception as e: logger.error(f"加载元数据失败: {e}") return {"collections": {}} def _save_metadata(self): - """保存元数据""" + """保存元数据(带文件锁,防止并发写入覆盖)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) try: with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(self._metadata, f, ensure_ascii=False, indent=2) + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + try: + json.dump(self._metadata, f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + finally: + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) except Exception as e: logger.error(f"保存元数据失败: {e}")