chore: 元数据并发安全增强与重排序配置更新
- knowledge: 写操作方法增加元数据重载,manager 增加 fcntl 文件锁 - config/engine: 重排序默认后端切换为云端,新增云端配置项
This commit is contained in:
@@ -29,6 +29,11 @@
|
||||
import os
|
||||
import json
|
||||
import threading
|
||||
try:
|
||||
import fcntl
|
||||
_HAS_FCNTL = True
|
||||
except ImportError:
|
||||
_HAS_FCNTL = False # Windows 环境无 fcntl
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from pathlib import Path
|
||||
import logging
|
||||
@@ -134,22 +139,36 @@ class KnowledgeBaseManager(
|
||||
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
||||
|
||||
def _load_metadata(self) -> dict:
|
||||
"""加载元数据"""
|
||||
"""加载元数据(带文件锁,确保多 worker 进程间一致)"""
|
||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||
if os.path.exists(metadata_path):
|
||||
try:
|
||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
|
||||
try:
|
||||
return json.load(f)
|
||||
finally:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||
except Exception as e:
|
||||
logger.error(f"加载元数据失败: {e}")
|
||||
return {"collections": {}}
|
||||
|
||||
def _save_metadata(self):
|
||||
"""保存元数据"""
|
||||
"""保存元数据(带文件锁,防止并发写入数据覆盖)"""
|
||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||
try:
|
||||
with open(metadata_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
finally:
|
||||
if _HAS_FCNTL:
|
||||
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||
except Exception as e:
|
||||
logger.error(f"保存元数据失败: {e}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user