chore: 元数据并发安全增强与重排序配置更新
- knowledge: 写操作方法增加元数据重载,manager 增加 fcntl 文件锁 - config/engine: 重排序默认后端切换为云端,新增云端配置项
This commit is contained in:
@@ -106,6 +106,11 @@ RECALL_MULTIPLIER = 3 # 候选池最小倍数 = top_k * 此值
|
|||||||
|
|
||||||
# ----- 重排序 -----
|
# ----- 重排序 -----
|
||||||
USE_RERANK = True
|
USE_RERANK = True
|
||||||
|
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "cloud") # cloud / local / fallback
|
||||||
|
RERANK_CLOUD_MODEL = os.getenv("RERANK_CLOUD_MODEL", "xop3qwen8breranker")
|
||||||
|
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", "")
|
||||||
|
RERANK_CLOUD_BASE_URL = os.getenv("RERANK_CLOUD_BASE_URL", "https://maas-api.cn-huabei-1.xf-yun.com/v2/rerank")
|
||||||
|
RERANK_CLOUD_TIMEOUT = int(os.getenv("RERANK_CLOUD_TIMEOUT", "15"))
|
||||||
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
||||||
RERANK_TOP_K = 15 # 重排序后保留数
|
RERANK_TOP_K = 15 # 重排序后保留数
|
||||||
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
||||||
|
|||||||
@@ -112,10 +112,10 @@ except ImportError:
|
|||||||
EMBEDDING_DEVICE = "auto"
|
EMBEDDING_DEVICE = "auto"
|
||||||
RERANK_DEVICE = "auto"
|
RERANK_DEVICE = "auto"
|
||||||
RERANK_USE_ONNX = False
|
RERANK_USE_ONNX = False
|
||||||
RERANK_BACKEND = "local"
|
RERANK_BACKEND = "cloud"
|
||||||
RERANK_CLOUD_MODEL = "xop3qwen8breranker"
|
RERANK_CLOUD_MODEL = "xop3qwen8breranker"
|
||||||
RERANK_CLOUD_API_KEY = ""
|
RERANK_CLOUD_API_KEY = ""
|
||||||
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
RERANK_CLOUD_BASE_URL = "https://maas-api.cn-huabei-1.xf-yun.com/v2/rerank"
|
||||||
RERANK_CLOUD_TIMEOUT = 15
|
RERANK_CLOUD_TIMEOUT = 15
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -134,6 +134,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
from .base import BM25Index
|
from .base import BM25Index
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if not kb_name or not kb_name.replace('_', '').isalnum():
|
if not kb_name or not kb_name.replace('_', '').isalnum():
|
||||||
return False, "向量库名称只能包含字母、数字和下划线"
|
return False, "向量库名称只能包含字母、数字和下划线"
|
||||||
|
|
||||||
@@ -190,6 +193,9 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
更新成功返回 True,向量库不存在返回 False
|
更新成功返回 True,向量库不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
collections = self._metadata.get("collections", {})
|
collections = self._metadata.get("collections", {})
|
||||||
if kb_name not in collections:
|
if kb_name not in collections:
|
||||||
return False
|
return False
|
||||||
@@ -228,6 +234,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if kb_name == PUBLIC_KB_NAME:
|
if kb_name == PUBLIC_KB_NAME:
|
||||||
return False, "公开知识库不能删除"
|
return False, "公开知识库不能删除"
|
||||||
|
|
||||||
@@ -422,4 +431,6 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
存在返回 True,不存在返回 False
|
存在返回 True,不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
return kb_name in self._metadata.get("collections", {})
|
return kb_name in self._metadata.get("collections", {})
|
||||||
|
|||||||
@@ -29,6 +29,11 @@
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import threading
|
import threading
|
||||||
|
try:
|
||||||
|
import fcntl
|
||||||
|
_HAS_FCNTL = True
|
||||||
|
except ImportError:
|
||||||
|
_HAS_FCNTL = False # Windows 环境无 fcntl
|
||||||
from typing import List, Dict, Optional, Tuple
|
from typing import List, Dict, Optional, Tuple
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import logging
|
import logging
|
||||||
@@ -134,22 +139,36 @@ class KnowledgeBaseManager(
|
|||||||
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
||||||
|
|
||||||
def _load_metadata(self) -> dict:
|
def _load_metadata(self) -> dict:
|
||||||
"""加载元数据"""
|
"""加载元数据(带文件锁,确保多 worker 进程间一致)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
if os.path.exists(metadata_path):
|
if os.path.exists(metadata_path):
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
|
||||||
|
try:
|
||||||
return json.load(f)
|
return json.load(f)
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"加载元数据失败: {e}")
|
logger.error(f"加载元数据失败: {e}")
|
||||||
return {"collections": {}}
|
return {"collections": {}}
|
||||||
|
|
||||||
def _save_metadata(self):
|
def _save_metadata(self):
|
||||||
"""保存元数据"""
|
"""保存元数据(带文件锁,防止并发写入数据覆盖)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'w', encoding='utf-8') as f:
|
with open(metadata_path, 'w', encoding='utf-8') as f:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
||||||
|
f.flush()
|
||||||
|
os.fsync(f.fileno())
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"保存元数据失败: {e}")
|
logger.error(f"保存元数据失败: {e}")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user