diff --git a/config.example.py b/config.example.py index 0954970..d69de81 100644 --- a/config.example.py +++ b/config.example.py @@ -106,6 +106,11 @@ RECALL_MULTIPLIER = 3 # 候选池最小倍数 = top_k * 此值 # ----- 重排序 ----- USE_RERANK = True +RERANK_BACKEND = os.getenv("RERANK_BACKEND", "cloud") # cloud / local / fallback +RERANK_CLOUD_MODEL = os.getenv("RERANK_CLOUD_MODEL", "xop3qwen8breranker") +RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", "") +RERANK_CLOUD_BASE_URL = os.getenv("RERANK_CLOUD_BASE_URL", "https://maas-api.cn-huabei-1.xf-yun.com/v2/rerank") +RERANK_CLOUD_TIMEOUT = int(os.getenv("RERANK_CLOUD_TIMEOUT", "15")) RERANK_CANDIDATES = 20 # 送入重排序的候选数 RERANK_TOP_K = 15 # 重排序后保留数 RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true" diff --git a/core/engine.py b/core/engine.py index 304e1a3..9805ecd 100644 --- a/core/engine.py +++ b/core/engine.py @@ -112,10 +112,10 @@ except ImportError: EMBEDDING_DEVICE = "auto" RERANK_DEVICE = "auto" RERANK_USE_ONNX = False - RERANK_BACKEND = "local" + RERANK_BACKEND = "cloud" RERANK_CLOUD_MODEL = "xop3qwen8breranker" RERANK_CLOUD_API_KEY = "" - RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" + RERANK_CLOUD_BASE_URL = "https://maas-api.cn-huabei-1.xf-yun.com/v2/rerank" RERANK_CLOUD_TIMEOUT = 15 diff --git a/knowledge/collection.py b/knowledge/collection.py index 26b3ee1..f450cf1 100644 --- a/knowledge/collection.py +++ b/knowledge/collection.py @@ -134,6 +134,9 @@ class CollectionMixin: """ from .base import BM25Index + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + if not kb_name or not kb_name.replace('_', '').isalnum(): return False, "向量库名称只能包含字母、数字和下划线" @@ -190,6 +193,9 @@ class CollectionMixin: Returns: 更新成功返回 True,向量库不存在返回 False """ + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + collections = self._metadata.get("collections", {}) if kb_name not in collections: return False @@ -228,6 +234,9 @@ class CollectionMixin: """ import shutil + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() + if kb_name == PUBLIC_KB_NAME: return False, "公开知识库不能删除" @@ -422,4 +431,6 @@ class CollectionMixin: Returns: 存在返回 True,不存在返回 False """ + # 从磁盘重新加载元数据,确保多 worker 进程间状态一致 + self._metadata = self._load_metadata() return kb_name in self._metadata.get("collections", {}) diff --git a/knowledge/manager.py b/knowledge/manager.py index a373cf5..711779f 100644 --- a/knowledge/manager.py +++ b/knowledge/manager.py @@ -29,6 +29,11 @@ import os import json import threading +try: + import fcntl + _HAS_FCNTL = True +except ImportError: + _HAS_FCNTL = False # Windows 环境无 fcntl from typing import List, Dict, Optional, Tuple from pathlib import Path import logging @@ -134,22 +139,36 @@ class KnowledgeBaseManager( logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}") def _load_metadata(self) -> dict: - """加载元数据""" + """加载元数据(带文件锁,确保多 worker 进程间一致)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) if os.path.exists(metadata_path): try: with open(metadata_path, 'r', encoding='utf-8') as f: - return json.load(f) + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) + try: + return json.load(f) + finally: + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) except Exception as e: logger.error(f"加载元数据失败: {e}") return {"collections": {}} def _save_metadata(self): - """保存元数据""" + """保存元数据(带文件锁,防止并发写入数据覆盖)""" metadata_path = os.path.join(self.base_path, KB_METADATA_FILE) try: with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(self._metadata, f, ensure_ascii=False, indent=2) + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + try: + json.dump(self._metadata, f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + finally: + if _HAS_FCNTL: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) except Exception as e: logger.error(f"保存元数据失败: {e}")