chore: 死代码清理与标注
- 删除 chat_routes.py 中未被使用的 reciprocal_rank_fusion 函数（engine 有同功能方法平替） - 删除 search_hybrid 中无效的 candidates 参数（未传递给 engine，实际由 RERANK_CANDIDATES 控制） - 标注 RAG_SEARCH_CANDIDATES 为死代码 - 标注 VECTOR_WEIGHT/BM25_WEIGHT 在动态 RRF 启用时被覆盖 - 标注 llm_budget.py 模块当前未集成到主流程 - 标注 MAX_LLM_CALLS_PER_QUERY/MAX_QUERY_REWRITES 暂不生效
This commit is contained in:
lacerate551
@@ -493,6 +493,94 @@ def _section_similarity(section_a: str, section_b: str) -> float:
|
||||
return overlap / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def _rescue_bm25_divergence(contexts: List[Dict], search_result: dict,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
BM25-CrossEncoder 分歧检测救援：当 BM25 排名靠前（top-3）的切片
|
||||
被 CrossEncoder rerank 压制（分数低于 min_score 或被截断）时，
|
||||
将其分数提升至保底值，使其通过后续的 min_score 过滤。
|
||||
|
||||
适用场景：
|
||||
- BM25 top-1/top-2 的切片（精确关键词匹配强信号）被 rerank 评分极低
|
||||
- 正确切片在 rerank 后被截断，根本不在 contexts 中
|
||||
|
||||
Args:
|
||||
contexts: 全部上下文切片
|
||||
search_result: engine.search_hybrid() 返回的原始结果（含 _bm25_top3）
|
||||
min_score: 最低分数阈值
|
||||
|
||||
Returns:
|
||||
修改后的 contexts
|
||||
"""
|
||||
if not contexts:
|
||||
return contexts
|
||||
|
||||
from config import (
|
||||
BM25_DIVERGENCE_RESCUE_ENABLED, BM25_DIVERGENCE_MAX_RANK,
|
||||
CLUSTER_RESCUE_FLOOR,
|
||||
)
|
||||
|
||||
if not BM25_DIVERGENCE_RESCUE_ENABLED:
|
||||
return contexts
|
||||
|
||||
bm25_top3 = search_result.get('_bm25_top3', [])
|
||||
if not bm25_top3:
|
||||
return contexts
|
||||
|
||||
# 构建 contexts 中已有切片的 ID 索引，用于快速查找
|
||||
existing_ids = {}
|
||||
for i, ctx in enumerate(contexts):
|
||||
chunk_id = ctx.get('meta', {}).get('chunk_id') or ctx.get('id')
|
||||
if chunk_id:
|
||||
existing_ids[chunk_id] = i
|
||||
|
||||
rescued_count = 0
|
||||
|
||||
for bm25_item in bm25_top3:
|
||||
rank = bm25_item.get('rank', 99)
|
||||
if rank > BM25_DIVERGENCE_MAX_RANK:
|
||||
continue
|
||||
|
||||
bm25_id = bm25_item.get('id')
|
||||
bm25_meta = bm25_item.get('meta', {})
|
||||
bm25_doc = bm25_item.get('doc', '')
|
||||
|
||||
# 情况 A：切片在 contexts 中但 score < min_score
|
||||
if bm25_id and bm25_id in existing_ids:
|
||||
ctx = contexts[existing_ids[bm25_id]]
|
||||
if ctx.get('score', 0) < min_score:
|
||||
ctx['score'] = CLUSTER_RESCUE_FLOOR
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况A): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}, "
|
||||
f"原score→{CLUSTER_RESCUE_FLOOR}"
|
||||
)
|
||||
continue
|
||||
|
||||
# 情况 B：切片不在 contexts 中（被 rerank 截断或已被过滤）
|
||||
# 从 _bm25_top3 备份中注入
|
||||
if bm25_doc and bm25_meta:
|
||||
injected_ctx = {
|
||||
'doc': bm25_doc,
|
||||
'meta': bm25_meta,
|
||||
'score': CLUSTER_RESCUE_FLOOR,
|
||||
}
|
||||
contexts.append(injected_ctx)
|
||||
rescued_count += 1
|
||||
logger.debug(
|
||||
f"BM25 分歧救援 (情况B-注入): rank={rank}, "
|
||||
f"source={bm25_meta.get('source', '')}, "
|
||||
f"section={bm25_meta.get('section', '')}"
|
||||
)
|
||||
|
||||
if rescued_count > 0:
|
||||
logger.info(f"BM25 分歧救援: 共救援 {rescued_count} 个切片")
|
||||
|
||||
return contexts
|
||||
|
||||
|
||||
def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
|
||||
min_score: float) -> List[Dict]:
|
||||
"""
|
||||
@@ -1752,54 +1840,7 @@ def chat_with_llm(message: str, history: List[Dict] = None, enable_web_search: b
|
||||
}
|
||||
|
||||
|
||||
def reciprocal_rank_fusion(results_list, weights=None, k=60):
|
||||
"""
|
||||
倒数排名融合算法
|
||||
|
||||
Args:
|
||||
results_list: 多个检索结果列表
|
||||
weights: 各结果权重
|
||||
k: RRF 参数
|
||||
|
||||
Returns:
|
||||
融合后的排序结果
|
||||
"""
|
||||
if weights is None:
|
||||
weights = [1.0] * len(results_list)
|
||||
|
||||
fused_scores = {}
|
||||
doc_data = {}
|
||||
|
||||
for results, weight in zip(results_list, weights):
|
||||
if not results or not results.get('ids'):
|
||||
continue
|
||||
|
||||
ids = results['ids'][0]
|
||||
docs = results['documents'][0] if results.get('documents') else [''] * len(ids)
|
||||
metas = results['metadatas'][0] if results.get('metadatas') else [{}] * len(ids)
|
||||
distances = results['distances'][0] if results.get('distances') else [0] * len(ids)
|
||||
|
||||
for rank, (doc_id, doc, meta, dist) in enumerate(zip(ids, docs, metas, distances)):
|
||||
if doc_id not in fused_scores:
|
||||
fused_scores[doc_id] = 0
|
||||
doc_data[doc_id] = {'doc': doc, 'meta': meta, 'dist': dist}
|
||||
|
||||
# RRF 分数
|
||||
fused_scores[doc_id] += weight / (rank + k)
|
||||
|
||||
# 按分数排序
|
||||
sorted_ids = sorted(fused_scores.keys(), key=lambda x: fused_scores[x], reverse=True)
|
||||
|
||||
return {
|
||||
'ids': sorted_ids,
|
||||
'documents': [doc_data[i]['doc'] for i in sorted_ids],
|
||||
'metadatas': [doc_data[i]['meta'] for i in sorted_ids],
|
||||
'scores': [fused_scores[i] for i in sorted_ids],
|
||||
'distances': [doc_data[i]['dist'] for i in sorted_ids]
|
||||
}
|
||||
|
||||
|
||||
def search_hybrid(query: str, top_k: int = 5, candidates: int = 15,
|
||||
def search_hybrid(query: str, top_k: int = 5,
|
||||
allowed_levels: list = None, allowed_collections: list = None,
|
||||
sub_queries: list = None):
|
||||
"""
|
||||
@@ -1808,7 +1849,6 @@ def search_hybrid(query: str, top_k: int = 5, candidates: int = 15,
|
||||
Args:
|
||||
query: 查询文本
|
||||
top_k: 返回数量
|
||||
candidates: 候选数量（用于 RERANK_CANDIDATES，由 config 控制）
|
||||
allowed_levels: 允许的安全级别
|
||||
allowed_collections: 允许的向量库列表
|
||||
sub_queries: 意图分析器生成的子查询列表（对比类查询用）
|
||||
@@ -1916,7 +1956,7 @@ def rag():
|
||||
import re
|
||||
from config import (
|
||||
IS_PROD, IS_DEV, ENABLE_SESSION,
|
||||
RAG_SEARCH_TOP_K, RAG_SEARCH_CANDIDATES,
|
||||
RAG_SEARCH_TOP_K,
|
||||
MAX_CONTEXT_CHUNKS, MAX_SOURCES_RETURNED,
|
||||
LLM_TEMPERATURE, LLM_MAX_TOKENS,
|
||||
MAX_HISTORY_ROUNDS, IMAGE_CONTEXT_HISTORY,
|
||||
@@ -2158,7 +2198,6 @@ def rag():
|
||||
search_result = search_hybrid(
|
||||
retrieval_query,
|
||||
top_k=RAG_SEARCH_TOP_K,
|
||||
candidates=RAG_SEARCH_CANDIDATES,
|
||||
allowed_collections=collections,
|
||||
sub_queries=sub_queries
|
||||
)
|
||||
@@ -2442,6 +2481,9 @@ def rag():
|
||||
# 4. 构建 prompt（Phase 6：LLM 图片感知）
|
||||
# Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争
|
||||
|
||||
# BM25 分歧检测救援：BM25 top-3 但 rerank 压制的切片
|
||||
contexts = _rescue_bm25_divergence(contexts, search_result, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
# 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时，
|
||||
# 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景）
|
||||
contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
|
||||
|
||||
288
config.py
Normal file
288
config.py
Normal file
@@ -0,0 +1,288 @@
|
||||
# RAG 知识库服务配置
|
||||
# ================================
|
||||
# 敏感信息通过环境变量注入，默认值为空字符串
|
||||
|
||||
import os
|
||||
|
||||
# 加载 .env 文件（敏感信息不写入代码，通过 .env 注入）
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env"))
|
||||
except ImportError:
|
||||
pass # python-dotenv 未安装时静默跳过
|
||||
|
||||
# ==============================================================================
|
||||
# 一、API 密钥与模型
|
||||
# ==============================================================================
|
||||
|
||||
# 通义千问 LLM 服务
|
||||
DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
|
||||
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
|
||||
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
|
||||
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
|
||||
INTENT_MODEL = os.getenv("INTENT_MODEL", "mimo-v2.5") # 意图分析模型
|
||||
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型（图片描述）
|
||||
|
||||
# 兼容旧变量名（逐步迁移到 DASHSCOPE_* 命名）
|
||||
API_KEY = DASHSCOPE_API_KEY
|
||||
BASE_URL = DASHSCOPE_BASE_URL
|
||||
MODEL = DASHSCOPE_MODEL
|
||||
|
||||
# ==============================================================================
|
||||
# 二、环境与功能开关
|
||||
# ==============================================================================
|
||||
|
||||
APP_ENV = os.getenv("APP_ENV", "dev") # dev / prod
|
||||
IS_DEV = APP_ENV == "dev"
|
||||
IS_PROD = APP_ENV == "prod"
|
||||
|
||||
# 开发模式开关（控制 mock token 登录、模拟用户等开发功能）
|
||||
# 默认开启，生产环境需在 .env 中设置 DEV_MODE=false
|
||||
DEV_MODE = os.getenv("DEV_MODE", "true").lower() != "false"
|
||||
|
||||
# 开发/生产环境自动切换
|
||||
ENABLE_SESSION = IS_DEV # 会话存储（仅开发环境）
|
||||
ENABLE_FEEDBACK = True # 反馈系统
|
||||
|
||||
# 扩展功能（手动开启）
|
||||
ENABLE_WEB_SEARCH = False # 网络搜索（需 SERPER_API_KEY）
|
||||
|
||||
# ==============================================================================
|
||||
# 三、路径配置
|
||||
# ==============================================================================
|
||||
|
||||
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
MODELS_DIR = os.path.join(PROJECT_ROOT, "models")
|
||||
EMBEDDING_MODEL_PATH = os.path.join(MODELS_DIR, "bge-base-zh-v1.5")
|
||||
RERANK_MODEL_PATH = os.path.join(MODELS_DIR, "bge-reranker-base")
|
||||
_vector_store_path = os.path.join(PROJECT_ROOT, "knowledge", "vector_store")
|
||||
CHROMA_DB_PATH = os.path.join(_vector_store_path, "chroma")
|
||||
DOCUMENTS_PATH = os.path.join(PROJECT_ROOT, "documents")
|
||||
BM25_INDEXES_PATH = os.path.join(_vector_store_path, "bm25")
|
||||
|
||||
# 文件存储类型: local / smb / s3 / http
|
||||
STORAGE_TYPE = os.getenv("STORAGE_TYPE", "local")
|
||||
|
||||
# SMB/CIFS 配置
|
||||
STORAGE_SMB_HOST = os.getenv("STORAGE_SMB_HOST", "")
|
||||
STORAGE_SMB_SHARE = os.getenv("STORAGE_SMB_SHARE", "")
|
||||
STORAGE_SMB_USERNAME = os.getenv("STORAGE_SMB_USERNAME", "")
|
||||
STORAGE_SMB_PASSWORD = os.getenv("STORAGE_SMB_PASSWORD", "")
|
||||
STORAGE_SMB_DOMAIN = os.getenv("STORAGE_SMB_DOMAIN", "")
|
||||
STORAGE_SMB_BASE_PATH = os.getenv("STORAGE_SMB_BASE_PATH", "")
|
||||
|
||||
# S3 配置
|
||||
STORAGE_S3_ENDPOINT = os.getenv("STORAGE_S3_ENDPOINT", "")
|
||||
STORAGE_S3_BUCKET = os.getenv("STORAGE_S3_BUCKET", "")
|
||||
STORAGE_S3_ACCESS_KEY = os.getenv("STORAGE_S3_ACCESS_KEY", "")
|
||||
STORAGE_S3_SECRET_KEY = os.getenv("STORAGE_S3_SECRET_KEY", "")
|
||||
STORAGE_S3_REGION = os.getenv("STORAGE_S3_REGION", "us-east-1")
|
||||
|
||||
# HTTP 文件服务配置
|
||||
STORAGE_HTTP_BASE_URL = os.getenv("STORAGE_HTTP_BASE_URL", "")
|
||||
STORAGE_HTTP_TOKEN = os.getenv("STORAGE_HTTP_TOKEN", "")
|
||||
STORAGE_HTTP_TIMEOUT = int(os.getenv("STORAGE_HTTP_TIMEOUT", "60"))
|
||||
|
||||
# ==============================================================================
|
||||
# 四、设备配置（GPU / CPU）
|
||||
# ==============================================================================
|
||||
|
||||
EMBEDDING_DEVICE = os.getenv("EMBEDDING_DEVICE", os.getenv("DEVICE", "auto"))
|
||||
RERANK_DEVICE = os.getenv("RERANK_DEVICE", os.getenv("DEVICE", "auto"))
|
||||
|
||||
# ==============================================================================
|
||||
# 五、LLM 参数
|
||||
# ==============================================================================
|
||||
|
||||
# ----- 通用问答 -----
|
||||
LLM_TEMPERATURE = 0.7 # 生成温度（0=确定性，1=随机性）
|
||||
LLM_MAX_TOKENS = 3000 # 最大输出 token 数
|
||||
|
||||
# ----- 意图分析（轻量、确定性高）-----
|
||||
# INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置
|
||||
INTENT_TEMPERATURE = 0.1
|
||||
INTENT_MAX_TOKENS = 2048 # 推理模型需要额外 token 用于思维链
|
||||
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
|
||||
|
||||
# ==============================================================================
|
||||
# 六、检索参数
|
||||
# ==============================================================================
|
||||
|
||||
# ----- 混合检索 -----
|
||||
USE_MULTI_KB = True # 多向量库模式
|
||||
USE_HYBRID_SEARCH = True # 向量 + BM25 混合检索
|
||||
VECTOR_WEIGHT = 0.5 # 向量检索权重（仅在 USE_MULTI_KB=False 时生效；动态 RRF 启用时被覆盖）
|
||||
BM25_WEIGHT = 0.5 # BM25 检索权重（同上）
|
||||
RAG_SEARCH_TOP_K = 30 # 最终返回结果数（不小于 MMR_TOP_K，避免截断 MMR 输出）
|
||||
RAG_SEARCH_CANDIDATES = 100 # [死代码] 候选池大小 — 实际由 RERANK_CANDIDATES 控制，此值未传递给 engine
|
||||
RECALL_MULTIPLIER = 3 # 候选池最小倍数 = top_k * 此值
|
||||
|
||||
# ----- 重排序 -----
|
||||
USE_RERANK = True # 是否启用重排序
|
||||
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
||||
RERANK_TOP_K = 15 # 重排序后保留数
|
||||
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
||||
RERANK_CONTEXT_MIN_SCORE = 0.05 # Phase 1：Rerank 分数低于此值的切片不送入 LLM
|
||||
|
||||
# ----- 云端 Reranker（DashScope API）-----
|
||||
# RERANK_BACKEND: "local"=本地模型(CPU/GPU)，"cloud"=云端API，"fallback"=优先云端、失败回退本地
|
||||
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "local")
|
||||
RERANK_CLOUD_MODEL = os.getenv("RERANK_CLOUD_MODEL", "xop3qwen8breranker")
|
||||
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", DASHSCOPE_API_KEY)
|
||||
RERANK_CLOUD_BASE_URL = os.getenv("RERANK_CLOUD_BASE_URL", "https://maas-api.cn-huabei-1.xf-yun.com/v1/rerank")
|
||||
RERANK_CLOUD_TIMEOUT = int(os.getenv("RERANK_CLOUD_TIMEOUT", "15")) # 云端请求超时(秒)
|
||||
|
||||
# ----- RRF 融合 -----
|
||||
RRF_K = 60 # RRF 常数（越大越平滑）
|
||||
DYNAMIC_RRF_ENABLED = True # 根据查询类型动态调整向量/BM25 权重
|
||||
|
||||
# ----- MMR 多样性去重 -----
|
||||
MMR_ENABLED = True
|
||||
MMR_USE_EMBEDDING = os.getenv("MMR_USE_EMBEDDING", "false").lower() == "true" # True=语义向量（慢），False=文本相似度（快，使用 jieba 词级 Jaccard）
|
||||
MMR_TOP_K = 30 # MMR 处理后保留数
|
||||
MMR_LAMBDA = 0.5 # 相关性 vs 多样性权衡（0=纯多样，1=纯相关）
|
||||
|
||||
# ----- 查询扩展 -----
|
||||
QUERY_EXPANSION_ENABLED = True
|
||||
QUERY_EXPANSION_THRESHOLD = 0.8 # 扩展词相似度阈值
|
||||
|
||||
# ----- 章节过滤 -----
|
||||
SECTION_FILTER_ENABLED = True # 查询提到章节时优先匹配对应切片
|
||||
|
||||
# ==============================================================================
|
||||
# 七、上下文构建
|
||||
# ==============================================================================
|
||||
|
||||
MAX_CONTEXT_CHUNKS = 20 # 送给 LLM 的最大文本切片数
|
||||
CONTEXT_MAX_CHARS = 8000 # Phase 2：上下文最大字符数（约 4000 token）
|
||||
CONTEXT_SOFT_LIMIT = 6000 # Phase 2：软限制，超过后只接受高分切片组
|
||||
MAX_SOURCES_RETURNED = 10 # 返回给前端的最大来源数
|
||||
MAX_HISTORY_ROUNDS = 10 # 对话历史最大轮数
|
||||
IMAGE_CONTEXT_HISTORY = 4 # 图片上下文取最近几轮历史
|
||||
DIRECT_CONTEXT_MAX_CHARS = 2000 # 直接回答模式上下文截断字符数
|
||||
|
||||
# ==============================================================================
|
||||
# 八、FAQ 与黑名单
|
||||
# ==============================================================================
|
||||
|
||||
# FAQ 召回与权重
|
||||
FAQ_RECALL_TOP_K = 3 # FAQ 集合单独召回数
|
||||
FAQ_BOOST_AMOUNT = 0.1 # FAQ 命中时距离减少量（提升排名）
|
||||
|
||||
# FAQ 时间衰减（防止过期 FAQ 长期霸榜）
|
||||
FAQ_DECAY_MONTHS = 6 # 超过此月数开始衰减
|
||||
FAQ_DECAY_RATE = 0.01 # 每超一个月的距离惩罚
|
||||
FAQ_DECAY_MAX = 0.1 # 最大衰减惩罚
|
||||
|
||||
# 黑名单（负反馈过滤）
|
||||
BLACKLIST_MIN_DISLIKES = 3 # 差评达到此数量进入黑名单
|
||||
BLACKLIST_CACHE_TTL = 300 # 黑名单缓存刷新间隔（秒）
|
||||
|
||||
# ==============================================================================
|
||||
# 九、缓存配置
|
||||
# ==============================================================================
|
||||
|
||||
# 查询结果缓存
|
||||
QUERY_CACHE_ENABLED = True
|
||||
QUERY_CACHE_SIZE = 500
|
||||
QUERY_CACHE_TTL = 3600 # 秒
|
||||
|
||||
# Embedding 缓存
|
||||
EMBEDDING_CACHE_ENABLED = True
|
||||
EMBEDDING_CACHE_SIZE = 2000
|
||||
EMBEDDING_CACHE_TTL = 86400
|
||||
|
||||
# Rerank 缓存
|
||||
RERANK_CACHE_ENABLED = True
|
||||
RERANK_CACHE_SIZE = 1000
|
||||
RERANK_CACHE_TTL = 3600
|
||||
|
||||
# 语义缓存（相似查询复用结果）
|
||||
SEMANTIC_CACHE_ENABLED = True
|
||||
SEMANTIC_CACHE_THRESHOLD = 0.92 # 相似度阈值
|
||||
|
||||
# 缓存写入最低置信度
|
||||
# 注意：ChromaDB cosine distance 范围 [0,2]，score = 1 - dist
|
||||
# 当前 embedding 模型的 cosine similarity 普遍在 0.03-0.06 之间
|
||||
# 搜索管线已通过 rerank 过滤低质量结果，此处不再额外限制
|
||||
CACHE_MIN_SCORE = 0.0
|
||||
|
||||
# LLM 调用预算（当前 llm_budget 模块未集成到主流程，以下配置暂不生效）
|
||||
MAX_LLM_CALLS_PER_QUERY = 2
|
||||
MAX_QUERY_REWRITES = 1
|
||||
|
||||
# ==============================================================================
|
||||
# 十、文档解析
|
||||
# ==============================================================================
|
||||
|
||||
# MinerU 解析器
|
||||
MINERU_DEVICE_MODE = os.getenv("MINERU_DEVICE_MODE", "cpu") # cpu / cuda
|
||||
|
||||
# MinerU 在线 API（优先使用，解析效果更好）
|
||||
MINERU_API_TOKEN = os.getenv("MINERU_API_TOKEN", "") # 在 https://mineru.net/apiManage/token 申请
|
||||
MINERU_API_URL = os.getenv("MINERU_API_URL", "https://mineru.net/api/v4/extract/task")
|
||||
MINERU_PREFER_ONLINE = os.getenv("MINERU_PREFER_ONLINE", "true").lower() == "false" # 优先使用在线 API
|
||||
|
||||
# MinerU 解析模式（本地 + 云端统一配置）
|
||||
MINERU_MODEL_VERSION = os.getenv("MINERU_MODEL_VERSION", "pipeline") # 云端解析模型: pipeline(快速推荐) / vlm(高精度慢) / MinerU-HTML
|
||||
MINERU_LOCAL_BACKEND = os.getenv("MINERU_LOCAL_BACKEND", "vlm-auto-engine") # 本地解析后端: pipeline(快速推荐) / vlm-auto-engine / hybrid-auto-engine
|
||||
MINERU_ONLINE_TIMEOUT = int(os.getenv("MINERU_ONLINE_TIMEOUT", "300")) # 云端解析轮询超时（秒），大文档/VLM 模式建议 600+
|
||||
|
||||
# 分块参数
|
||||
CHUNK_SIZE = 1000
|
||||
CHUNK_OVERLAP = 100
|
||||
MIN_CHUNK_SIZE = 200
|
||||
MAX_CHUNK_SIZE = 1200
|
||||
|
||||
# 自适应 TopK（根据置信度动态调整返回数）
|
||||
ADAPTIVE_TOPK_ENABLED = True
|
||||
ADAPTIVE_LOW_CONFIDENCE = 0.5
|
||||
ADAPTIVE_HIGH_CONFIDENCE = 0.8
|
||||
ADAPTIVE_EXPAND_RATIO = 2.0
|
||||
ADAPTIVE_SHRINK_RATIO = 0.5
|
||||
ADAPTIVE_MIN_TOPK = 15
|
||||
ADAPTIVE_MAX_TOPK = 20
|
||||
|
||||
# 连续切片完整性保护（枚举/条款/清单类问题）
|
||||
CONTEXT_EXPANSION_ENABLED = True
|
||||
CONTEXT_EXPANSION_BEFORE = 1
|
||||
CONTEXT_EXPANSION_AFTER = 8
|
||||
CONTEXT_EXPANSION_MAX_CHUNKS = 50
|
||||
EXPANSION_SCORE_THRESHOLD = 0.3 # Phase 3：Rerank 分数低于此值的切片不扩展邻居
|
||||
MAX_EXPANDED_NEIGHBORS = 8 # Phase 3：每个种子切片最多扩展的邻居数
|
||||
CONFIDENCE_WARN_THRESHOLD = 0.15 # Phase 4：top-3 均分低于此值时，提示 LLM 谨慎回答
|
||||
CONFIDENCE_CAUTION_THRESHOLD = 0.30 # Phase 4：top-3 均分低于此值时，提示 LLM 优先引用原文
|
||||
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
|
||||
ENUM_QUERY_MMR_LAMBDA = 0.85
|
||||
|
||||
# ----- 章节聚类救援（Section-Cluster Rescue）-----
|
||||
# 当同一 section 下多个切片（text+table）同时出现在候选集中，
|
||||
# 即使单个切片 CrossEncoder 分数很低，也视为强信号进行提升/救援。
|
||||
SECTION_CLUSTER_BOOST_ENABLED = True # 引擎层：聚类提升（rerank 后、扩展前）
|
||||
SECTION_CLUSTER_RESCUE_ENABLED = True # 路由层：聚类救援（min_score 过滤前）
|
||||
BM25_DIVERGENCE_RESCUE_ENABLED = True # 路由层：BM25-CrossEncoder 分歧检测救援
|
||||
BM25_DIVERGENCE_MAX_RANK = 3 # 仅救援 BM25 rank <= 此值的切片（top-3 是强信号）
|
||||
CLUSTER_MIN_MEMBERS = 3 # 触发聚类的最小切片数
|
||||
CLUSTER_MIN_TYPES = 2 # 触发聚类的最小类型多样性（text+table=2）
|
||||
CLUSTER_SEED_FLOOR = 0.35 # 引擎层聚类提升后的最低分数（略高于 EXPANSION_SCORE_THRESHOLD=0.3）
|
||||
CLUSTER_RESCUE_FLOOR = 0.06 # 路由层救援保底分数（略高于 RERANK_CONTEXT_MIN_SCORE=0.05）
|
||||
CLUSTER_MAX_BOOST_PER_SECTION = 8 # 引擎层：每个 section 最大提升切片数
|
||||
CLUSTER_MAX_SECTIONS = 3 # 全局最大提升/救援 section 数
|
||||
CLUSTER_MAX_RESCUE_PER_SECTION = 6 # 路由层：每个 section 最大救援切片数
|
||||
CLUSTER_SECTION_PREFIX_LEVELS = 1 # section_path 归一化保留的层级数（按章节顶层分组）
|
||||
|
||||
# ==============================================================================
|
||||
# 十一、可选功能配置
|
||||
# ==============================================================================
|
||||
|
||||
# 网络搜索（需 Serper API）
|
||||
SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
|
||||
|
||||
# ==============================================================================
|
||||
# 工具函数
|
||||
# ==============================================================================
|
||||
|
||||
def get_llm_client():
|
||||
"""获取 LLM 客户端实例"""
|
||||
from openai import OpenAI
|
||||
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
|
||||
@@ -2,6 +2,10 @@
|
||||
"""
|
||||
LLM 调用预算控制器
|
||||
|
||||
注意：此模块当前未集成到主流程（engine.py / chat_routes.py 均未调用）。
|
||||
相关配置项 MAX_LLM_CALLS_PER_QUERY / MAX_QUERY_REWRITES 暂不生效。
|
||||
未来如需启用 LLM 调用预算控制，在 chat_routes.py 的 generate_stream 中集成即可。
|
||||
|
||||
控制每次查询的 LLM 调用次数，防止过度消耗
|
||||
|
||||
功能：
|
||||
|
||||
Reference in New Issue
Block a user