302 lines
13 KiB
Python
302 lines
13 KiB
Python
# RAG 服务配置文件模板
|
||
# ================================
|
||
# 请复制为 config.py 并填入你的 API 密钥
|
||
# 敏感信息通过环境变量注入，默认值为空字符串
|
||
|
||
import os
|
||
|
||
# ==============================================================================
|
||
# 一、API 密钥与模型
|
||
# ==============================================================================
|
||
|
||
# OpenAI 兼容 LLM 服务（当前默认使用小米 MiMo）
|
||
DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
|
||
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
|
||
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
|
||
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
|
||
INTENT_MODEL = os.getenv("INTENT_MODEL", "mimo-v2.5") # 意图分析模型
|
||
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 图片理解模型
|
||
|
||
# 兼容旧变量名（逐步迁移到 DASHSCOPE_* 命名）
|
||
API_KEY = DASHSCOPE_API_KEY
|
||
BASE_URL = DASHSCOPE_BASE_URL
|
||
MODEL = DASHSCOPE_MODEL
|
||
|
||
# ==============================================================================
|
||
# 二、环境与功能开关
|
||
# ==============================================================================
|
||
|
||
APP_ENV = os.getenv("APP_ENV", "dev") # dev / prod
|
||
IS_DEV = APP_ENV == "dev"
|
||
IS_PROD = APP_ENV == "prod"
|
||
DEV_MODE = os.getenv("DEV_MODE", "true").lower() != "false"
|
||
|
||
# 开发/生产环境自动切换
|
||
ENABLE_SESSION = IS_DEV # 会话存储（仅开发环境）
|
||
ENABLE_FEEDBACK = True # 反馈系统
|
||
|
||
# 扩展功能（手动开启）
|
||
ENABLE_WEB_SEARCH = False # 网络搜索（需 SERPER_API_KEY）
|
||
|
||
# ==============================================================================
|
||
# 三、路径配置
|
||
# ==============================================================================
|
||
|
||
PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||
MODELS_DIR = os.path.join(PROJECT_ROOT, "models")
|
||
EMBEDDING_MODEL_PATH = os.path.join(MODELS_DIR, "bge-base-zh-v1.5")
|
||
RERANK_MODEL_PATH = os.path.join(MODELS_DIR, "bge-reranker-base")
|
||
_vector_store_path = os.path.join(PROJECT_ROOT, "knowledge", "vector_store")
|
||
CHROMA_DB_PATH = os.path.join(_vector_store_path, "chroma")
|
||
DOCUMENTS_PATH = os.path.join(PROJECT_ROOT, "documents")
|
||
BM25_INDEXES_PATH = os.path.join(_vector_store_path, "bm25")
|
||
|
||
# 文件存储类型: local / smb / s3 / http
|
||
STORAGE_TYPE = os.getenv("STORAGE_TYPE", "local")
|
||
|
||
# SMB/CIFS 配置
|
||
STORAGE_SMB_HOST = os.getenv("STORAGE_SMB_HOST", "")
|
||
STORAGE_SMB_SHARE = os.getenv("STORAGE_SMB_SHARE", "")
|
||
STORAGE_SMB_USERNAME = os.getenv("STORAGE_SMB_USERNAME", "")
|
||
STORAGE_SMB_PASSWORD = os.getenv("STORAGE_SMB_PASSWORD", "")
|
||
STORAGE_SMB_DOMAIN = os.getenv("STORAGE_SMB_DOMAIN", "")
|
||
STORAGE_SMB_BASE_PATH = os.getenv("STORAGE_SMB_BASE_PATH", "")
|
||
|
||
# S3 配置
|
||
STORAGE_S3_ENDPOINT = os.getenv("STORAGE_S3_ENDPOINT", "")
|
||
STORAGE_S3_BUCKET = os.getenv("STORAGE_S3_BUCKET", "")
|
||
STORAGE_S3_ACCESS_KEY = os.getenv("STORAGE_S3_ACCESS_KEY", "")
|
||
STORAGE_S3_SECRET_KEY = os.getenv("STORAGE_S3_SECRET_KEY", "")
|
||
STORAGE_S3_REGION = os.getenv("STORAGE_S3_REGION", "us-east-1")
|
||
|
||
# HTTP 文件服务配置
|
||
STORAGE_HTTP_BASE_URL = os.getenv("STORAGE_HTTP_BASE_URL", "")
|
||
STORAGE_HTTP_TOKEN = os.getenv("STORAGE_HTTP_TOKEN", "")
|
||
STORAGE_HTTP_TIMEOUT = int(os.getenv("STORAGE_HTTP_TIMEOUT", "60"))
|
||
|
||
# ==============================================================================
|
||
# 四、设备配置（GPU / CPU）
|
||
# ==============================================================================
|
||
|
||
EMBEDDING_DEVICE = os.getenv("EMBEDDING_DEVICE", os.getenv("DEVICE", "auto"))
|
||
RERANK_DEVICE = os.getenv("RERANK_DEVICE", os.getenv("DEVICE", "auto"))
|
||
|
||
# ==============================================================================
|
||
# 五、LLM 参数
|
||
# ==============================================================================
|
||
|
||
# ----- 通用问答 -----
|
||
LLM_TEMPERATURE = 0.7 # 生成温度（0=确定性，1=随机性）
|
||
LLM_MAX_TOKENS = 3000 # 最大输出 token 数
|
||
LLM_DISABLE_THINKING = os.getenv("LLM_DISABLE_THINKING", "true").lower() != "false"
|
||
|
||
# ----- 意图分析（轻量、确定性高）-----
|
||
INTENT_TEMPERATURE = 0.1
|
||
INTENT_MAX_TOKENS = 2048
|
||
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
|
||
|
||
# ==============================================================================
|
||
# 六、检索参数
|
||
# ==============================================================================
|
||
|
||
# ----- 混合检索 -----
|
||
USE_MULTI_KB = True # 多向量库模式
|
||
USE_HYBRID_SEARCH = True # 向量 + BM25 混合检索
|
||
VECTOR_WEIGHT = 0.5 # 向量检索权重
|
||
BM25_WEIGHT = 0.5 # BM25 检索权重
|
||
RAG_SEARCH_TOP_K = 30 # 最终返回结果数（不小于 MMR_TOP_K，避免截断 MMR 输出）
|
||
RAG_SEARCH_CANDIDATES = 100 # 候选池大小（越大召回越全，越慢）
|
||
RECALL_MULTIPLIER = 3 # 候选池最小倍数 = top_k * 此值
|
||
|
||
# ----- 重排序 -----
|
||
USE_RERANK = True
|
||
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "cloud") # cloud / local / fallback
|
||
RERANK_CLOUD_MODEL = os.getenv("RERANK_CLOUD_MODEL", "xop3qwen8breranker")
|
||
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", "")
|
||
RERANK_CLOUD_BASE_URL = os.getenv("RERANK_CLOUD_BASE_URL", "https://maas-api.cn-huabei-1.xf-yun.com/v2/rerank")
|
||
RERANK_CLOUD_TIMEOUT = int(os.getenv("RERANK_CLOUD_TIMEOUT", "15"))
|
||
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
||
RERANK_TOP_K = 15 # 重排序后保留数
|
||
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
||
RERANK_CONTEXT_MIN_SCORE = 0.05 # Rerank 分数低于此值的切片不送入 LLM
|
||
|
||
# ----- RRF 融合 -----
|
||
RRF_K = 60 # RRF 常数（越大越平滑）
|
||
DYNAMIC_RRF_ENABLED = True # 根据查询类型动态调整向量/BM25 权重
|
||
|
||
# ----- MMR 多样性去重 -----
|
||
MMR_ENABLED = True
|
||
MMR_USE_EMBEDDING = True # True=语义向量（慢），False=文本相似度（快）
|
||
MMR_TOP_K = 30 # MMR 处理后保留数
|
||
MMR_LAMBDA = 0.5 # 相关性 vs 多样性权衡（0=纯多样，1=纯相关）
|
||
|
||
# ----- 查询扩展 -----
|
||
QUERY_EXPANSION_ENABLED = True
|
||
QUERY_EXPANSION_THRESHOLD = 0.8 # 扩展词相似度阈值
|
||
|
||
# ----- 章节过滤 -----
|
||
SECTION_FILTER_ENABLED = True # 查询提到章节时优先匹配对应切片
|
||
|
||
# ==============================================================================
|
||
# 七、上下文构建
|
||
# ==============================================================================
|
||
|
||
MAX_CONTEXT_CHUNKS = 20 # 送给 LLM 的最大文本切片数
|
||
CONTEXT_MAX_CHARS = 8000 # 上下文最大字符数（约 4000 token）
|
||
CONTEXT_SOFT_LIMIT = 6000 # 软限制，超过后只接受高分切片组
|
||
MAX_SOURCES_RETURNED = 10 # 返回给前端的最大来源数
|
||
MAX_HISTORY_ROUNDS = 10 # 对话历史最大轮数
|
||
IMAGE_CONTEXT_HISTORY = 4 # 图片上下文取最近几轮历史
|
||
DIRECT_CONTEXT_MAX_CHARS = 2000 # 直接回答模式上下文截断字符数
|
||
|
||
# ==============================================================================
|
||
# 八、FAQ 与黑名单
|
||
# ==============================================================================
|
||
|
||
# FAQ 召回与权重
|
||
FAQ_RECALL_TOP_K = 3 # FAQ 集合单独召回数
|
||
FAQ_BOOST_AMOUNT = 0.1 # FAQ 命中时距离减少量（提升排名）
|
||
|
||
# FAQ 时间衰减（防止过期 FAQ 长期霸榜）
|
||
FAQ_DECAY_MONTHS = 6 # 超过此月数开始衰减
|
||
FAQ_DECAY_RATE = 0.01 # 每超一个月的距离惩罚
|
||
FAQ_DECAY_MAX = 0.1 # 最大衰减惩罚
|
||
|
||
# 黑名单（负反馈过滤）
|
||
BLACKLIST_MIN_DISLIKES = 3 # 差评达到此数量进入黑名单
|
||
BLACKLIST_CACHE_TTL = 300 # 黑名单缓存刷新间隔（秒）
|
||
|
||
# ==============================================================================
|
||
# 九、缓存配置
|
||
# ==============================================================================
|
||
|
||
# 查询结果缓存
|
||
QUERY_CACHE_ENABLED = True
|
||
QUERY_CACHE_SIZE = 500
|
||
QUERY_CACHE_TTL = 3600 # 秒
|
||
|
||
# Embedding 缓存
|
||
EMBEDDING_CACHE_ENABLED = True
|
||
EMBEDDING_CACHE_SIZE = 2000
|
||
EMBEDDING_CACHE_TTL = 86400
|
||
|
||
# Rerank 缓存
|
||
RERANK_CACHE_ENABLED = True
|
||
RERANK_CACHE_SIZE = 1000
|
||
RERANK_CACHE_TTL = 3600
|
||
|
||
# 语义缓存（相似查询复用结果）
|
||
SEMANTIC_CACHE_ENABLED = True
|
||
SEMANTIC_CACHE_THRESHOLD = 0.92 # 相似度阈值
|
||
|
||
# 缓存写入最低置信度
|
||
# 注意：ChromaDB cosine distance 范围 [0,2]，score = 1 - dist
|
||
# 当前 embedding 模型的 cosine similarity 普遍在 0.03-0.06 之间
|
||
# 搜索管线已通过 rerank 过滤低质量结果，此处不再额外限制
|
||
CACHE_MIN_SCORE = 0.0
|
||
|
||
# LLM 调用预算
|
||
MAX_LLM_CALLS_PER_QUERY = 2
|
||
MAX_QUERY_REWRITES = 1
|
||
|
||
# ==============================================================================
|
||
# 十、文档解析
|
||
# ==============================================================================
|
||
|
||
# MinerU 解析器
|
||
MINERU_DEVICE_MODE = os.getenv("MINERU_DEVICE_MODE", "cpu") # cpu / cuda
|
||
MINERU_MODEL_VERSION = os.getenv("MINERU_MODEL_VERSION", "vlm")
|
||
MINERU_LOCAL_BACKEND = os.getenv("MINERU_LOCAL_BACKEND", "pipeline")
|
||
MINERU_ONLINE_TIMEOUT = int(os.getenv("MINERU_ONLINE_TIMEOUT", "300"))
|
||
|
||
# MinerU 在线 API（作为本地解析失败时的备选）
|
||
MINERU_API_TOKEN = os.getenv("MINERU_API_TOKEN", "") # 在 https://mineru.net/apiManage/token 申请
|
||
MINERU_API_URL = os.getenv("MINERU_API_URL", "https://mineru.net/api/v4/extract/task")
|
||
MINERU_PREFER_ONLINE = os.getenv("MINERU_PREFER_ONLINE", "true").lower() == "true"
|
||
MINERU_PREFER_V2 = os.getenv("MINERU_PREFER_V2", "true").lower() == "true" # 优先使用 v2 格式（含 style 信息）
|
||
|
||
# 标题识别规则引擎
|
||
# 规则定义见 parsers/heading_rules.py，支持通过 config.py 覆盖
|
||
HEADING_RULES_CONFIG = None # None=使用内置默认规则; dict 列表=自定义规则覆盖
|
||
HEADING_SHORT_TEXT_ENABLED = True # 是否启用短中文文本标题识别（最易误判的规则，可单独关闭）
|
||
HEADING_SHORT_TEXT_MAX_LENGTH = 20 # 短文本最大字符数阈值
|
||
|
||
# 表单类型二次校正
|
||
# MinerU 解析 Word 文档时，某些表单被标记为 text，根据内容特征修正为 table
|
||
FORM_RECLASSIFY_ENABLED = True # 是否启用 text->table 表单检测
|
||
FORM_RECLASSIFY_MIN_INDICATORS = 2 # 最少命中几个表单特征指标才校正
|
||
|
||
# 分块参数
|
||
CHUNK_SIZE = 1000
|
||
CHUNK_OVERLAP = 100
|
||
MIN_CHUNK_SIZE = 200
|
||
MAX_CHUNK_SIZE = 1200
|
||
|
||
# 自适应 TopK（根据置信度动态调整返回数）
|
||
ADAPTIVE_TOPK_ENABLED = True
|
||
ADAPTIVE_LOW_CONFIDENCE = 0.5
|
||
ADAPTIVE_HIGH_CONFIDENCE = 0.8
|
||
ADAPTIVE_EXPAND_RATIO = 2.0
|
||
ADAPTIVE_SHRINK_RATIO = 0.5
|
||
ADAPTIVE_MIN_TOPK = 15
|
||
ADAPTIVE_MAX_TOPK = 20
|
||
|
||
# 连续切片完整性保护（枚举/条款/清单类问题）
|
||
CONTEXT_EXPANSION_ENABLED = True
|
||
CONTEXT_EXPANSION_BEFORE = 1
|
||
CONTEXT_EXPANSION_AFTER = 5
|
||
CONTEXT_EXPANSION_MAX_CHUNKS = 50
|
||
EXPANSION_SCORE_THRESHOLD = 0.3
|
||
MAX_EXPANDED_NEIGHBORS = 4
|
||
CONFIDENCE_WARN_THRESHOLD = 0.15 # top-3 均分低于此值时，提示 LLM 谨慎回答
|
||
CONFIDENCE_CAUTION_THRESHOLD = 0.30 # top-3 均分低于此值时，提示 LLM 优先引用原文
|
||
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
|
||
ENUM_QUERY_MMR_LAMBDA = 0.85
|
||
|
||
# 章节聚类提升与 BM25/CrossEncoder 分歧救援
|
||
SECTION_CLUSTER_BOOST_ENABLED = True
|
||
SECTION_CLUSTER_RESCUE_ENABLED = True
|
||
BM25_DIVERGENCE_RESCUE_ENABLED = True
|
||
BM25_DIVERGENCE_MAX_RANK = 3
|
||
CLUSTER_MIN_MEMBERS = 3
|
||
CLUSTER_MIN_TYPES = 2
|
||
CLUSTER_SEED_FLOOR = 0.35
|
||
CLUSTER_RESCUE_FLOOR = 0.06
|
||
CLUSTER_MAX_BOOST_PER_SECTION = 8
|
||
CLUSTER_MAX_SECTIONS = 3
|
||
CLUSTER_MAX_RESCUE_PER_SECTION = 6
|
||
CLUSTER_SECTION_PREFIX_LEVELS = 2
|
||
|
||
# ==============================================================================
|
||
# 十一、可选功能配置
|
||
# ==============================================================================
|
||
|
||
# 网络搜索（需 Serper API）
|
||
SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
|
||
|
||
# ==============================================================================
|
||
# 工具函数
|
||
# ==============================================================================
|
||
|
||
def get_llm_client():
|
||
"""获取 LLM 客户端实例"""
|
||
from openai import OpenAI
|
||
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
|
||
|
||
|
||
_intent_client = None
|
||
|
||
|
||
def get_intent_client():
|
||
"""获取意图分析专用 LLM 客户端（进程内复用连接池）。"""
|
||
global _intent_client
|
||
if _intent_client is None:
|
||
if not DASHSCOPE_API_KEY:
|
||
raise ValueError("DASHSCOPE_API_KEY 未配置，请在环境变量中设置")
|
||
from openai import OpenAI
|
||
_intent_client = OpenAI(
|
||
api_key=DASHSCOPE_API_KEY,
|
||
base_url=DASHSCOPE_BASE_URL,
|
||
)
|
||
return _intent_client
|