Compare commits
9 Commits
server-bas
...
fc11b11dda
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc11b11dda | ||
|
|
99f5cf519e | ||
|
|
15c0aec9a6 | ||
|
|
8c7a6eb3fa | ||
|
|
fda1b2f049 | ||
|
|
148559ee3c | ||
|
|
431af0217a | ||
|
|
aa04bb94a6 | ||
|
|
84a8be0ce8 |
23
.dockerignore
Normal file
23
.dockerignore
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# 大体积数据目录(通过 volume 挂载,不需要打进镜像)
|
||||||
|
models/
|
||||||
|
knowledge/vector_store/
|
||||||
|
documents/
|
||||||
|
.data/
|
||||||
|
data/
|
||||||
|
|
||||||
|
# Python 虚拟环境
|
||||||
|
venv/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Git
|
||||||
|
.git/
|
||||||
|
|
||||||
|
# IDE 和编辑器
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
|
||||||
|
# 其他
|
||||||
|
*.log
|
||||||
|
.env*
|
||||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -120,9 +120,8 @@ test_*.json
|
|||||||
rag_response.json
|
rag_response.json
|
||||||
nul
|
nul
|
||||||
|
|
||||||
# 调试脚本和临时计划(仅本地使用)
|
# 临时调试脚本(下划线开头)
|
||||||
scripts/
|
scripts/_*.py
|
||||||
plans/
|
|
||||||
|
|
||||||
# Qoder 工具目录
|
# Qoder 工具目录
|
||||||
.qoder/
|
.qoder/
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ API 路由层 — Flask 应用工厂
|
|||||||
|
|
||||||
本模块实现 Flask 应用工厂模式,负责:
|
本模块实现 Flask 应用工厂模式,负责:
|
||||||
- 创建和配置 Flask 应用实例
|
- 创建和配置 Flask 应用实例
|
||||||
- 初始化核心服务(同步服务)
|
- 初始化核心服务(AgenticRAG、同步服务)
|
||||||
- 注册所有 API Blueprint
|
- 注册所有 API Blueprint
|
||||||
- 配置前端静态文件路由
|
- 配置前端静态文件路由
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ def create_app() -> 'Flask':
|
|||||||
|
|
||||||
1. 创建 Flask 应用,配置 CORS
|
1. 创建 Flask 应用,配置 CORS
|
||||||
2. 初始化 Repository(会话存储)
|
2. 初始化 Repository(会话存储)
|
||||||
3. 初始化核心服务(同步服务)
|
3. 初始化核心服务(AgenticRAG、同步服务)
|
||||||
4. 注册所有 API Blueprint
|
4. 注册所有 API Blueprint
|
||||||
5. 配置前端静态文件路由
|
5. 配置前端静态文件路由
|
||||||
6. 执行生产环境配置校验
|
6. 执行生产环境配置校验
|
||||||
@@ -93,6 +93,19 @@ def create_app() -> 'Flask':
|
|||||||
|
|
||||||
# ==================== 核心服务初始化 ====================
|
# ==================== 核心服务初始化 ====================
|
||||||
|
|
||||||
|
# Agentic RAG 引擎
|
||||||
|
try:
|
||||||
|
from core.agentic import AgenticRAG
|
||||||
|
from config import ENABLE_WEB_SEARCH
|
||||||
|
agentic_rag = AgenticRAG(
|
||||||
|
enable_web_search=ENABLE_WEB_SEARCH,
|
||||||
|
)
|
||||||
|
app.config['AGENTIC_RAG'] = agentic_rag
|
||||||
|
logger.info(f"Agentic RAG 引擎已初始化(网络搜索={'启用' if ENABLE_WEB_SEARCH else '关闭'})")
|
||||||
|
except Exception as e:
|
||||||
|
app.config['AGENTIC_RAG'] = None
|
||||||
|
logger.warning(f"Agentic RAG 初始化失败: {e}")
|
||||||
|
|
||||||
# 同步服务
|
# 同步服务
|
||||||
try:
|
try:
|
||||||
from knowledge.sync import KnowledgeSyncService
|
from knowledge.sync import KnowledgeSyncService
|
||||||
|
|||||||
@@ -11,7 +11,6 @@
|
|||||||
from flask import Blueprint, request, jsonify
|
from flask import Blueprint, request, jsonify
|
||||||
from auth.gateway import require_gateway_auth, require_role, get_user_permissions, MOCK_USERS
|
from auth.gateway import require_gateway_auth, require_role, get_user_permissions, MOCK_USERS
|
||||||
import os
|
import os
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
@@ -22,37 +21,6 @@ load_dotenv(env_path)
|
|||||||
auth_bp = Blueprint('auth', __name__)
|
auth_bp = Blueprint('auth', __name__)
|
||||||
|
|
||||||
|
|
||||||
# ==================== 登录速率限制 ====================
|
|
||||||
# 简单的内存级速率限制:每个 IP 在时间窗口内最多允许 N 次登录尝试
|
|
||||||
_login_attempts = {} # {ip: [(timestamp, ...), ...]}
|
|
||||||
_RATE_LIMIT_WINDOW = 300 # 5 分钟窗口
|
|
||||||
_RATE_LIMIT_MAX = 10 # 窗口内最多 10 次尝试
|
|
||||||
|
|
||||||
|
|
||||||
def _check_rate_limit(client_ip: str) -> bool:
|
|
||||||
"""检查是否超出登录速率限制,返回 True 表示允许"""
|
|
||||||
now = time.time()
|
|
||||||
if client_ip not in _login_attempts:
|
|
||||||
_login_attempts[client_ip] = []
|
|
||||||
|
|
||||||
# 清理过期记录
|
|
||||||
_login_attempts[client_ip] = [
|
|
||||||
t for t in _login_attempts[client_ip]
|
|
||||||
if now - t < _RATE_LIMIT_WINDOW
|
|
||||||
]
|
|
||||||
|
|
||||||
if len(_login_attempts[client_ip]) >= _RATE_LIMIT_MAX:
|
|
||||||
return False
|
|
||||||
|
|
||||||
_login_attempts[client_ip].append(now)
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _is_dev_mode() -> bool:
|
|
||||||
"""统一的开发模式判断"""
|
|
||||||
return os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
|
||||||
|
|
||||||
|
|
||||||
@auth_bp.route('/auth/login', methods=['POST'])
|
@auth_bp.route('/auth/login', methods=['POST'])
|
||||||
def mock_login():
|
def mock_login():
|
||||||
"""
|
"""
|
||||||
@@ -82,14 +50,10 @@ def mock_login():
|
|||||||
- manager / manager123 (经理,财务部)
|
- manager / manager123 (经理,财务部)
|
||||||
- user / test123 (普通用户,技术部)
|
- user / test123 (普通用户,技术部)
|
||||||
"""
|
"""
|
||||||
if not _is_dev_mode():
|
# 默认开启开发模式(生产环境需设置 DEV_MODE=false)
|
||||||
|
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||||||
return jsonify({"error": "仅开发环境可用,请设置 DEV_MODE=true"}), 403
|
return jsonify({"error": "仅开发环境可用,请设置 DEV_MODE=true"}), 403
|
||||||
|
|
||||||
# 速率限制检查
|
|
||||||
client_ip = request.remote_addr or 'unknown'
|
|
||||||
if not _check_rate_limit(client_ip):
|
|
||||||
return jsonify({"error": f"登录尝试过于频繁,请 {_RATE_LIMIT_WINDOW // 60} 分钟后再试"}), 429
|
|
||||||
|
|
||||||
data = request.json or {}
|
data = request.json or {}
|
||||||
username = data.get('username')
|
username = data.get('username')
|
||||||
password = data.get('password')
|
password = data.get('password')
|
||||||
@@ -115,9 +79,7 @@ def mock_login():
|
|||||||
def get_stats():
|
def get_stats():
|
||||||
"""获取系统统计信息(仅管理员)"""
|
"""获取系统统计信息(仅管理员)"""
|
||||||
from flask import current_app
|
from flask import current_app
|
||||||
session_manager = current_app.config.get('SESSION_MANAGER')
|
session_manager = current_app.config['SESSION_MANAGER']
|
||||||
if not session_manager:
|
|
||||||
return jsonify({"error": "会话管理器未启用"}), 503
|
|
||||||
return jsonify(session_manager.get_stats())
|
return jsonify(session_manager.get_stats())
|
||||||
|
|
||||||
|
|
||||||
@@ -169,7 +131,8 @@ def get_users():
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
if not _is_dev_mode():
|
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||||
|
if not dev_mode:
|
||||||
return jsonify({"error": "仅开发环境可用"}), 403
|
return jsonify({"error": "仅开发环境可用"}), 403
|
||||||
|
|
||||||
users = []
|
users = []
|
||||||
@@ -196,26 +159,12 @@ def update_user(user_id):
|
|||||||
"is_active": false
|
"is_active": false
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
if not _is_dev_mode():
|
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||||
|
if not dev_mode:
|
||||||
return jsonify({"error": "仅开发环境可用"}), 403
|
return jsonify({"error": "仅开发环境可用"}), 403
|
||||||
|
|
||||||
# 验证目标用户是否存在
|
# 模拟用户不支持真正的状态切换,直接返回成功
|
||||||
target_user = None
|
return jsonify({"message": "操作成功(模拟)", "user_id": user_id})
|
||||||
for username, info in MOCK_USERS.items():
|
|
||||||
if info['user_id'] == user_id:
|
|
||||||
target_user = info
|
|
||||||
break
|
|
||||||
|
|
||||||
if not target_user:
|
|
||||||
return jsonify({"error": f"用户 {user_id} 不存在"}), 404
|
|
||||||
|
|
||||||
data = request.json or {}
|
|
||||||
# 模拟操作:记录请求但不实际执行(mock 用户数据是静态的)
|
|
||||||
return jsonify({
|
|
||||||
"message": "操作成功(模拟)",
|
|
||||||
"user_id": user_id,
|
|
||||||
"applied_changes": data
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
@auth_bp.route('/auth/change-password', methods=['POST'])
|
@auth_bp.route('/auth/change-password', methods=['POST'])
|
||||||
@@ -230,7 +179,8 @@ def change_password():
|
|||||||
"new_password": "xxx"
|
"new_password": "xxx"
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
if not _is_dev_mode():
|
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||||
|
if not dev_mode:
|
||||||
return jsonify({"error": "仅开发环境可用"}), 403
|
return jsonify({"error": "仅开发环境可用"}), 403
|
||||||
|
|
||||||
data = request.json or {}
|
data = request.json or {}
|
||||||
@@ -243,12 +193,5 @@ def change_password():
|
|||||||
if len(new_password) < 6:
|
if len(new_password) < 6:
|
||||||
return jsonify({"error": "新密码至少6位"}), 400
|
return jsonify({"error": "新密码至少6位"}), 400
|
||||||
|
|
||||||
# 验证当前用户的旧密码
|
# 模拟环境直接返回成功
|
||||||
user = request.current_user
|
|
||||||
username = user.get('username', '')
|
|
||||||
mock_user = MOCK_USERS.get(username)
|
|
||||||
if mock_user and mock_user['password'] != old_password:
|
|
||||||
return jsonify({"error": "旧密码错误"}), 401
|
|
||||||
|
|
||||||
# 模拟环境返回成功(不实际修改密码,mock 数据是静态的)
|
|
||||||
return jsonify({"message": "密码修改成功(模拟)"})
|
return jsonify({"message": "密码修改成功(模拟)"})
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ from auth.gateway import require_gateway_auth
|
|||||||
from auth.security import validate_query, filter_response
|
from auth.security import validate_query, filter_response
|
||||||
from config import RAG_CHAT_MODEL
|
from config import RAG_CHAT_MODEL
|
||||||
from core.llm_utils import call_llm, call_llm_stream
|
from core.llm_utils import call_llm, call_llm_stream
|
||||||
from core.prompt_guard import detect_injection, sanitize_user_input
|
|
||||||
|
|
||||||
|
|
||||||
def _get_vlm_cache(image_path: str) -> Optional[str]:
|
def _get_vlm_cache(image_path: str) -> Optional[str]:
|
||||||
@@ -263,6 +262,35 @@ def _strip_semantic_prefix(doc: str, chunk_type: str) -> str:
|
|||||||
|
|
||||||
return doc
|
return doc
|
||||||
|
|
||||||
|
def _process_table_doc(doc: str, meta: Dict) -> str:
|
||||||
|
"""
|
||||||
|
处理单个切片的 doc:精简表格语义前缀 + 注入嵌入图片 URL
|
||||||
|
|
||||||
|
集中处理两处共用逻辑(正常路径和预算截断路径),避免重复代码。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
doc: 切片原始 doc
|
||||||
|
meta: 切片 metadata
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
处理后的 doc
|
||||||
|
"""
|
||||||
|
doc = _strip_semantic_prefix(doc, meta.get('chunk_type', ''))
|
||||||
|
if meta.get('chunk_type') == 'table' and meta.get('images_json'):
|
||||||
|
try:
|
||||||
|
img_list = json.loads(meta['images_json'])
|
||||||
|
if img_list:
|
||||||
|
img_urls = [
|
||||||
|
f"/images/{img.get('id', '')}"
|
||||||
|
for img in img_list
|
||||||
|
if isinstance(img, dict) and img.get('id')
|
||||||
|
]
|
||||||
|
if img_urls:
|
||||||
|
doc += "\n\n[该表格包含以下图片,可在回答中引用]: " + ", ".join(img_urls)
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
pass
|
||||||
|
return doc
|
||||||
|
|
||||||
|
|
||||||
def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks: int,
|
def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks: int,
|
||||||
min_score: float = 0.0) -> List[Dict]:
|
min_score: float = 0.0) -> List[Dict]:
|
||||||
@@ -415,36 +443,6 @@ def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks
|
|||||||
return ordered
|
return ordered
|
||||||
|
|
||||||
|
|
||||||
def _process_table_doc(doc: str, meta: Dict) -> str:
|
|
||||||
"""
|
|
||||||
处理单个切片的 doc:精简表格语义前缀 + 注入嵌入图片 URL
|
|
||||||
|
|
||||||
集中处理两处共用逻辑(正常路径和预算截断路径),避免重复代码。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
doc: 切片原始 doc
|
|
||||||
meta: 切片 metadata
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
处理后的 doc
|
|
||||||
"""
|
|
||||||
doc = _strip_semantic_prefix(doc, meta.get('chunk_type', ''))
|
|
||||||
if meta.get('chunk_type') == 'table' and meta.get('images_json'):
|
|
||||||
try:
|
|
||||||
img_list = json.loads(meta['images_json'])
|
|
||||||
if img_list:
|
|
||||||
img_urls = [
|
|
||||||
f"/images/{img.get('id', '')}"
|
|
||||||
for img in img_list
|
|
||||||
if isinstance(img, dict) and img.get('id')
|
|
||||||
]
|
|
||||||
if img_urls:
|
|
||||||
doc += "\n\n[该表格包含以下图片,可在回答中引用]: " + ", ".join(img_urls)
|
|
||||||
except (json.JSONDecodeError, TypeError):
|
|
||||||
pass
|
|
||||||
return doc
|
|
||||||
|
|
||||||
|
|
||||||
def _section_similarity(section_a: str, section_b: str) -> float:
|
def _section_similarity(section_a: str, section_b: str) -> float:
|
||||||
"""
|
"""
|
||||||
计算两个章节路径的层级相似度(数据驱动,无需硬编码格式假设)。
|
计算两个章节路径的层级相似度(数据驱动,无需硬编码格式假设)。
|
||||||
@@ -657,6 +655,16 @@ def _build_context_with_budget(contexts: List[Dict], max_chars: int, soft_limit:
|
|||||||
return "\n\n".join(parts)
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_agentic_rag() -> 'AgenticRAG':
|
||||||
|
"""
|
||||||
|
获取 AgenticRAG 实例
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
AgenticRAG: 当前应用中的 AgenticRAG 实例
|
||||||
|
"""
|
||||||
|
return current_app.config['AGENTIC_RAG']
|
||||||
|
|
||||||
|
|
||||||
def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
自动为回答添加引用标记(按段落级别匹配,jieba 分词精准匹配)
|
自动为回答添加引用标记(按段落级别匹配,jieba 分词精准匹配)
|
||||||
@@ -682,16 +690,12 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
|||||||
if not contexts:
|
if not contexts:
|
||||||
return {"answer_with_refs": answer, "citations": []}
|
return {"answer_with_refs": answer, "citations": []}
|
||||||
|
|
||||||
# 按 (collection, chunk_id) 复合键组织 contexts,防止跨库同名文件覆盖
|
# 按 chunk_id 组织 contexts
|
||||||
ctx_by_chunk = {}
|
ctx_by_chunk = {}
|
||||||
for ctx in contexts:
|
for ctx in contexts:
|
||||||
meta = ctx.get('meta', {})
|
meta = ctx.get('meta', {})
|
||||||
chunk_id = meta.get('chunk_id') or f"{meta.get('source')}_{meta.get('chunk_index', 0)}"
|
chunk_id = meta.get('chunk_id') or f"{meta.get('source')}_{meta.get('chunk_index', 0)}"
|
||||||
coll = meta.get('_collection') or meta.get('collection') or ''
|
ctx_by_chunk[chunk_id] = ctx
|
||||||
composite_key = f"{coll}/{chunk_id}" if coll else chunk_id
|
|
||||||
# 保存原始 chunk_id,用于对外输出(ref tag / citation)
|
|
||||||
ctx['_raw_chunk_id'] = chunk_id
|
|
||||||
ctx_by_chunk[composite_key] = ctx
|
|
||||||
|
|
||||||
# jieba 分词函数(fallback 到字符级)
|
# jieba 分词函数(fallback 到字符级)
|
||||||
try:
|
try:
|
||||||
@@ -751,46 +755,25 @@ def _attach_citations(answer: str, contexts: List[Dict]) -> Dict[str, Any]:
|
|||||||
if len(candidates) > 1 and (candidates[0][1] - candidates[1][1]) < 0.1:
|
if len(candidates) > 1 and (candidates[0][1] - candidates[1][1]) < 0.1:
|
||||||
selected_ids.append(candidates[1][0])
|
selected_ids.append(candidates[1][0])
|
||||||
|
|
||||||
# 按 _raw_chunk_id 去重:不同 composite_key 可能指向同一个底层 chunk
|
|
||||||
# 避免同一 chunk 产生重复引用标记(如 [3][3])
|
|
||||||
seen_raw_ids = set()
|
|
||||||
deduped_ids = []
|
|
||||||
for cid in selected_ids:
|
|
||||||
raw_id = ctx_by_chunk[cid].get('_raw_chunk_id', cid)
|
|
||||||
if raw_id not in seen_raw_ids:
|
|
||||||
seen_raw_ids.add(raw_id)
|
|
||||||
deduped_ids.append(cid)
|
|
||||||
selected_ids = deduped_ids
|
|
||||||
|
|
||||||
if selected_ids:
|
if selected_ids:
|
||||||
for cid in selected_ids:
|
for cid in selected_ids:
|
||||||
if cid not in cited_set:
|
if cid not in cited_set:
|
||||||
cited_set.add(cid)
|
cited_set.add(cid)
|
||||||
cited_chunks_ordered.append(cid)
|
cited_chunks_ordered.append(cid)
|
||||||
# 在段落末尾插入引用标记(使用原始 chunk_id,不暴露复合键)
|
# 在段落末尾插入引用标记(多个引用连续排列)
|
||||||
ref_tags = "".join(
|
ref_tags = "".join(f"[ref:{cid}]" for cid in selected_ids)
|
||||||
f"[ref:{ctx_by_chunk[cid].get('_raw_chunk_id', cid)}]"
|
|
||||||
for cid in selected_ids
|
|
||||||
)
|
|
||||||
result_parts.append(f"{para}{ref_tags}{sep}")
|
result_parts.append(f"{para}{ref_tags}{sep}")
|
||||||
else:
|
else:
|
||||||
result_parts.append(para + sep)
|
result_parts.append(para + sep)
|
||||||
|
|
||||||
# 构建引用列表(按出现顺序),使用原始 chunk_id 构建 citation
|
# 构建引用列表(按出现顺序)
|
||||||
# 按 _raw_chunk_id 去重,避免同一 chunk 产生重复引用条目
|
|
||||||
citations = []
|
citations = []
|
||||||
seen_citation_raw_ids = set()
|
for chunk_id in cited_chunks_ordered:
|
||||||
for composite_key in cited_chunks_ordered:
|
ctx = ctx_by_chunk.get(chunk_id)
|
||||||
ctx = ctx_by_chunk.get(composite_key)
|
|
||||||
if ctx:
|
if ctx:
|
||||||
raw_id = ctx.get('_raw_chunk_id') or composite_key
|
|
||||||
if raw_id in seen_citation_raw_ids:
|
|
||||||
continue # 同一 chunk 已在引用列表中,跳过
|
|
||||||
seen_citation_raw_ids.add(raw_id)
|
|
||||||
meta = ctx.get('meta', {})
|
meta = ctx.get('meta', {})
|
||||||
full_content = ctx.get('doc', '')
|
full_content = ctx.get('doc', '')
|
||||||
citation = _build_citation(meta, full_content)
|
citation = _build_citation(meta, full_content)
|
||||||
citation['chunk_id'] = raw_id
|
|
||||||
citations.append(citation)
|
citations.append(citation)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -879,7 +862,7 @@ def _build_citation(meta: Dict, full_content: str = '') -> Dict[str, Any]:
|
|||||||
"chunk_id": chunk_id_raw,
|
"chunk_id": chunk_id_raw,
|
||||||
"chunk_index": chunk_index, # 全局切片序号,用于精准定位文档位置
|
"chunk_index": chunk_index, # 全局切片序号,用于精准定位文档位置
|
||||||
"source": meta.get('source', ''),
|
"source": meta.get('source', ''),
|
||||||
"collection": meta.get('_collection') or meta.get('collection', ''), # 所属向量库,用于前端文档预览跳转
|
"collection": meta.get('_collection', ''), # 所属向量库,用于前端文档预览跳转
|
||||||
"doc_type": meta.get('doc_type', 'other'),
|
"doc_type": meta.get('doc_type', 'other'),
|
||||||
"section": _clean_section(meta.get('section', '')),
|
"section": _clean_section(meta.get('section', '')),
|
||||||
"preview": meta.get('preview', ''),
|
"preview": meta.get('preview', ''),
|
||||||
@@ -1023,8 +1006,8 @@ def score_image_relevance(query: str, meta: Dict, doc: str = '') -> float:
|
|||||||
|
|
||||||
# 3. 整体文本相似度(字符级别)
|
# 3. 整体文本相似度(字符级别)
|
||||||
if search_text:
|
if search_text:
|
||||||
# 复用已过滤停用词的 query_keywords,避免字符级误删(如"表现"→"现")
|
# 检查查询的核心词是否在描述中
|
||||||
query_core = "".join(query_keywords)
|
query_core = re.sub(r'[图表图片如图所示]', '', query) # 去掉泛词
|
||||||
if query_core:
|
if query_core:
|
||||||
overlap = len(set(query_core) & set(search_text))
|
overlap = len(set(query_core) & set(search_text))
|
||||||
score += min(overlap * 0.2, 3.0)
|
score += min(overlap * 0.2, 3.0)
|
||||||
@@ -1151,39 +1134,26 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
"""
|
"""
|
||||||
import re
|
import re
|
||||||
|
|
||||||
# 动态预算:数据驱动,不依赖硬编码关键词列表
|
# 动态预算:列举型查询允许更多图片
|
||||||
# 核心策略:宽松预选 + 后置过滤(_filter_images_by_answer)精准裁剪
|
list_keywords = ["哪些", "有什么", "包含", "列出", "所有", "全部"]
|
||||||
|
is_list_query = any(kw in query for kw in list_keywords)
|
||||||
|
|
||||||
# 精确查图:用户指定了具体图号(如 "图2.3")—— 结构化模式匹配,非硬编码
|
# 检测查询中的图片编号(如 "图2.1")
|
||||||
figure_pattern = r'图\s*(\d+\.?\d*)'
|
figure_pattern = r'图\s*(\d+\.?\d*)'
|
||||||
figure_matches = re.findall(figure_pattern, query)
|
figure_matches = re.findall(figure_pattern, query)
|
||||||
has_figure_query = bool(figure_matches)
|
has_figure_query = bool(figure_matches)
|
||||||
|
|
||||||
# 数据驱动的图片意图检测:检查检索结果中是否包含图片/图表类型切片
|
# 新增:从检索文本中提取图表引用(见表2.2、见图2.5 等)
|
||||||
# 原理:如果向量检索返回了 image/chart 类型 chunk 或含 images_json 的 table chunk,
|
# 同时记录引用所在的文件来源
|
||||||
# 说明知识库中存在与查询语义相关的图片内容,应给予展示机会
|
|
||||||
has_image_data = False
|
|
||||||
_image_chunk_count = 0
|
|
||||||
_table_image_count = 0
|
|
||||||
for ctx in contexts:
|
|
||||||
meta = ctx.get('meta', {})
|
|
||||||
ct = meta.get('chunk_type', '')
|
|
||||||
if ct in ('image', 'chart'):
|
|
||||||
_image_chunk_count += 1
|
|
||||||
has_image_data = True
|
|
||||||
if ct == 'table' and meta.get('images_json'):
|
|
||||||
_table_image_count += 1
|
|
||||||
has_image_data = True
|
|
||||||
|
|
||||||
# 从检索文本中提取图表引用(见表2.2、见图2.5 等)
|
|
||||||
# 重要:只从语义相关的 top 5 文本块提取,避免不相关引用干扰
|
# 重要:只从语义相关的 top 5 文本块提取,避免不相关引用干扰
|
||||||
referenced_figures = {} # {图号: set(文件来源)}
|
referenced_figures = {} # {图号: set(文件来源)}
|
||||||
referenced_tables = {} # {表号: set(文件来源)}
|
referenced_tables = {} # {表号: set(文件来源)}
|
||||||
|
|
||||||
for ctx in contexts[:5]:
|
for ctx in contexts[:5]: # 只从前5个最相关的文本块提取引用
|
||||||
doc_text = ctx.get('doc', '')
|
doc_text = ctx.get('doc', '')
|
||||||
source = ctx.get('meta', {}).get('source', '')
|
source = ctx.get('meta', {}).get('source', '')
|
||||||
|
|
||||||
|
# 提取 "见图X.X"、"如图X.X" 或单独的 "图X.X"
|
||||||
fig_refs = re.findall(r'(?:[见如])?图\s*(\d+\.?\d*)', doc_text)
|
fig_refs = re.findall(r'(?:[见如])?图\s*(\d+\.?\d*)', doc_text)
|
||||||
for fig_num in fig_refs:
|
for fig_num in fig_refs:
|
||||||
if fig_num not in referenced_figures:
|
if fig_num not in referenced_figures:
|
||||||
@@ -1191,6 +1161,7 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
if source:
|
if source:
|
||||||
referenced_figures[fig_num].add(source)
|
referenced_figures[fig_num].add(source)
|
||||||
|
|
||||||
|
# 提取 "见表X.X"、"如表X.X" 或单独的 "表X.X"
|
||||||
table_refs = re.findall(r'(?:[见如])?表\s*(\d+\.?\d*)', doc_text)
|
table_refs = re.findall(r'(?:[见如])?表\s*(\d+\.?\d*)', doc_text)
|
||||||
for table_num in table_refs:
|
for table_num in table_refs:
|
||||||
if table_num not in referenced_tables:
|
if table_num not in referenced_tables:
|
||||||
@@ -1202,84 +1173,63 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
|
|
||||||
# 获取检索结果中涉及的主要文件来源
|
# 获取检索结果中涉及的主要文件来源
|
||||||
primary_sources = set()
|
primary_sources = set()
|
||||||
for ctx in contexts[:5]:
|
for ctx in contexts[:5]: # 只看前5个最相关的
|
||||||
source = ctx.get('meta', {}).get('source', '')
|
source = ctx.get('meta', {}).get('source', '')
|
||||||
if source:
|
if source:
|
||||||
primary_sources.add(source)
|
primary_sources.add(source)
|
||||||
|
|
||||||
# 动态预算:根据检索结果数据驱动设置
|
# 图片意图检测:区分精确查图和泛指
|
||||||
# 后置过滤 _filter_images_by_answer 会根据回答内容精准裁剪
|
strong_image_keywords = ["示意图", "流程图", "结构图", "过程线", "曲线图", "分布图", "图示", "看图", "显示图"]
|
||||||
|
weak_image_keywords = ["图片", "图表", "如图", "图", "统计"]
|
||||||
|
|
||||||
|
has_strong_image_intent = any(kw in query for kw in strong_image_keywords)
|
||||||
|
has_weak_image_intent = any(kw in query for kw in weak_image_keywords)
|
||||||
|
|
||||||
|
# 精确查图:用户指定了具体图号(如 "图2.3"),只返回最匹配的 1-2 张
|
||||||
if has_figure_query:
|
if has_figure_query:
|
||||||
# 精确查图:用户指定了具体图号
|
|
||||||
MAX_IMAGES = 2
|
MAX_IMAGES = 2
|
||||||
MIN_SCORE = 5.0
|
MIN_SCORE = 5.0 # 精确匹配应该高分
|
||||||
elif has_image_data:
|
# 强图片意图:明确要看某种图
|
||||||
# 检索结果中有图片数据 → 宽松预算,给后置过滤留足候选空间
|
elif has_strong_image_intent:
|
||||||
MAX_IMAGES = 5
|
|
||||||
MIN_SCORE = 2.0
|
|
||||||
elif has_referenced_figures:
|
|
||||||
# 检索文本中引用了图表编号
|
|
||||||
MAX_IMAGES = 3
|
MAX_IMAGES = 3
|
||||||
MIN_SCORE = 2.0
|
MIN_SCORE = 5.0 # 提高阈值,避免不相关图片通过
|
||||||
|
# 列举型查询
|
||||||
|
elif is_list_query:
|
||||||
|
MAX_IMAGES = 5
|
||||||
|
MIN_SCORE = 3.0
|
||||||
|
# 有图表引用:检索文本中提到了图表
|
||||||
|
elif has_referenced_figures:
|
||||||
|
MAX_IMAGES = 3
|
||||||
|
MIN_SCORE = 2.0 # 降低阈值,让引用的图表能通过
|
||||||
|
# 弱图片意图:只是提到"图"字,可能是泛指(如 "发电量图")
|
||||||
|
elif has_weak_image_intent:
|
||||||
|
MAX_IMAGES = 1 # 只返回最相关的一张
|
||||||
|
MIN_SCORE = 2.0 # 降低阈值,让语义相关的图片能通过
|
||||||
|
# 普通查询
|
||||||
else:
|
else:
|
||||||
# 无图片数据 → 保守默认值
|
|
||||||
MAX_IMAGES = 2
|
MAX_IMAGES = 2
|
||||||
MIN_SCORE = 3.0
|
MIN_SCORE = 3.0
|
||||||
|
|
||||||
# 动态调整:当表格嵌入大量图片时,提升上限以展示完整内容
|
# 获取检索结果中涉及的主要章节(只看前 3 个最相关的文本块)
|
||||||
if has_image_data and _table_image_count > 0:
|
primary_sections = set()
|
||||||
total_table_images = 0
|
|
||||||
for ctx in contexts:
|
|
||||||
meta = ctx.get('meta', {})
|
|
||||||
if meta.get('chunk_type') == 'table' and meta.get('images_json'):
|
|
||||||
try:
|
|
||||||
total_table_images += len(json.loads(meta['images_json']))
|
|
||||||
except (json.JSONDecodeError, TypeError):
|
|
||||||
pass
|
|
||||||
if total_table_images > MAX_IMAGES:
|
|
||||||
MAX_IMAGES = min(total_table_images, 15) # 上限 15,防止图片过多
|
|
||||||
|
|
||||||
# 获取检索结果中涉及的主要章节路径(只看前 3 个最相关的文本块)
|
|
||||||
primary_section_paths = set()
|
|
||||||
for ctx in contexts[:3]:
|
for ctx in contexts[:3]:
|
||||||
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||||
if section:
|
if section:
|
||||||
primary_section_paths.add(section)
|
# 提取章节编号
|
||||||
|
# 优先匹配 X.X 格式(如 "2.3发电"),再匹配 第X章 格式
|
||||||
# ========== P1.5 预计算:表格主题相关性评分 ==========
|
section_num = re.search(r'(\d+\.\d+)', section)
|
||||||
# 从查询中提取关键词片段(使用 jieba 分词 + 2字及以上的词,自动过滤停用词)
|
if not section_num:
|
||||||
try:
|
# 尝试匹配 "第X章" 格式
|
||||||
import jieba
|
chapter_match = re.search(r'第\s*(\d+)\s*章', section)
|
||||||
_query_kw_segments = [w for w in jieba.lcut(query)
|
if chapter_match:
|
||||||
if len(w) >= 2 and re.search(r'[\u4e00-\u9fff]', w)]
|
section_num = chapter_match
|
||||||
except ImportError:
|
if section_num:
|
||||||
# jieba 不可用时回退到 bigram
|
primary_sections.add(section_num.group(1))
|
||||||
_chars = re.findall(r'[\u4e00-\u9fff]', query)
|
|
||||||
_query_kw_segments = [_chars[i] + _chars[i+1] for i in range(len(_chars) - 1)]
|
|
||||||
|
|
||||||
# 对所有含 images_json 的表格切片计算主题匹配分
|
|
||||||
_table_topic_scores = {}
|
|
||||||
for _tc in contexts:
|
|
||||||
_tm = _tc.get('meta', {})
|
|
||||||
if _tm.get('chunk_type') == 'table' and _tm.get('images_json'):
|
|
||||||
_t_section = (_tm.get('section', '') or _tm.get('section_path', ''))
|
|
||||||
_t_title = _tm.get('title', '') or ''
|
|
||||||
_t_combined = _t_title + _t_section
|
|
||||||
_score = sum(1 for kw in _query_kw_segments if kw in _t_combined)
|
|
||||||
_table_topic_scores[id(_tc)] = _score
|
|
||||||
|
|
||||||
# 自适应阈值:要求至少匹配 70% 的最佳表格得分(至少 2 分)
|
|
||||||
# 例如查询 "设施设备的参考样式" → 4 个关键词 → 最佳匹配 4 → 阈值 max(2, 2) = 2
|
|
||||||
# "形象识别标识" 只匹配 "参考"+"样式" = 2 → 但阈值=3(70%of4)时被过滤
|
|
||||||
_best_table_score = max(_table_topic_scores.values()) if _table_topic_scores else 0
|
|
||||||
_table_topic_threshold = max(2, round(_best_table_score * 0.7)) if _best_table_score >= 2 else 0
|
|
||||||
|
|
||||||
scored_images = []
|
scored_images = []
|
||||||
for ctx in contexts:
|
for ctx in contexts:
|
||||||
meta = ctx.get('meta', {})
|
meta = ctx.get('meta', {})
|
||||||
chunk_type = meta.get('chunk_type', 'text')
|
chunk_type = meta.get('chunk_type', 'text')
|
||||||
s = None # P1 评分,用于 P1.5 继承
|
|
||||||
doc = ctx.get('doc', '') # 默认文档内容
|
|
||||||
|
|
||||||
# 处理图片类型和有关联图片的表格类型
|
# 处理图片类型和有关联图片的表格类型
|
||||||
if meta.get('image_path') and chunk_type in ('image', 'chart', 'table'):
|
if meta.get('image_path') and chunk_type in ('image', 'chart', 'table'):
|
||||||
@@ -1309,19 +1259,16 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
|
|
||||||
# 图片章节
|
# 图片章节
|
||||||
img_section = meta.get('section', '') or meta.get('section_path', '')
|
img_section = meta.get('section', '') or meta.get('section_path', '')
|
||||||
|
# 只匹配 X.X 格式的章节号,避免匹配年份
|
||||||
|
img_section_num = re.search(r'(\d+\.\d+)', img_section)
|
||||||
|
img_section_id = img_section_num.group(1) if img_section_num else None
|
||||||
|
|
||||||
# ========== 章节关联检测:基于层级相似度,无需硬编码格式假设 ==========
|
# ========== 核心修复:图片必须与主要文本切片章节关联 ==========
|
||||||
# 计算图片章节与主要检索结果的最高相似度
|
# 如果有主要章节,且图片章节不在其中,大幅降低分数
|
||||||
section_penalty = 0.0
|
section_penalty = 0.0
|
||||||
max_section_sim = 0.0
|
if primary_sections and img_section_id and img_section_id not in primary_sections:
|
||||||
if primary_section_paths:
|
|
||||||
for ps in primary_section_paths:
|
|
||||||
sim = _section_similarity(img_section, ps)
|
|
||||||
max_section_sim = max(max_section_sim, sim)
|
|
||||||
# 当相似度低于阈值且有足够的章节信息时,判定为不相关
|
|
||||||
if primary_section_paths and img_section and max_section_sim < 0.3:
|
|
||||||
# 图片章节与主要检索结果不匹配,惩罚
|
# 图片章节与主要检索结果不匹配,惩罚
|
||||||
section_penalty = -5.0
|
section_penalty = -5.0 # 大幅降低分数
|
||||||
# 除非图片被文本切片明确引用
|
# 除非图片被文本切片明确引用
|
||||||
is_referenced = False
|
is_referenced = False
|
||||||
for fig_num in referenced_figures:
|
for fig_num in referenced_figures:
|
||||||
@@ -1344,10 +1291,10 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
for fig_num, sources in referenced_figures.items():
|
for fig_num, sources in referenced_figures.items():
|
||||||
# 只检查 doc 字段,不检查 meta(避免 section 中的误匹配)
|
# 只检查 doc 字段,不检查 meta(避免 section 中的误匹配)
|
||||||
if f"图{fig_num}" in doc or f"图 {fig_num}" in doc:
|
if f"图{fig_num}" in doc or f"图 {fig_num}" in doc:
|
||||||
# 使用层级相似度判断章节关联性
|
# 检查图片章节是否与主要章节匹配
|
||||||
section_match = max_section_sim >= 0.3
|
section_match = img_section_id and img_section_id in primary_sections
|
||||||
|
|
||||||
# 章节匹配时才加分
|
# P2:只有章节匹配才加分,移除"s >= 5.0"漏洞
|
||||||
if section_match:
|
if section_match:
|
||||||
# 图号匹配加分
|
# 图号匹配加分
|
||||||
s += 8.0
|
s += 8.0
|
||||||
@@ -1362,10 +1309,10 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
for table_num, sources in referenced_tables.items():
|
for table_num, sources in referenced_tables.items():
|
||||||
# 只检查 doc 字段
|
# 只检查 doc 字段
|
||||||
if f"表{table_num}" in doc or f"表 {table_num}" in doc:
|
if f"表{table_num}" in doc or f"表 {table_num}" in doc:
|
||||||
# 使用层级相似度判断章节关联性
|
# 检查图片章节是否与主要章节匹配
|
||||||
section_match = max_section_sim >= 0.3
|
section_match = img_section_id and img_section_id in primary_sections
|
||||||
|
|
||||||
# 章节匹配时才加分
|
# P2:只有章节匹配才加分,移除"s >= 5.0"漏洞
|
||||||
if section_match:
|
if section_match:
|
||||||
# 表号匹配加分
|
# 表号匹配加分
|
||||||
s += 8.0
|
s += 8.0
|
||||||
@@ -1393,47 +1340,6 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
|||||||
# ========== P1.5:处理表格切片的 images_json(跨页表格多图)==========
|
# ========== P1.5:处理表格切片的 images_json(跨页表格多图)==========
|
||||||
# 当表格切片有 images_json 字段时,添加所有关联图片
|
# 当表格切片有 images_json 字段时,添加所有关联图片
|
||||||
if chunk_type == 'table' and meta.get('images_json'):
|
if chunk_type == 'table' and meta.get('images_json'):
|
||||||
# 如果 P1 未执行(表格无 image_path),需要独立计算分数
|
|
||||||
if s is None:
|
|
||||||
doc = ctx.get('image_description', '') or meta.get('vlm_desc', '') or ctx.get('doc', '')
|
|
||||||
s = score_image_relevance(query, meta, doc)
|
|
||||||
|
|
||||||
# 章节相关性过滤:使用层级相似度,无需硬编码章节格式假设
|
|
||||||
table_section = meta.get('section', '') or meta.get('section_path', '')
|
|
||||||
|
|
||||||
section_relevant = True
|
|
||||||
if primary_section_paths and table_section:
|
|
||||||
# 计算表格章节与所有主要章节的最高相似度
|
|
||||||
max_sim = max(
|
|
||||||
(_section_similarity(table_section, ps) for ps in primary_section_paths),
|
|
||||||
default=0.0
|
|
||||||
)
|
|
||||||
if max_sim < 0.3:
|
|
||||||
section_relevant = False
|
|
||||||
elif primary_section_paths and not table_section:
|
|
||||||
# 表格无章节信息时优雅降级:不做章节过滤,仅依赖主题分数
|
|
||||||
pass
|
|
||||||
|
|
||||||
if not section_relevant:
|
|
||||||
# 例外:如果表格标题/内容被查询直接提及,仍视为相关
|
|
||||||
table_title = meta.get('title', '') or ''
|
|
||||||
table_doc = ctx.get('doc', '') or ''
|
|
||||||
if table_title and table_title in query:
|
|
||||||
section_relevant = True
|
|
||||||
elif table_doc and any(kw in table_doc for kw in query.split() if len(kw) >= 2):
|
|
||||||
section_relevant = True
|
|
||||||
|
|
||||||
if not section_relevant:
|
|
||||||
logger.debug(f"P1.5 跳过无关表格图片: path={table_section}, title={meta.get('title', '')}")
|
|
||||||
continue # 跳过此表格的所有嵌入图片
|
|
||||||
|
|
||||||
# 标题/主题相关性过滤:基于预计算的关键词匹配评分
|
|
||||||
# 如果最佳表格得分 >= 2,则过滤掉得分为 0 的表格
|
|
||||||
_this_topic_score = _table_topic_scores.get(id(ctx), 0)
|
|
||||||
if _this_topic_score < _table_topic_threshold:
|
|
||||||
logger.debug(f"P1.5 跳过主题不匹配表格: topic_score={_this_topic_score}, threshold={_table_topic_threshold}, title={meta.get('title', '')}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
images_list = json.loads(meta['images_json'])
|
images_list = json.loads(meta['images_json'])
|
||||||
for img_info in images_list:
|
for img_info in images_list:
|
||||||
@@ -1793,18 +1699,6 @@ def rag():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"创建会话失败: {e}")
|
logger.debug(f"创建会话失败: {e}")
|
||||||
|
|
||||||
# ==================== collections 历史推断(开发环境) ====================
|
|
||||||
# 当 collections 未显式指定(或仅为默认 public_kb)且有会话历史时,
|
|
||||||
# 从历史消息中推断上次使用的 KB,自动恢复以避免用户忘切 KB
|
|
||||||
if (not collections or collections == ['public_kb']) and history:
|
|
||||||
for _msg in reversed(history):
|
|
||||||
if _msg.get("role") == "assistant":
|
|
||||||
_meta = _msg.get("metadata", {})
|
|
||||||
if isinstance(_meta, dict) and _meta.get("collections"):
|
|
||||||
collections = _meta["collections"]
|
|
||||||
logger.info(f"[KB推断] 从历史推断 collections: {collections}")
|
|
||||||
break
|
|
||||||
|
|
||||||
# 提前获取 session_repo 引用,避免在生成器内部访问 current_app
|
# 提前获取 session_repo 引用,避免在生成器内部访问 current_app
|
||||||
# (生成器执行时应用上下文可能已结束)
|
# (生成器执行时应用上下文可能已结束)
|
||||||
session_repo_ref = None
|
session_repo_ref = None
|
||||||
@@ -1904,9 +1798,53 @@ def rag():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"意图分析失败: {e},继续执行检索流程")
|
logger.warning(f"意图分析失败: {e},继续执行检索流程")
|
||||||
|
|
||||||
# 构建检索查询:使用改写后的完整问题(解决追问偏离问题)
|
|
||||||
retrieval_query = intent.rewritten_query if (intent and intent.rewritten_query) else message
|
retrieval_query = intent.rewritten_query if (intent and intent.rewritten_query) else message
|
||||||
|
|
||||||
|
# 1.5 语义缓存检查(跳过检索+生成全流程)
|
||||||
|
_semantic_cache_emb = None
|
||||||
|
try:
|
||||||
|
from core.semantic_cache import get_semantic_cache
|
||||||
|
from core.engine import get_engine as _get_eng
|
||||||
|
_sc = get_semantic_cache()
|
||||||
|
_eng = _get_eng()
|
||||||
|
if _sc and _eng and hasattr(_eng, 'embedding_model'):
|
||||||
|
# 缓存 key 使用 retrieval_query + collections,避免不同上下文的追问命中错误缓存
|
||||||
|
_cache_collections = ','.join(sorted(collections)) if collections else ''
|
||||||
|
_cache_key_text = f"{retrieval_query}|{_cache_collections}"
|
||||||
|
_semantic_cache_emb = _eng.embedding_model.encode(_cache_key_text)
|
||||||
|
cached = _sc.get(_semantic_cache_emb)
|
||||||
|
# 防御性校验:必须是 RAG 回答缓存(非 intent_analyzer 缓存),且回答非空
|
||||||
|
if cached is not None and cached.get("cache_type") == "rag_answer" and cached.get("answer"):
|
||||||
|
logger.info(f"[语义缓存] 命中: {message[:50]}...")
|
||||||
|
cached_answer = cached.get("answer", "")
|
||||||
|
# 流式返回缓存的答案
|
||||||
|
yield f"data: {json.dumps({'type': 'start', 'message': '正在检索知识库...'}, ensure_ascii=False)}\n\n"
|
||||||
|
# 分块发送缓存答案
|
||||||
|
chunk_size = 20
|
||||||
|
for i in range(0, len(cached_answer), chunk_size):
|
||||||
|
chunk = cached_answer[i:i+chunk_size]
|
||||||
|
full_answer.append(chunk)
|
||||||
|
yield f"data: {json.dumps({'type': 'chunk', 'content': chunk}, ensure_ascii=False)}\n\n"
|
||||||
|
finish_event = {
|
||||||
|
"type": "finish",
|
||||||
|
"answer": cached_answer,
|
||||||
|
"mode": "rag",
|
||||||
|
"session_id": session_id,
|
||||||
|
"sources": cached.get("sources", []),
|
||||||
|
"citations": cached.get("citations", []),
|
||||||
|
"images": cached.get("images", []),
|
||||||
|
"tables": cached.get("tables", []),
|
||||||
|
"sections": [],
|
||||||
|
"duration_ms": int((_time.time() - start_time) * 1000),
|
||||||
|
"confidence_score": 1.0,
|
||||||
|
"semantic_cache_hit": True
|
||||||
|
}
|
||||||
|
yield f"data: {json.dumps(finish_event, ensure_ascii=False)}\n\n"
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"[语义缓存] 检查失败: {e}")
|
||||||
|
_semantic_cache_emb = None
|
||||||
|
|
||||||
# 1. 发送开始事件
|
# 1. 发送开始事件
|
||||||
yield f"data: {json.dumps({'type': 'start', 'message': '正在检索知识库...'}, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps({'type': 'start', 'message': '正在检索知识库...'}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
@@ -1917,7 +1855,7 @@ def rag():
|
|||||||
sub_queries = intent.sub_queries
|
sub_queries = intent.sub_queries
|
||||||
|
|
||||||
search_result = search_hybrid(
|
search_result = search_hybrid(
|
||||||
retrieval_query,
|
message,
|
||||||
top_k=RAG_SEARCH_TOP_K,
|
top_k=RAG_SEARCH_TOP_K,
|
||||||
candidates=RAG_SEARCH_CANDIDATES,
|
candidates=RAG_SEARCH_CANDIDATES,
|
||||||
allowed_collections=collections,
|
allowed_collections=collections,
|
||||||
@@ -1938,19 +1876,18 @@ def rag():
|
|||||||
metas = search_result.get('metadatas', [[]])[0]
|
metas = search_result.get('metadatas', [[]])[0]
|
||||||
scores = search_result.get('scores', [[]])[0]
|
scores = search_result.get('scores', [[]])[0]
|
||||||
|
|
||||||
# 图片相关性提升:数据驱动检测(无硬编码关键词)
|
# 图片相关性提升:检测图片编号或强意图
|
||||||
import re
|
import re
|
||||||
figure_pattern = r'图\s*(\d+\.?\d*)'
|
figure_pattern = r'图\s*(\d+\.?\d*)'
|
||||||
figure_matches = re.findall(figure_pattern, retrieval_query)
|
figure_matches = re.findall(figure_pattern, message)
|
||||||
has_figure_query = bool(figure_matches)
|
has_figure_query = bool(figure_matches)
|
||||||
|
|
||||||
# 数据驱动:检查检索结果中是否有图片/图表类型切片
|
# Bug 4 修复:缩小 strong_image_keywords,移除歧义词
|
||||||
has_image_data = any(
|
# "过程线" 是水文术语不是图片意图,"图片"/"图表"/"如图" 太泛
|
||||||
m.get('chunk_type') in ('image', 'chart') for m in metas
|
strong_image_keywords = ["示意图", "流程图", "结构图", "曲线图", "分布图", "图示", "看图", "显示图", "给我看"]
|
||||||
)
|
has_image_intent = has_figure_query or any(kw in message for kw in strong_image_keywords)
|
||||||
has_image_intent = has_figure_query or has_image_data
|
|
||||||
|
|
||||||
# 给图片/图表切片打 boost 标记,供后续 select_images 使用
|
# Bug 2 修复:不重排 contexts,只在 meta 里打标记给后续的 select_images 用
|
||||||
if has_image_intent:
|
if has_image_intent:
|
||||||
for i, (doc, meta, score) in enumerate(zip(docs, metas, scores)):
|
for i, (doc, meta, score) in enumerate(zip(docs, metas, scores)):
|
||||||
if meta.get('chunk_type') in ('image', 'chart'):
|
if meta.get('chunk_type') in ('image', 'chart'):
|
||||||
@@ -1984,8 +1921,12 @@ def rag():
|
|||||||
meta['_retrieval_rank'] = rank
|
meta['_retrieval_rank'] = rank
|
||||||
# 确保 _collection 字段存在(单知识库路径下 ChromaDB 原生不返回此字段)
|
# 确保 _collection 字段存在(单知识库路径下 ChromaDB 原生不返回此字段)
|
||||||
if not meta.get('_collection'):
|
if not meta.get('_collection'):
|
||||||
# 优先使用入库时写入的 collection 字段
|
if collections and len(collections) == 1:
|
||||||
meta['_collection'] = meta.get('collection') or (collections[0] if collections else 'public_kb')
|
meta['_collection'] = collections[0]
|
||||||
|
elif collections:
|
||||||
|
meta['_collection'] = collections[0] # 多知识库时回退到第一个
|
||||||
|
else:
|
||||||
|
meta['_collection'] = 'public_kb'
|
||||||
source_name = meta.get('source', '未知')
|
source_name = meta.get('source', '未知')
|
||||||
if source_name not in seen_sources or score > seen_sources[source_name]['score']:
|
if source_name not in seen_sources or score > seen_sources[source_name]['score']:
|
||||||
doc_type = meta.get('doc_type', 'other')
|
doc_type = meta.get('doc_type', 'other')
|
||||||
@@ -2072,12 +2013,18 @@ def rag():
|
|||||||
missing_figures = referenced_figures - existing_figure_images
|
missing_figures = referenced_figures - existing_figure_images
|
||||||
missing_tables = referenced_tables - existing_table_images
|
missing_tables = referenced_tables - existing_table_images
|
||||||
|
|
||||||
# 计算主要章节路径(用于补充检索过滤)
|
# 计算主要章节(用于补充检索过滤)
|
||||||
primary_section_paths_for_supp = set()
|
primary_sections_for_supplement = set()
|
||||||
for ctx in text_contexts[:3]:
|
for ctx in text_contexts[:3]:
|
||||||
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
section = ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||||
if section:
|
if section:
|
||||||
primary_section_paths_for_supp.add(section)
|
section_num = re.search(r'(\d+\.\d+)', section)
|
||||||
|
if not section_num:
|
||||||
|
chapter_match = re.search(r'第\s*(\d+)\s*章', section)
|
||||||
|
if chapter_match:
|
||||||
|
section_num = chapter_match
|
||||||
|
if section_num:
|
||||||
|
primary_sections_for_supplement.add(section_num.group(1))
|
||||||
|
|
||||||
if missing_figures or missing_tables:
|
if missing_figures or missing_tables:
|
||||||
# 补充检索
|
# 补充检索
|
||||||
@@ -2134,18 +2081,14 @@ def rag():
|
|||||||
|
|
||||||
if is_match:
|
if is_match:
|
||||||
# 额外检查:图片章节是否与主要章节匹配
|
# 额外检查:图片章节是否与主要章节匹配
|
||||||
# 使用层级相似度判断,无需硬编码格式假设
|
# 避免补充检索到不相关的图片
|
||||||
supp_section = supp_meta.get('section', '') or supp_meta.get('section_path', '')
|
supp_section = supp_meta.get('section', '') or supp_meta.get('section_path', '')
|
||||||
|
supp_section_num = re.search(r'(\d+\.\d+)', supp_section)
|
||||||
|
supp_section_id = supp_section_num.group(1) if supp_section_num else None
|
||||||
|
|
||||||
if primary_section_paths_for_supp and supp_section:
|
# 如果图片章节不在主要章节中,跳过
|
||||||
supp_max_sim = max(
|
if primary_sections_for_supplement and supp_section_id and supp_section_id not in primary_sections_for_supplement:
|
||||||
(_section_similarity(supp_section, ps) for ps in primary_section_paths_for_supp),
|
# 不是主要章节的图片,跳过
|
||||||
default=0.0
|
|
||||||
)
|
|
||||||
if supp_max_sim < 0.3:
|
|
||||||
continue
|
|
||||||
elif primary_section_paths_for_supp and not supp_section:
|
|
||||||
# 补充检索的图片无章节信息时优雅降级:跳过
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Bug 6a 修复:补充检索的图片也要做 full_description 替换
|
# Bug 6a 修复:补充检索的图片也要做 full_description 替换
|
||||||
@@ -2184,12 +2127,12 @@ def rag():
|
|||||||
import asyncio
|
import asyncio
|
||||||
from knowledge.lazy_enhance import enhance_retrieved_chunks
|
from knowledge.lazy_enhance import enhance_retrieved_chunks
|
||||||
kb_name = collections[0] if collections else 'public_kb'
|
kb_name = collections[0] if collections else 'public_kb'
|
||||||
asyncio.run(enhance_retrieved_chunks(contexts, retrieval_query, kb_name))
|
asyncio.run(enhance_retrieved_chunks(contexts, message, kb_name))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"懒加载增强失败: {e}")
|
logger.warning(f"懒加载增强失败: {e}")
|
||||||
|
|
||||||
# 3. 选择要展示的图片(Phase 5)
|
# 3. 选择要展示的图片(Phase 5)
|
||||||
selected_images = select_images(contexts, retrieval_query)
|
selected_images = select_images(contexts, message)
|
||||||
|
|
||||||
# 调试事件:图片选择详情
|
# 调试事件:图片选择详情
|
||||||
if IS_DEV:
|
if IS_DEV:
|
||||||
@@ -2197,7 +2140,7 @@ def rag():
|
|||||||
|
|
||||||
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
# 4. 构建 prompt(Phase 6:LLM 图片感知)
|
||||||
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
# Bug 1 修复:文本切片用于 top 5 名额竞争,图片描述不参与竞争
|
||||||
text_contexts = _order_text_contexts_for_prompt(contexts, retrieval_query, MAX_CONTEXT_CHUNKS,
|
text_contexts = _order_text_contexts_for_prompt(contexts, message, MAX_CONTEXT_CHUNKS,
|
||||||
min_score=RERANK_CONTEXT_MIN_SCORE)
|
min_score=RERANK_CONTEXT_MIN_SCORE)
|
||||||
# Phase 2:按字符预算构建上下文
|
# Phase 2:按字符预算构建上下文
|
||||||
_is_comparison = intent and intent.intent == "comparison"
|
_is_comparison = intent and intent.intent == "comparison"
|
||||||
@@ -2218,11 +2161,11 @@ def rag():
|
|||||||
context_text = "\n\n".join(enum_parts)
|
context_text = "\n\n".join(enum_parts)
|
||||||
else:
|
else:
|
||||||
context_text = _build_context_with_budget(text_contexts, CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT)
|
context_text = _build_context_with_budget(text_contexts, CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT)
|
||||||
|
|
||||||
# 表格救援:CrossEncoder 对表格评分偏低,导致表格被预算截断
|
# 表格救援:CrossEncoder 对表格评分偏低,导致表格被预算截断
|
||||||
# 当查询涉及表格但上下文中没有表格数据时,从被截断的切片中补回
|
# 当查询涉及表格但上下文中没有表格数据时,从被截断的切片中补回
|
||||||
context_text = _rescue_table_chunks(text_contexts, context_text, retrieval_query)
|
context_text = _rescue_table_chunks(text_contexts, context_text, retrieval_query)
|
||||||
|
|
||||||
|
|
||||||
# Phase 4:计算置信度分数(top-3 平均 Rerank 分数)
|
# Phase 4:计算置信度分数(top-3 平均 Rerank 分数)
|
||||||
_top_scores = [ctx.get('score', 0) for ctx in text_contexts[:3]]
|
_top_scores = [ctx.get('score', 0) for ctx in text_contexts[:3]]
|
||||||
_confidence_score = round(sum(_top_scores) / len(_top_scores), 4) if _top_scores else 0.0
|
_confidence_score = round(sum(_top_scores) / len(_top_scores), 4) if _top_scores else 0.0
|
||||||
@@ -2230,17 +2173,6 @@ def rag():
|
|||||||
# Bug 6b 优化:直接使用 selected_images 中的 full_description
|
# Bug 6b 优化:直接使用 selected_images 中的 full_description
|
||||||
# 这样 LLM 既能看到文本切片,也能知道图片内容
|
# 这样 LLM 既能看到文本切片,也能知道图片内容
|
||||||
if selected_images:
|
if selected_images:
|
||||||
# 区分表格嵌入图片和独立图片
|
|
||||||
has_table_embedded_images = any(
|
|
||||||
img.get('type') == 'table_image' for img in selected_images
|
|
||||||
)
|
|
||||||
# 检查上下文中是否有表格含嵌入图片
|
|
||||||
has_table_with_images = any(
|
|
||||||
ctx.get('meta', {}).get('chunk_type') == 'table'
|
|
||||||
and ctx.get('meta', {}).get('images_json')
|
|
||||||
for ctx in text_contexts
|
|
||||||
)
|
|
||||||
|
|
||||||
image_descriptions = []
|
image_descriptions = []
|
||||||
for i, img in enumerate(selected_images, 1):
|
for i, img in enumerate(selected_images, 1):
|
||||||
# 直接使用 select_images 时带上的 full_description
|
# 直接使用 select_images 时带上的 full_description
|
||||||
@@ -2253,16 +2185,6 @@ def rag():
|
|||||||
image_descriptions.append(f"【图片{i}】{full_desc}{source_info}")
|
image_descriptions.append(f"【图片{i}】{full_desc}{source_info}")
|
||||||
if image_descriptions:
|
if image_descriptions:
|
||||||
context_text += "\n\n【相关图片信息】\n" + "\n\n".join(image_descriptions)
|
context_text += "\n\n【相关图片信息】\n" + "\n\n".join(image_descriptions)
|
||||||
|
|
||||||
# 根据图片类型给出不同的回答指令
|
|
||||||
if has_table_embedded_images and has_table_with_images:
|
|
||||||
context_text += (
|
|
||||||
"\n\n【回答要求】参考资料中包含表格及其嵌入图片。"
|
|
||||||
"请以**表格形式**呈现数据(保持原始表格结构),"
|
|
||||||
"并在对应单元格中使用 `` 格式嵌入图片。"
|
|
||||||
"不要将表格内容转为纯文本描述,不要把图片与表格分开展示。"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# 添加指令让 LLM 介绍图片
|
# 添加指令让 LLM 介绍图片
|
||||||
context_text += "\n\n【回答要求】回答时请简要介绍每张图片的内容和用途。"
|
context_text += "\n\n【回答要求】回答时请简要介绍每张图片的内容和用途。"
|
||||||
|
|
||||||
@@ -2296,7 +2218,7 @@ def rag():
|
|||||||
)
|
)
|
||||||
enhanced_context = instruction_instruction + "\n\n" + enhanced_context
|
enhanced_context = instruction_instruction + "\n\n" + enhanced_context
|
||||||
|
|
||||||
if _is_enum_query(retrieval_query):
|
if _is_enum_query(message):
|
||||||
enum_instruction = (
|
enum_instruction = (
|
||||||
"\n\n【回答要求】如果参考资料中包含编号列表、禁止情形、要求或条款,"
|
"\n\n【回答要求】如果参考资料中包含编号列表、禁止情形、要求或条款,"
|
||||||
"请按资料中的原始顺序完整列出;不要合并相邻条目,不要跳项,"
|
"请按资料中的原始顺序完整列出;不要合并相邻条目,不要跳项,"
|
||||||
@@ -2393,8 +2315,6 @@ def rag():
|
|||||||
assistant_metadata['sources'] = sources
|
assistant_metadata['sources'] = sources
|
||||||
if citation_result.get('citations'):
|
if citation_result.get('citations'):
|
||||||
assistant_metadata['citations'] = citation_result['citations']
|
assistant_metadata['citations'] = citation_result['citations']
|
||||||
# 记录本次检索使用的向量库(用于后续追问时自动恢复 KB 选择)
|
|
||||||
assistant_metadata['collections'] = collections
|
|
||||||
session_repo_ref.add_message(session_id, 'assistant', filtered_answer, assistant_metadata)
|
session_repo_ref.add_message(session_id, 'assistant', filtered_answer, assistant_metadata)
|
||||||
# 更新会话最后活跃时间
|
# 更新会话最后活跃时间
|
||||||
if hasattr(session_repo_ref, 'update_last_active'):
|
if hasattr(session_repo_ref, 'update_last_active'):
|
||||||
@@ -2435,14 +2355,39 @@ def rag():
|
|||||||
"rerank_cached": rerank_cached,
|
"rerank_cached": rerank_cached,
|
||||||
"total_ms": duration_ms
|
"total_ms": duration_ms
|
||||||
}
|
}
|
||||||
|
# 10.5 写入语义缓存
|
||||||
|
try:
|
||||||
|
from core.semantic_cache import get_semantic_cache
|
||||||
|
_sc = get_semantic_cache()
|
||||||
|
if _sc and filtered_answer:
|
||||||
|
if _semantic_cache_emb is None:
|
||||||
|
from core.engine import get_engine as _get_eng
|
||||||
|
_eng = _get_eng()
|
||||||
|
if _eng and hasattr(_eng, 'embedding_model'):
|
||||||
|
_cache_collections = ','.join(sorted(collections)) if collections else ''
|
||||||
|
_cache_key_text = f"{retrieval_query}|{_cache_collections}"
|
||||||
|
_semantic_cache_emb = _eng.embedding_model.encode(_cache_key_text)
|
||||||
|
if _semantic_cache_emb is not None:
|
||||||
|
_sc.set(_semantic_cache_emb, {
|
||||||
|
"cache_type": "rag_answer",
|
||||||
|
"answer": filtered_answer,
|
||||||
|
"sources": sources,
|
||||||
|
"citations": citation_result.get("citations", []),
|
||||||
|
"images": rich_media.get("images", []),
|
||||||
|
"tables": rich_media.get("tables", []),
|
||||||
|
})
|
||||||
|
logger.debug(f"[语义缓存] 写入成功: {message[:50]}...")
|
||||||
|
except Exception as e:
|
||||||
|
logger.info(f"[语义缓存] 写入失败: {e}")
|
||||||
|
|
||||||
yield f"data: {json.dumps(finish_event, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps(finish_event, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
import logging as _logging
|
import traceback
|
||||||
_logging.getLogger(__name__).error(f"[SSE] RAG 流异常: {e}", exc_info=True)
|
|
||||||
error_event = {
|
error_event = {
|
||||||
"type": "error",
|
"type": "error",
|
||||||
"message": "服务内部错误,请稍后重试"
|
"message": str(e),
|
||||||
|
"traceback": traceback.format_exc()
|
||||||
}
|
}
|
||||||
yield f"data: {json.dumps(error_event, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps(error_event, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
@@ -2471,27 +2416,12 @@ def search():
|
|||||||
"""
|
"""
|
||||||
data = request.json or {}
|
data = request.json or {}
|
||||||
query = data.get('query', '')
|
query = data.get('query', '')
|
||||||
query = sanitize_user_input(query)
|
|
||||||
injection_matches = detect_injection(query)
|
|
||||||
if injection_matches:
|
|
||||||
logger.warning(f"[Chat] 检测到可疑注入: {injection_matches}")
|
|
||||||
top_k = data.get('top_k', 5)
|
top_k = data.get('top_k', 5)
|
||||||
collections = data.get('collections') # 后端传入的知识库列表
|
collections = data.get('collections') # 后端传入的知识库列表
|
||||||
|
|
||||||
if not query:
|
if not query:
|
||||||
return jsonify({'error': 'query is required'}), 400
|
return jsonify({'error': 'query is required'}), 400
|
||||||
|
|
||||||
# 输入安全校验(注入检测、违禁词、长度限制)
|
|
||||||
is_valid, reason = validate_query(query)
|
|
||||||
if not is_valid:
|
|
||||||
return jsonify({'error': reason}), 400
|
|
||||||
|
|
||||||
# top_k 范围校验
|
|
||||||
try:
|
|
||||||
top_k = max(1, min(int(top_k), 50))
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
top_k = 5
|
|
||||||
|
|
||||||
# 如果没有指定 collections,使用默认的公开库
|
# 如果没有指定 collections,使用默认的公开库
|
||||||
if not collections:
|
if not collections:
|
||||||
collections = ['public_kb']
|
collections = ['public_kb']
|
||||||
|
|||||||
@@ -45,7 +45,6 @@ import logging
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
from werkzeug.utils import secure_filename
|
from werkzeug.utils import secure_filename
|
||||||
from auth.gateway import require_gateway_auth
|
from auth.gateway import require_gateway_auth
|
||||||
from config import DEV_MODE
|
|
||||||
from core.status_codes import (
|
from core.status_codes import (
|
||||||
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST,
|
UPLOAD_SUCCESS, BATCH_UPLOAD_SUCCESS, BAD_REQUEST,
|
||||||
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION,
|
NO_FILE, NO_FILE_SELECTED, NO_COLLECTION,
|
||||||
@@ -170,9 +169,9 @@ def serve_document_file(doc_path: str) -> Tuple[Any, int]:
|
|||||||
文件内容或错误响应
|
文件内容或错误响应
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
仅在 DEV_MODE=true 时可用(需在 .env 中显式设置)
|
仅在 DEV_MODE=true 时可用
|
||||||
"""
|
"""
|
||||||
if not DEV_MODE:
|
if os.environ.get('DEV_MODE', 'true').lower() == 'false':
|
||||||
return jsonify({"error": "仅开发环境可用"}), 403
|
return jsonify({"error": "仅开发环境可用"}), 403
|
||||||
|
|
||||||
from config import DOCUMENTS_PATH
|
from config import DOCUMENTS_PATH
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ def get_history(session_id):
|
|||||||
user_id = request.current_user["user_id"]
|
user_id = request.current_user["user_id"]
|
||||||
|
|
||||||
# 验证会话归属
|
# 验证会话归属
|
||||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
sessions = session_manager.get_user_sessions(user_id)
|
||||||
session_ids = [s["session_id"] for s in sessions]
|
session_ids = [s["session_id"] for s in sessions]
|
||||||
|
|
||||||
if session_id not in session_ids:
|
if session_id not in session_ids:
|
||||||
@@ -90,7 +90,7 @@ def delete_session(session_id):
|
|||||||
user_id = request.current_user["user_id"]
|
user_id = request.current_user["user_id"]
|
||||||
|
|
||||||
# 验证会话归属
|
# 验证会话归属
|
||||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
sessions = session_manager.get_user_sessions(user_id)
|
||||||
session_ids = [s["session_id"] for s in sessions]
|
session_ids = [s["session_id"] for s in sessions]
|
||||||
|
|
||||||
if session_id not in session_ids:
|
if session_id not in session_ids:
|
||||||
@@ -111,7 +111,7 @@ def clear_history(session_id):
|
|||||||
user_id = request.current_user["user_id"]
|
user_id = request.current_user["user_id"]
|
||||||
|
|
||||||
# 验证会话归属
|
# 验证会话归属
|
||||||
sessions = session_manager.get_user_sessions(user_id, limit=20)
|
sessions = session_manager.get_user_sessions(user_id)
|
||||||
session_ids = [s["session_id"] for s in sessions]
|
session_ids = [s["session_id"] for s in sessions]
|
||||||
|
|
||||||
if session_id not in session_ids:
|
if session_id not in session_ids:
|
||||||
|
|||||||
@@ -20,12 +20,12 @@
|
|||||||
|
|
||||||
## 模式说明
|
## 模式说明
|
||||||
|
|
||||||
开发模式 (DEV_MODE=true):
|
开发模式 (DEV_MODE=true,默认):
|
||||||
- 支持 mock token 模拟用户:Authorization: Bearer mock-token-admin
|
- 支持 mock token 模拟用户:Authorization: Bearer mock-token-admin
|
||||||
- 无 Header 时自动使用开发测试用户
|
- 无 Header 时自动使用开发测试用户
|
||||||
- 适用于前端测试和开发调试
|
- 适用于前端测试和开发调试
|
||||||
|
|
||||||
生产模式 (DEV_MODE=false,默认):
|
生产模式 (DEV_MODE=false):
|
||||||
- 不需要 Header,直接放行
|
- 不需要 Header,直接放行
|
||||||
- 权限由后端完全控制,通过 collections 参数传入
|
- 权限由后端完全控制,通过 collections 参数传入
|
||||||
- RAG 服务完全无状态,只负责问答检索
|
- RAG 服务完全无状态,只负责问答检索
|
||||||
@@ -34,11 +34,17 @@
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
from flask import request, jsonify
|
from flask import request, jsonify
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Optional
|
||||||
from config import DEV_MODE
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
|
# 加载 .env 文件(从项目根目录)
|
||||||
|
env_path = Path(__file__).parent.parent / '.env'
|
||||||
|
load_dotenv(env_path)
|
||||||
|
|
||||||
|
|
||||||
# ==================== 模拟用户数据(开发环境)====================
|
# ==================== 模拟用户数据(开发环境)====================
|
||||||
# 用于前端模拟登录测试,仅 DEV_MODE=true 时生效(需在 .env 中显式开启)
|
# 用于前端模拟登录测试,仅 DEV_MODE=true 时生效
|
||||||
MOCK_USERS = {
|
MOCK_USERS = {
|
||||||
'admin': {
|
'admin': {
|
||||||
'user_id': 'admin001',
|
'user_id': 'admin001',
|
||||||
@@ -77,19 +83,19 @@ def require_gateway_auth(f):
|
|||||||
"""
|
"""
|
||||||
网关认证装饰器 - 从 Header 读取用户信息
|
网关认证装饰器 - 从 Header 读取用户信息
|
||||||
|
|
||||||
开发模式 (DEV_MODE=true):
|
开发模式 (DEV_MODE=true,默认):
|
||||||
- 支持 mock token: Authorization: Bearer mock-token-admin
|
- 支持 mock token: Authorization: Bearer mock-token-admin
|
||||||
- 无 Header 时自动使用开发测试用户(admin 角色)
|
- 无 Header 时自动使用开发测试用户(admin 角色)
|
||||||
|
|
||||||
生产模式 (DEV_MODE=false,默认):
|
生产模式 (DEV_MODE=false):
|
||||||
- 不需要 Header,直接放行
|
- 不需要 Header,直接放行
|
||||||
- 用户信息设为默认值
|
- 用户信息设为默认值
|
||||||
- 权限由后端通过 collections 参数控制
|
- 权限由后端通过 collections 参数控制
|
||||||
"""
|
"""
|
||||||
@wraps(f)
|
@wraps(f)
|
||||||
def decorated(*args, **kwargs):
|
def decorated(*args, **kwargs):
|
||||||
# 开发模式开关(统一由 config.py 管理)
|
# 开发模式开关(默认开启,生产环境设置 DEV_MODE=false)
|
||||||
dev_mode = DEV_MODE
|
dev_mode = os.environ.get('DEV_MODE', 'true').lower() != 'false'
|
||||||
|
|
||||||
# 开发模式:支持 mock token
|
# 开发模式:支持 mock token
|
||||||
if dev_mode:
|
if dev_mode:
|
||||||
@@ -196,24 +202,11 @@ def can_delete_collection(role: str) -> bool:
|
|||||||
|
|
||||||
def require_role(*roles):
|
def require_role(*roles):
|
||||||
"""
|
"""
|
||||||
角色验证装饰器(开发和生产环境均生效)
|
兼容旧代码 - 权限由后端管理,此装饰器不再执行权限验证
|
||||||
|
|
||||||
需搭配 @require_gateway_auth 使用(先设置 current_user,再验证角色)。
|
|
||||||
|
|
||||||
开发模式: 检查 mock token 对应用户的角色
|
|
||||||
生产模式: 检查网关注入的 X-User-Role Header
|
|
||||||
"""
|
"""
|
||||||
def decorator(f):
|
def decorator(f):
|
||||||
@wraps(f)
|
@wraps(f)
|
||||||
def decorated(*args, **kwargs):
|
def decorated(*args, **kwargs):
|
||||||
if roles:
|
|
||||||
user = get_current_user()
|
|
||||||
if user is None or user.get('role') not in roles:
|
|
||||||
from flask import jsonify
|
|
||||||
return jsonify({
|
|
||||||
"error": "权限不足,需要角色: {}".format(', '.join(roles)),
|
|
||||||
"status": "FORBIDDEN"
|
|
||||||
}), 403
|
|
||||||
return f(*args, **kwargs)
|
return f(*args, **kwargs)
|
||||||
return decorated
|
return decorated
|
||||||
return decorator
|
return decorator
|
||||||
@@ -221,12 +214,11 @@ def require_role(*roles):
|
|||||||
|
|
||||||
def require_collection_permission(operation: str):
|
def require_collection_permission(operation: str):
|
||||||
"""
|
"""
|
||||||
集合权限验证装饰器(开发模式下为占位实现,生产环境权限由网关控制)
|
兼容旧代码 - 权限由后端管理,此装饰器不再执行权限验证
|
||||||
"""
|
"""
|
||||||
def decorator(f):
|
def decorator(f):
|
||||||
@wraps(f)
|
@wraps(f)
|
||||||
def decorated(*args, **kwargs):
|
def decorated(*args, **kwargs):
|
||||||
# 生产环境下权限由网关/后端统一管控,此处放行
|
|
||||||
return f(*args, **kwargs)
|
return f(*args, **kwargs)
|
||||||
return decorated
|
return decorated
|
||||||
return decorator
|
return decorator
|
||||||
|
|||||||
@@ -109,6 +109,11 @@ USE_RERANK = True
|
|||||||
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
||||||
RERANK_TOP_K = 15 # 重排序后保留数
|
RERANK_TOP_K = 15 # 重排序后保留数
|
||||||
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
RERANK_USE_ONNX = os.getenv("RERANK_USE_ONNX", "true").lower() == "true"
|
||||||
|
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "local") # local / cloud / fallback
|
||||||
|
RERANK_CLOUD_MODEL = os.getenv("RERANK_CLOUD_MODEL", "qwen3-rerank")
|
||||||
|
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", "")
|
||||||
|
RERANK_CLOUD_BASE_URL = os.getenv("RERANK_CLOUD_BASE_URL", "https://dashscope.aliyuncs.com/compatible-api/v1/reranks")
|
||||||
|
RERANK_CLOUD_TIMEOUT = int(os.getenv("RERANK_CLOUD_TIMEOUT", "15"))
|
||||||
RERANK_CONTEXT_MIN_SCORE = 0.05 # Rerank 分数低于此值的切片不送入 LLM
|
RERANK_CONTEXT_MIN_SCORE = 0.05 # Rerank 分数低于此值的切片不送入 LLM
|
||||||
|
|
||||||
# ----- RRF 融合 -----
|
# ----- RRF 融合 -----
|
||||||
@@ -181,10 +186,7 @@ SEMANTIC_CACHE_ENABLED = True
|
|||||||
SEMANTIC_CACHE_THRESHOLD = 0.92 # 相似度阈值
|
SEMANTIC_CACHE_THRESHOLD = 0.92 # 相似度阈值
|
||||||
|
|
||||||
# 缓存写入最低置信度
|
# 缓存写入最低置信度
|
||||||
# 注意:ChromaDB cosine distance 范围 [0,2],score = 1 - dist
|
CACHE_MIN_SCORE = 0.3
|
||||||
# 当前 embedding 模型的 cosine similarity 普遍在 0.03-0.06 之间
|
|
||||||
# 搜索管线已通过 rerank 过滤低质量结果,此处不再额外限制
|
|
||||||
CACHE_MIN_SCORE = 0.0
|
|
||||||
|
|
||||||
# LLM 调用预算
|
# LLM 调用预算
|
||||||
MAX_LLM_CALLS_PER_QUERY = 2
|
MAX_LLM_CALLS_PER_QUERY = 2
|
||||||
@@ -201,18 +203,6 @@ MINERU_DEVICE_MODE = os.getenv("MINERU_DEVICE_MODE", "cpu") # cpu / cuda
|
|||||||
MINERU_API_TOKEN = os.getenv("MINERU_API_TOKEN", "") # 在 https://mineru.net/apiManage/token 申请
|
MINERU_API_TOKEN = os.getenv("MINERU_API_TOKEN", "") # 在 https://mineru.net/apiManage/token 申请
|
||||||
MINERU_API_URL = os.getenv("MINERU_API_URL", "https://mineru.net/api/v4/extract/task")
|
MINERU_API_URL = os.getenv("MINERU_API_URL", "https://mineru.net/api/v4/extract/task")
|
||||||
MINERU_PREFER_ONLINE = os.getenv("MINERU_PREFER_ONLINE", "true").lower() == "true"
|
MINERU_PREFER_ONLINE = os.getenv("MINERU_PREFER_ONLINE", "true").lower() == "true"
|
||||||
MINERU_PREFER_V2 = os.getenv("MINERU_PREFER_V2", "true").lower() == "true" # 优先使用 v2 格式(含 style 信息)
|
|
||||||
|
|
||||||
# 标题识别规则引擎
|
|
||||||
# 规则定义见 parsers/heading_rules.py,支持通过 config.py 覆盖
|
|
||||||
HEADING_RULES_CONFIG = None # None=使用内置默认规则; dict 列表=自定义规则覆盖
|
|
||||||
HEADING_SHORT_TEXT_ENABLED = True # 是否启用短中文文本标题识别(最易误判的规则,可单独关闭)
|
|
||||||
HEADING_SHORT_TEXT_MAX_LENGTH = 20 # 短文本最大字符数阈值
|
|
||||||
|
|
||||||
# 表单类型二次校正
|
|
||||||
# MinerU 解析 Word 文档时,某些表单被标记为 text,根据内容特征修正为 table
|
|
||||||
FORM_RECLASSIFY_ENABLED = True # 是否启用 text->table 表单检测
|
|
||||||
FORM_RECLASSIFY_MIN_INDICATORS = 2 # 最少命中几个表单特征指标才校正
|
|
||||||
|
|
||||||
# 分块参数
|
# 分块参数
|
||||||
CHUNK_SIZE = 1000
|
CHUNK_SIZE = 1000
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ RAG 核心引擎模块
|
|||||||
|
|
||||||
包含:
|
包含:
|
||||||
- engine: RAGEngine 单例类,管理模型和共享资源
|
- engine: RAGEngine 单例类,管理模型和共享资源
|
||||||
|
- agentic: AgenticRAG 智能问答
|
||||||
- bm25_index: BM25 关键词检索索引
|
- bm25_index: BM25 关键词检索索引
|
||||||
- chunker: 文本分块器
|
- chunker: 文本分块器
|
||||||
"""
|
"""
|
||||||
|
|||||||
390
core/agentic.py
Normal file
390
core/agentic.py
Normal file
@@ -0,0 +1,390 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 知识库智能问答系统
|
||||||
|
|
||||||
|
核心能力:
|
||||||
|
1. 知识库检索 - 向量检索 + BM25 + Rerank
|
||||||
|
2. 网络搜索 - 当知识库不足时自动搜索(需配置SERPER_API_KEY)
|
||||||
|
3. 图谱检索 - 实体关系推理(需配置Neo4j)
|
||||||
|
4. 多源融合 - 智能处理知识库和网络内容
|
||||||
|
5. Agent决策 - 动态决定检索、改写、分解等操作
|
||||||
|
|
||||||
|
使用方式:
|
||||||
|
from core.agentic import AgenticRAG
|
||||||
|
|
||||||
|
rag = AgenticRAG()
|
||||||
|
result = rag.process("你的问题")
|
||||||
|
print(result["answer"])
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from openai import OpenAI
|
||||||
|
|
||||||
|
# 配置日志
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 导入基础模块
|
||||||
|
from core.engine import get_engine
|
||||||
|
from core.llm_utils import call_llm, quick_yes_no, parse_json_from_response
|
||||||
|
|
||||||
|
# 导入基础常量和配置
|
||||||
|
from .agentic_base import (
|
||||||
|
API_KEY, BASE_URL, MODEL,
|
||||||
|
HAS_SERPER,
|
||||||
|
HAS_BUDGET, SEMANTIC_CACHE_ENABLED,
|
||||||
|
MAX_CONTEXT_TOKENS, MAX_CONTEXT_COUNT, RERANK_THRESHOLD,
|
||||||
|
SOURCE_KB, SOURCE_WEB,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 尝试导入语义缓存
|
||||||
|
try:
|
||||||
|
from core.semantic_cache import SemanticCache
|
||||||
|
except ImportError:
|
||||||
|
SemanticCache = None
|
||||||
|
|
||||||
|
# 导入 Mixin 类
|
||||||
|
from .agentic_query import QueryRewriteMixin
|
||||||
|
from .agentic_search import SearchMixin
|
||||||
|
from .agentic_answer import AnswerMixin
|
||||||
|
from .agentic_citation import CitationMixin
|
||||||
|
from .agentic_media import RichMediaMixin
|
||||||
|
from .agentic_quality import QualityMixin
|
||||||
|
from .agentic_context import ContextMixin
|
||||||
|
from .agentic_meta import MetaQuestionMixin
|
||||||
|
|
||||||
|
|
||||||
|
class AgenticRAG(
|
||||||
|
QueryRewriteMixin,
|
||||||
|
SearchMixin,
|
||||||
|
AnswerMixin,
|
||||||
|
CitationMixin,
|
||||||
|
RichMediaMixin,
|
||||||
|
QualityMixin,
|
||||||
|
ContextMixin,
|
||||||
|
MetaQuestionMixin
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Agentic RAG - 知识库智能问答
|
||||||
|
|
||||||
|
通过 Mixin 模式组合功能:
|
||||||
|
- QueryRewriteMixin: 查询重写
|
||||||
|
- SearchMixin: 检索功能
|
||||||
|
- AnswerMixin: 答案生成
|
||||||
|
- CitationMixin: 引用处理
|
||||||
|
- RichMediaMixin: 富媒体处理
|
||||||
|
- QualityMixin: 质量评估
|
||||||
|
- ContextMixin: 上下文处理
|
||||||
|
- MetaQuestionMixin: 元问题处理
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
max_iterations: int = 3,
|
||||||
|
enable_web_search: bool = True,
|
||||||
|
**kwargs
|
||||||
|
):
|
||||||
|
"""初始化"""
|
||||||
|
self.max_iterations = max_iterations
|
||||||
|
self.enable_web_search = enable_web_search and HAS_SERPER
|
||||||
|
self.client = OpenAI(api_key=API_KEY, base_url=BASE_URL)
|
||||||
|
|
||||||
|
# 信息来源标记
|
||||||
|
self.SOURCE_KB = SOURCE_KB
|
||||||
|
self.SOURCE_WEB = SOURCE_WEB
|
||||||
|
|
||||||
|
# 初始化置信度门控
|
||||||
|
try:
|
||||||
|
from core.confidence_gate import create_gate
|
||||||
|
self.confidence_gate = create_gate()
|
||||||
|
except ImportError:
|
||||||
|
self.confidence_gate = None
|
||||||
|
|
||||||
|
# 初始化多维质量评估器
|
||||||
|
try:
|
||||||
|
from core.quality_assessor import create_assessor
|
||||||
|
self.quality_assessor = create_assessor()
|
||||||
|
except ImportError:
|
||||||
|
self.quality_assessor = None
|
||||||
|
|
||||||
|
# 初始化推理反思器
|
||||||
|
try:
|
||||||
|
from core.reasoning_reflector import create_reflector
|
||||||
|
self.reasoning_reflector = create_reflector()
|
||||||
|
except ImportError:
|
||||||
|
self.reasoning_reflector = None
|
||||||
|
|
||||||
|
# 初始化循环防护器
|
||||||
|
try:
|
||||||
|
from core.loop_guard import create_guard
|
||||||
|
self.loop_guard = create_guard(max_iterations=max_iterations)
|
||||||
|
except ImportError:
|
||||||
|
self.loop_guard = None
|
||||||
|
|
||||||
|
# 初始化语义缓存
|
||||||
|
self.semantic_cache = None
|
||||||
|
self.embedding_model = None
|
||||||
|
if SEMANTIC_CACHE_ENABLED and SemanticCache:
|
||||||
|
try:
|
||||||
|
engine = get_engine()
|
||||||
|
if engine and hasattr(engine, 'embedding_model'):
|
||||||
|
self.embedding_model = engine.embedding_model
|
||||||
|
emb_dim = 768
|
||||||
|
# 优先使用新 API,兼容旧版本
|
||||||
|
if hasattr(self.embedding_model, 'get_embedding_dimension'):
|
||||||
|
emb_dim = self.embedding_model.get_embedding_dimension()
|
||||||
|
elif hasattr(self.embedding_model, 'get_sentence_embedding_dimension'):
|
||||||
|
emb_dim = self.embedding_model.get_sentence_embedding_dimension()
|
||||||
|
self.semantic_cache = SemanticCache(
|
||||||
|
dim=emb_dim,
|
||||||
|
threshold=0.92,
|
||||||
|
max_size=5000
|
||||||
|
)
|
||||||
|
logger.info(f"语义缓存已启用,维度={emb_dim}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"语义缓存初始化失败: {e}")
|
||||||
|
self.semantic_cache = None
|
||||||
|
|
||||||
|
# Context Compression 配置
|
||||||
|
self.MAX_CONTEXT_TOKENS = MAX_CONTEXT_TOKENS
|
||||||
|
self.MAX_CONTEXT_COUNT = MAX_CONTEXT_COUNT
|
||||||
|
self.RERANK_THRESHOLD = RERANK_THRESHOLD
|
||||||
|
|
||||||
|
# Answer Grounding 配置
|
||||||
|
self.MAX_GROUNDING_RETRY = 1
|
||||||
|
self.grounding_retry_count = 0
|
||||||
|
|
||||||
|
def should_rewrite(self, query: str, history: list = None) -> bool:
|
||||||
|
"""判断是否需要重写查询"""
|
||||||
|
# 口语化表达模式
|
||||||
|
colloquial_patterns = [
|
||||||
|
"这个", "那个", "它", "这", "那",
|
||||||
|
"上面", "下面", "刚才", "之前",
|
||||||
|
"能不能", "可以吗", "行不行",
|
||||||
|
"怎么办", "怎么弄", "咋整"
|
||||||
|
]
|
||||||
|
|
||||||
|
for pattern in colloquial_patterns:
|
||||||
|
if pattern in query:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# 查询太短
|
||||||
|
if len(query) < 5:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# 有对话历史时,可能需要实体补全
|
||||||
|
if history:
|
||||||
|
for msg in reversed(history[-3:]):
|
||||||
|
if msg.get("role") == "user":
|
||||||
|
prev_query = msg.get("content", "")
|
||||||
|
# 如果当前查询缺少主语,可能需要补全
|
||||||
|
if any(kw in query for kw in ["标准", "规定", "流程", "制度"]):
|
||||||
|
if not any(kw in query for kw in ["报销", "出差", "请假", "工资", "合同"]):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def process(self, query: str, verbose: bool = True, history: list = None,
|
||||||
|
allowed_levels: list = None, role: str = None, department: str = None,
|
||||||
|
emit_log=None) -> dict:
|
||||||
|
"""
|
||||||
|
主流程:智能问答
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: 用户问题
|
||||||
|
verbose: 是否打印详细过程
|
||||||
|
history: 对话历史
|
||||||
|
allowed_levels: 允许访问的安全级别
|
||||||
|
role: 用户角色
|
||||||
|
department: 用户部门
|
||||||
|
emit_log: 日志发射函数(流式输出)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"answer": str,
|
||||||
|
"sources": list,
|
||||||
|
"images": list,
|
||||||
|
"tables": list,
|
||||||
|
"citations": list,
|
||||||
|
"log_trace": list
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
log_trace = []
|
||||||
|
|
||||||
|
# 1. 检查元问题
|
||||||
|
if self._is_meta_question(query):
|
||||||
|
answer = self._answer_meta_question(query, allowed_levels, role, department)
|
||||||
|
return {
|
||||||
|
"answer": answer,
|
||||||
|
"sources": [],
|
||||||
|
"images": [],
|
||||||
|
"tables": [],
|
||||||
|
"citations": [],
|
||||||
|
"log_trace": [{"phase": "meta_question", "query": query}]
|
||||||
|
}
|
||||||
|
|
||||||
|
# 2. 查询重写
|
||||||
|
current_query = query
|
||||||
|
if self.should_rewrite(query, history):
|
||||||
|
current_query = self._rewrite_query(query, history)
|
||||||
|
log_trace.append({"phase": "rewrite", "original": query, "rewritten": current_query})
|
||||||
|
if emit_log:
|
||||||
|
emit_log(f"📝 查询重写: {query} → {current_query}")
|
||||||
|
|
||||||
|
# 3. 知识库检索
|
||||||
|
contexts = []
|
||||||
|
try:
|
||||||
|
engine = get_engine()
|
||||||
|
if not engine._initialized:
|
||||||
|
engine.initialize()
|
||||||
|
|
||||||
|
# 获取用户可访问的向量库
|
||||||
|
from knowledge.manager import get_kb_manager
|
||||||
|
kb_mgr = get_kb_manager()
|
||||||
|
accessible = kb_mgr.get_accessible_collections(role or 'user', department or '', 'read')
|
||||||
|
|
||||||
|
# 统一使用 search_knowledge() — 生产路径的同一 API
|
||||||
|
# search_knowledge() 返回 dict: {ids, documents, metadatas, distances},每项为 list[list]
|
||||||
|
# top_k 与生产路径对齐(30),确保 Rerank 后仍有足够结果
|
||||||
|
results = engine.search_knowledge(
|
||||||
|
query=current_query,
|
||||||
|
top_k=30,
|
||||||
|
collections=accessible if accessible else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
docs = results.get('documents', [[]])[0]
|
||||||
|
metas = results.get('metadatas', [[]])[0]
|
||||||
|
dists = results.get('distances', [[]])[0]
|
||||||
|
|
||||||
|
for doc, meta, score in zip(docs, metas, dists):
|
||||||
|
contexts.append({
|
||||||
|
'doc': doc,
|
||||||
|
'meta': meta,
|
||||||
|
'score': 1 - score if score <= 1 else 1 / (1 + score), # 距离→相似度
|
||||||
|
'source_type': self.SOURCE_KB,
|
||||||
|
'query': current_query
|
||||||
|
})
|
||||||
|
|
||||||
|
log_trace.append({"phase": "kb_search", "query": current_query, "results": len(contexts)})
|
||||||
|
if emit_log:
|
||||||
|
emit_log(f"🔍 知识库检索: {len(contexts)} 条结果")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"知识库检索失败: {e}")
|
||||||
|
log_trace.append({"phase": "kb_search", "error": str(e)})
|
||||||
|
|
||||||
|
# 3.5 图片独立检索 + 打分选择(与生产路径对齐)
|
||||||
|
selected_images = []
|
||||||
|
try:
|
||||||
|
from api.chat_routes import select_images
|
||||||
|
selected_images = select_images(contexts, current_query)
|
||||||
|
if selected_images and emit_log:
|
||||||
|
emit_log(f"🖼️ 图片选择: {len(selected_images)} 张相关图片")
|
||||||
|
log_trace.append({"phase": "image_selection", "count": len(selected_images)})
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"图片选择失败: {e}")
|
||||||
|
|
||||||
|
# 4. 上下文压缩
|
||||||
|
contexts = self._compress_contexts(current_query, contexts)
|
||||||
|
|
||||||
|
# 5. 网络搜索(如果需要)
|
||||||
|
web_contexts = []
|
||||||
|
if self.enable_web_search and (
|
||||||
|
not contexts or
|
||||||
|
not self._is_kb_result_sufficient(current_query, [c['doc'] for c in contexts]) or
|
||||||
|
self._should_web_search(current_query)
|
||||||
|
):
|
||||||
|
web_contexts = self._web_search_flow(current_query, log_trace, emit_log, verbose, allowed_levels)
|
||||||
|
contexts.extend(web_contexts)
|
||||||
|
|
||||||
|
# 7. 生成答案(注入图片描述到上下文)
|
||||||
|
if contexts:
|
||||||
|
# 将选中图片的描述注入上下文,让 LLM 能"看到"图片内容
|
||||||
|
if selected_images:
|
||||||
|
image_contexts = []
|
||||||
|
for i, img in enumerate(selected_images, 1):
|
||||||
|
full_desc = img.get('full_description', '') or img.get('description', '')
|
||||||
|
if full_desc:
|
||||||
|
img_source = img.get('source', '')
|
||||||
|
img_page = img.get('page', '')
|
||||||
|
source_info = f"(来源:{img_source} 第{img_page}页)" if img_source and img_page else ""
|
||||||
|
image_contexts.append({
|
||||||
|
'doc': f"【图片{i}】{full_desc}{source_info}",
|
||||||
|
'meta': {'source': img_source, 'page': img_page, 'chunk_type': img.get('type', 'image')},
|
||||||
|
'score': img.get('score', 0),
|
||||||
|
'source_type': self.SOURCE_KB,
|
||||||
|
'query': current_query
|
||||||
|
})
|
||||||
|
# 图片上下文追加到知识库上下文前面(让 LLM 优先看到图片信息)
|
||||||
|
contexts = image_contexts + contexts
|
||||||
|
|
||||||
|
answer = self._generate_fused_answer(current_query, contexts, allowed_levels)
|
||||||
|
|
||||||
|
# 答案验证(防止幻觉)
|
||||||
|
if self.grounding_retry_count < self.MAX_GROUNDING_RETRY:
|
||||||
|
answer = self._verify_and_refine_answer(current_query, answer, contexts)
|
||||||
|
else:
|
||||||
|
answer = self._generate_no_context_answer(current_query, allowed_levels)
|
||||||
|
|
||||||
|
# 8. 构建引用
|
||||||
|
citations = self._attach_citations(answer, contexts)
|
||||||
|
|
||||||
|
# 9. 图片结果:优先使用 select_images 的结构化结果(含 URL + 打分)
|
||||||
|
# 回退到 _extract_rich_media(从 metadata 提取)
|
||||||
|
if selected_images:
|
||||||
|
images_result = selected_images
|
||||||
|
else:
|
||||||
|
rich_media = self._extract_rich_media(contexts)
|
||||||
|
images_result = rich_media.get("images", [])
|
||||||
|
|
||||||
|
return {
|
||||||
|
"answer": answer,
|
||||||
|
"sources": citations.get("sources", []),
|
||||||
|
"images": images_result,
|
||||||
|
"tables": citations.get("tables", []) if isinstance(citations, dict) else [],
|
||||||
|
"citations": citations.get("citations", []),
|
||||||
|
"log_trace": log_trace
|
||||||
|
}
|
||||||
|
|
||||||
|
def chat_search(self, query: str, history: list = None, role: str = None,
|
||||||
|
department: str = None, allowed_levels: list = None) -> dict:
|
||||||
|
"""聊天式检索接口"""
|
||||||
|
return self.process(
|
||||||
|
query,
|
||||||
|
verbose=False,
|
||||||
|
history=history,
|
||||||
|
role=role,
|
||||||
|
department=department,
|
||||||
|
allowed_levels=allowed_levels
|
||||||
|
)
|
||||||
|
|
||||||
|
def chat(self):
|
||||||
|
"""命令行交互模式"""
|
||||||
|
print("🤖 Agentic RAG 已启动,输入 'quit' 退出")
|
||||||
|
print("-" * 50)
|
||||||
|
|
||||||
|
history = []
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
query = input("\n👤 你: ").strip()
|
||||||
|
if not query:
|
||||||
|
continue
|
||||||
|
if query.lower() in ['quit', 'exit', 'q']:
|
||||||
|
print("👋 再见!")
|
||||||
|
break
|
||||||
|
|
||||||
|
result = self.process(query, verbose=True, history=history)
|
||||||
|
print(f"\n🤖 AI: {result['answer']}")
|
||||||
|
|
||||||
|
if result['sources']:
|
||||||
|
print("\n📚 来源:")
|
||||||
|
for src in result['sources'][:3]:
|
||||||
|
print(f" - {src['source']}")
|
||||||
|
|
||||||
|
history.append({"role": "user", "content": query})
|
||||||
|
history.append({"role": "assistant", "content": result['answer']})
|
||||||
|
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print("\n👋 再见!")
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ 错误: {e}")
|
||||||
214
core/agentic_answer.py
Normal file
214
core/agentic_answer.py
Normal file
@@ -0,0 +1,214 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 答案生成 Mixin
|
||||||
|
|
||||||
|
包含答案生成、上下文构建、融合回答等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger, MODEL, SOURCE_KB, SOURCE_WEB
|
||||||
|
from core.llm_utils import call_llm
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class AnswerMixin:
|
||||||
|
"""答案生成方法"""
|
||||||
|
|
||||||
|
def _generate_fused_answer(self, query: str, contexts: list, allowed_levels: list = None) -> str:
|
||||||
|
"""生成融合答案 - 智能处理多源信息"""
|
||||||
|
# 分离不同来源
|
||||||
|
kb_contexts = [c for c in contexts if c.get('source_type') == self.SOURCE_KB]
|
||||||
|
web_contexts = [c for c in contexts if c.get('source_type') == self.SOURCE_WEB]
|
||||||
|
|
||||||
|
# 如果没有任何上下文,检测是否因权限限制
|
||||||
|
if not contexts:
|
||||||
|
return self._generate_no_context_answer(query, allowed_levels)
|
||||||
|
|
||||||
|
# 正常生成答案
|
||||||
|
context_str = self._build_context_string(kb_contexts, web_contexts)
|
||||||
|
prompt = self._build_normal_answer_prompt(query, context_str, kb_contexts, web_contexts)
|
||||||
|
|
||||||
|
result = call_llm(
|
||||||
|
self.client, prompt, MODEL,
|
||||||
|
temperature=0.7,
|
||||||
|
max_tokens=2000
|
||||||
|
)
|
||||||
|
return result or f"生成答案失败"
|
||||||
|
|
||||||
|
def _build_context_string(self, kb_contexts, web_contexts):
|
||||||
|
"""构建上下文字符串 - FAQ 优先策略,按分数排序"""
|
||||||
|
# 分离 FAQ 和普通知识库内容
|
||||||
|
faq_contexts = [c for c in kb_contexts if c.get('meta', {}).get('chunk_type') == 'faq']
|
||||||
|
regular_contexts = [c for c in kb_contexts if c.get('meta', {}).get('chunk_type') != 'faq']
|
||||||
|
|
||||||
|
# 按分数降序排列,确保最相关的内容优先展示
|
||||||
|
regular_contexts.sort(key=lambda c: c.get('score', 0), reverse=True)
|
||||||
|
|
||||||
|
# FAQ 部分(优先展示)
|
||||||
|
faq_parts = []
|
||||||
|
for i, c in enumerate(faq_contexts[:3], 1):
|
||||||
|
meta = c['meta']
|
||||||
|
answer = meta.get('faq_answer', c['doc'])
|
||||||
|
faq_parts.append(f"[FAQ-{i}] 常见问题\n问题:{c['doc']}\n标准答案:{answer}")
|
||||||
|
|
||||||
|
# 普通知识库部分(用 12 条,提升覆盖率)
|
||||||
|
kb_parts = []
|
||||||
|
for i, c in enumerate(regular_contexts[:12], 1):
|
||||||
|
meta = c['meta']
|
||||||
|
source_str = meta.get('source', '未知')
|
||||||
|
section = meta.get('section', '')
|
||||||
|
source_info = f"{source_str}"
|
||||||
|
if section:
|
||||||
|
source_info += f" > {section[:60]}"
|
||||||
|
kb_parts.append(f"[知识库-{i}] {source_info}\n{c['doc']}")
|
||||||
|
|
||||||
|
web_parts = []
|
||||||
|
for i, c in enumerate(web_contexts[:5], 1):
|
||||||
|
meta = c['meta']
|
||||||
|
web_parts.append(f"[网络-{i}] {meta.get('title', '')}\n来源:{meta.get('source', '')}\n{c['doc']}")
|
||||||
|
|
||||||
|
return "\n\n".join(faq_parts + kb_parts + web_parts)
|
||||||
|
|
||||||
|
def _build_normal_answer_prompt(self, query, context_str, kb_contexts, web_contexts):
|
||||||
|
"""构建正常回答的提示词(与生产路径 generate_answer_stream 对齐)"""
|
||||||
|
# 检测是否有图片上下文
|
||||||
|
has_images = any(c.get('meta', {}).get('chunk_type') in ('image', 'chart', 'table')
|
||||||
|
for c in kb_contexts)
|
||||||
|
|
||||||
|
image_instruction = ""
|
||||||
|
if has_images:
|
||||||
|
image_instruction = "\n5. 如果参考资料中包含【图片N】信息,请在回答中简要介绍每张图片的内容和用途"
|
||||||
|
|
||||||
|
return f"""你是一个严谨的知识库问答助手。你必须且只能根据用户提供的【参考资料】回答问题。
|
||||||
|
|
||||||
|
【参考资料】
|
||||||
|
{context_str}
|
||||||
|
|
||||||
|
【用户问题】
|
||||||
|
{query}
|
||||||
|
|
||||||
|
【回答要求】
|
||||||
|
1. 如果参考资料中有答案,必须引用对应内容回答,并在回答末尾标注引用编号(如[1]、[2])
|
||||||
|
2. 如果参考资料中确实没有相关信息,简短说明"参考资料中没有相关信息"即可,不要编造或补充资料外的内容
|
||||||
|
3. 禁止使用参考资料以外的知识进行补充或推测
|
||||||
|
4. 分点列举,条理清晰,语言简洁{image_instruction}
|
||||||
|
|
||||||
|
请仔细阅读以上全部参考资料后回答:"""
|
||||||
|
|
||||||
|
def _build_answer_prompt_with_permission(self, query, context_str, levels_str, sources_str, kb_contexts, web_contexts):
|
||||||
|
"""构建带权限提示的回答提示词"""
|
||||||
|
return f"""你是一个严谨的智能助手。
|
||||||
|
|
||||||
|
【用户问题】
|
||||||
|
{query}
|
||||||
|
|
||||||
|
【重要提示】
|
||||||
|
检测到与用户问题更相关的信息可能存在于「{levels_str}」级别的文档中,但用户当前的权限级别无法访问。
|
||||||
|
|
||||||
|
【可访问的信息来源】
|
||||||
|
{context_str}
|
||||||
|
|
||||||
|
【回答要求】
|
||||||
|
1. 首先明确告知用户:当前回答基于您有权限访问的文档,可能不完整
|
||||||
|
2. 基于现有信息如实回答
|
||||||
|
3. 建议用户如需完整信息,请联系管理员申请相应权限
|
||||||
|
|
||||||
|
请回答:"""
|
||||||
|
|
||||||
|
def _generate_no_context_answer(self, query: str, allowed_levels: list = None) -> str:
|
||||||
|
"""无上下文时的回答 — 诚实告知,不编造"""
|
||||||
|
return "参考资料中没有找到与该问题相关的信息,无法根据现有知识库内容回答您的问题。"
|
||||||
|
|
||||||
|
def _verify_and_refine_answer(self, query: str, answer: str, contexts: list) -> str:
|
||||||
|
"""验证并精炼答案 - 防止幻觉
|
||||||
|
|
||||||
|
返回值始终是干净的答案文本,不包含验证推理过程。
|
||||||
|
"""
|
||||||
|
prompt = f"""请检查以下回答是否存在"幻觉"(与参考信息不符的内容)。
|
||||||
|
|
||||||
|
【用户问题】
|
||||||
|
{query}
|
||||||
|
|
||||||
|
【参考信息】
|
||||||
|
{chr(10).join([f"[{i+1}] {c['doc'][:200]}" for i, c in enumerate(contexts[:8])])}
|
||||||
|
|
||||||
|
【AI回答】
|
||||||
|
{answer}
|
||||||
|
|
||||||
|
【检查规则】
|
||||||
|
1. 逐条核对回答中的事实是否能在参考信息中找到依据
|
||||||
|
2. 如果没有幻觉,只回复一个英文单词:PASS
|
||||||
|
3. 如果有幻觉,只输出修正后的完整回答(不要输出检查过程、不要加标题、不要加"检查结果"等前缀)
|
||||||
|
|
||||||
|
修正后的回答:"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = call_llm(
|
||||||
|
self.client, prompt, MODEL,
|
||||||
|
temperature=0.1,
|
||||||
|
max_tokens=2000
|
||||||
|
)
|
||||||
|
if not result:
|
||||||
|
return answer
|
||||||
|
# 如果返回 PASS 或很短的确认,说明无幻觉
|
||||||
|
cleaned = result.strip()
|
||||||
|
if cleaned.upper() == "PASS" or len(cleaned) < 10:
|
||||||
|
return answer
|
||||||
|
# 有幻觉时,返回修正后的干净答案(去掉可能的前缀)
|
||||||
|
for prefix in ["修正后的回答:", "修正后回答:", "修正回答:", "修正后:"]:
|
||||||
|
if cleaned.startswith(prefix):
|
||||||
|
cleaned = cleaned[len(prefix):].strip()
|
||||||
|
return cleaned
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"答案验证失败: {e}")
|
||||||
|
return answer
|
||||||
|
|
||||||
|
def _generate_uncertain_answer(self, query: str, contexts: list) -> str:
|
||||||
|
"""生成不确定性回答"""
|
||||||
|
context_str = "\n".join([c['doc'][:200] for c in contexts[:3]])
|
||||||
|
|
||||||
|
prompt = f"""用户问题:{query}
|
||||||
|
|
||||||
|
找到的信息可能不够完整或相关性不高:
|
||||||
|
{context_str}
|
||||||
|
|
||||||
|
请基于这些信息给出一个谨慎的回答,明确说明哪些部分是有依据的,哪些部分可能需要更多验证。
|
||||||
|
|
||||||
|
回答:"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = call_llm(
|
||||||
|
self.client, prompt, MODEL,
|
||||||
|
temperature=0.7,
|
||||||
|
max_tokens=1000
|
||||||
|
)
|
||||||
|
return result or "根据现有信息无法确定答案。"
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"生成不确定性回答失败: {e}")
|
||||||
|
return "根据现有信息无法确定答案。"
|
||||||
|
|
||||||
|
def _direct_answer(self, query: str, history: list = None) -> str:
|
||||||
|
"""直接使用 LLM 回答(无知识库检索)"""
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": "你是一个专业的助手,请用中文回答用户的问题。"}
|
||||||
|
]
|
||||||
|
|
||||||
|
if history:
|
||||||
|
for h in history[-4:]:
|
||||||
|
if h.get("role") in ["user", "assistant"]:
|
||||||
|
messages.append({"role": h["role"], "content": h.get("content", "")})
|
||||||
|
|
||||||
|
messages.append({"role": "user", "content": query})
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = call_llm(
|
||||||
|
self.client, "", MODEL,
|
||||||
|
temperature=0.7,
|
||||||
|
max_tokens=1500,
|
||||||
|
messages=messages
|
||||||
|
)
|
||||||
|
return result or "抱歉,我无法回答这个问题。"
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"直接回答失败: {e}")
|
||||||
|
return f"回答生成失败:{str(e)}"
|
||||||
62
core/agentic_base.py
Normal file
62
core/agentic_base.py
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 基础模块
|
||||||
|
|
||||||
|
包含常量、导入和共享配置
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# 配置日志
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 尝试导入搜索API配置
|
||||||
|
try:
|
||||||
|
from config import SERPER_API_KEY
|
||||||
|
HAS_SERPER = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_SERPER = False
|
||||||
|
SERPER_API_KEY = None
|
||||||
|
|
||||||
|
# LLM 预算控制
|
||||||
|
try:
|
||||||
|
from core.llm_budget import get_budget_controller, should_use_agent, CallType
|
||||||
|
HAS_BUDGET = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_BUDGET = False
|
||||||
|
CallType = None
|
||||||
|
|
||||||
|
# 语义缓存
|
||||||
|
try:
|
||||||
|
from config import SEMANTIC_CACHE_ENABLED, SEMANTIC_CACHE_THRESHOLD
|
||||||
|
HAS_SEMANTIC_CACHE_CONFIG = True
|
||||||
|
except ImportError:
|
||||||
|
SEMANTIC_CACHE_ENABLED = False
|
||||||
|
SEMANTIC_CACHE_THRESHOLD = 0.92
|
||||||
|
HAS_SEMANTIC_CACHE_CONFIG = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
from core.semantic_cache import SemanticCache, get_semantic_cache
|
||||||
|
HAS_SEMANTIC_CACHE = True
|
||||||
|
except ImportError:
|
||||||
|
HAS_SEMANTIC_CACHE = False
|
||||||
|
SemanticCache = None
|
||||||
|
|
||||||
|
# LLM 配置
|
||||||
|
try:
|
||||||
|
from config import API_KEY, BASE_URL, MODEL
|
||||||
|
except ImportError:
|
||||||
|
API_KEY = None
|
||||||
|
BASE_URL = None
|
||||||
|
MODEL = None
|
||||||
|
|
||||||
|
# 来源标记
|
||||||
|
SOURCE_KB = "知识库"
|
||||||
|
SOURCE_WEB = "网络搜索"
|
||||||
|
|
||||||
|
# Context Compression 配置
|
||||||
|
MAX_CONTEXT_TOKENS = 8000 # 最大上下文 token 数(与生产路径对齐)
|
||||||
|
MAX_CONTEXT_COUNT = 20 # 最大上下文数量
|
||||||
|
RERANK_THRESHOLD = 0.3 # Rerank 过滤阈值
|
||||||
|
|
||||||
|
# Answer Grounding 配置
|
||||||
|
MAX_GROUNDING_RETRY = 1 # 幻觉修正最多重试次数
|
||||||
237
core/agentic_citation.py
Normal file
237
core/agentic_citation.py
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 引用处理 Mixin
|
||||||
|
|
||||||
|
包含来源提取、引用构建、引用附加等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger, SOURCE_KB
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class CitationMixin:
|
||||||
|
"""引用处理方法"""
|
||||||
|
|
||||||
|
def _extract_sources(self, contexts: list) -> list:
|
||||||
|
"""提取来源列表,返回结构化定位信息"""
|
||||||
|
source_map = {}
|
||||||
|
|
||||||
|
for c in contexts:
|
||||||
|
meta = c.get('meta', {})
|
||||||
|
source_type = c.get('source_type', '未知')
|
||||||
|
|
||||||
|
if source_type == self.SOURCE_KB:
|
||||||
|
source_key = meta.get('source', '未知')
|
||||||
|
page = meta.get('page')
|
||||||
|
page_end = meta.get('page_end', page)
|
||||||
|
section = meta.get('section', '')
|
||||||
|
doc_type = meta.get('doc_type', 'other')
|
||||||
|
preview = meta.get('preview', '')
|
||||||
|
section_chunk_id = meta.get('section_chunk_id')
|
||||||
|
else:
|
||||||
|
source_key = meta.get('title', meta.get('source', '未知'))
|
||||||
|
page = None
|
||||||
|
page_end = None
|
||||||
|
section = ''
|
||||||
|
doc_type = 'other'
|
||||||
|
preview = ''
|
||||||
|
section_chunk_id = None
|
||||||
|
|
||||||
|
if source_key not in source_map:
|
||||||
|
source_map[source_key] = {
|
||||||
|
"source": source_key,
|
||||||
|
"type": source_type,
|
||||||
|
"doc_type": doc_type,
|
||||||
|
"count": 0,
|
||||||
|
"pages": [],
|
||||||
|
"sections": set(),
|
||||||
|
"previews": [],
|
||||||
|
"section_chunk_ids": set()
|
||||||
|
}
|
||||||
|
|
||||||
|
source_map[source_key]["count"] += 1
|
||||||
|
|
||||||
|
if page:
|
||||||
|
page_range = (page, page_end if page_end else page)
|
||||||
|
if page_range not in source_map[source_key]["pages"]:
|
||||||
|
source_map[source_key]["pages"].append(page_range)
|
||||||
|
|
||||||
|
if section:
|
||||||
|
source_map[source_key]["sections"].add(section)
|
||||||
|
|
||||||
|
if preview and len(source_map[source_key]["previews"]) < 3:
|
||||||
|
if preview not in source_map[source_key]["previews"]:
|
||||||
|
source_map[source_key]["previews"].append(preview)
|
||||||
|
|
||||||
|
if section_chunk_id:
|
||||||
|
source_map[source_key]["section_chunk_ids"].add(section_chunk_id)
|
||||||
|
|
||||||
|
sources = []
|
||||||
|
for key, info in source_map.items():
|
||||||
|
source_str = info["source"]
|
||||||
|
doc_type = info.get("doc_type", "other")
|
||||||
|
location_parts = []
|
||||||
|
|
||||||
|
if doc_type == 'pdf':
|
||||||
|
if info["pages"]:
|
||||||
|
valid_pages = [(s, e) for s, e in info["pages"] if s > 1 or e > 1]
|
||||||
|
if valid_pages or not info["sections"]:
|
||||||
|
page_strs = []
|
||||||
|
for start, end in sorted(info["pages"], key=lambda x: x[0]):
|
||||||
|
if start == end:
|
||||||
|
page_strs.append(f"第{start}页")
|
||||||
|
else:
|
||||||
|
page_strs.append(f"第{start}-{end}页")
|
||||||
|
location_parts.append(", ".join(page_strs))
|
||||||
|
|
||||||
|
if info["sections"]:
|
||||||
|
sections_list = sorted(info["sections"])[:3]
|
||||||
|
sections_str = "、".join(sections_list)
|
||||||
|
if len(info["sections"]) > 3:
|
||||||
|
sections_str += f"等{len(info['sections'])}个章节"
|
||||||
|
location_parts.append(sections_str)
|
||||||
|
|
||||||
|
elif doc_type == 'word':
|
||||||
|
if info["sections"]:
|
||||||
|
sections_list = sorted(info["sections"])[:3]
|
||||||
|
sections_str = "、".join(sections_list)
|
||||||
|
if len(info["sections"]) > 3:
|
||||||
|
sections_str += f"等{len(info['sections'])}个章节"
|
||||||
|
location_parts.append(sections_str)
|
||||||
|
|
||||||
|
if info.get("section_chunk_ids"):
|
||||||
|
chunk_ids = sorted(info["section_chunk_ids"])[:5]
|
||||||
|
if chunk_ids:
|
||||||
|
chunk_str = f"第{chunk_ids[0]}"
|
||||||
|
if len(chunk_ids) > 1:
|
||||||
|
chunk_str = f"第{chunk_ids[0]}-{chunk_ids[-1]}段"
|
||||||
|
location_parts.append(chunk_str)
|
||||||
|
|
||||||
|
elif doc_type == 'excel':
|
||||||
|
if info["sections"]:
|
||||||
|
sections_list = sorted(info["sections"])[:3]
|
||||||
|
sections_str = "、".join(sections_list)
|
||||||
|
location_parts.append(sections_str)
|
||||||
|
|
||||||
|
else:
|
||||||
|
if info["pages"]:
|
||||||
|
valid_pages = [(s, e) for s, e in info["pages"] if s > 1 or e > 1]
|
||||||
|
if valid_pages or not info["sections"]:
|
||||||
|
page_strs = []
|
||||||
|
for start, end in sorted(info["pages"], key=lambda x: x[0]):
|
||||||
|
if start == end:
|
||||||
|
page_strs.append(f"第{start}页")
|
||||||
|
else:
|
||||||
|
page_strs.append(f"第{start}-{end}页")
|
||||||
|
location_parts.append(", ".join(page_strs))
|
||||||
|
|
||||||
|
if info["sections"]:
|
||||||
|
sections_list = sorted(info["sections"])[:3]
|
||||||
|
sections_str = "、".join(sections_list)
|
||||||
|
if len(info["sections"]) > 3:
|
||||||
|
sections_str += f"等{len(info['sections'])}个章节"
|
||||||
|
location_parts.append(sections_str)
|
||||||
|
|
||||||
|
if location_parts:
|
||||||
|
source_str = f"{source_str} ({' | '.join(location_parts)})"
|
||||||
|
|
||||||
|
sources.append({
|
||||||
|
"source": source_str,
|
||||||
|
"type": info["type"],
|
||||||
|
"count": info["count"],
|
||||||
|
"doc_type": doc_type,
|
||||||
|
"previews": info.get("previews", []),
|
||||||
|
"section_chunk_ids": sorted(info.get("section_chunk_ids", []))[:5]
|
||||||
|
})
|
||||||
|
|
||||||
|
return sources
|
||||||
|
|
||||||
|
def _build_citation(self, meta: dict) -> dict:
|
||||||
|
"""根据文档类型构建定位信息"""
|
||||||
|
# 从 chunk_id 中提取全局切片序号(格式: "filename_N")
|
||||||
|
chunk_id_raw = meta.get('chunk_id', '')
|
||||||
|
chunk_index = None
|
||||||
|
if chunk_id_raw and '_' in str(chunk_id_raw):
|
||||||
|
try:
|
||||||
|
chunk_index = int(str(chunk_id_raw).rsplit('_', 1)[-1])
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
chunk_index = meta.get('chunk_index')
|
||||||
|
else:
|
||||||
|
chunk_index = meta.get('chunk_index')
|
||||||
|
|
||||||
|
citation = {
|
||||||
|
"chunk_id": chunk_id_raw,
|
||||||
|
"chunk_index": chunk_index, # 全局切片序号,用于前端文档预览跳转
|
||||||
|
"source": meta.get('source', ''),
|
||||||
|
"collection": meta.get('_collection', ''), # 所属向量库,用于前端文档预览跳转
|
||||||
|
"doc_type": meta.get('doc_type', 'other'),
|
||||||
|
"section": meta.get('section', ''),
|
||||||
|
"preview": meta.get('preview', ''),
|
||||||
|
"content": meta.get('preview', ''), # 初始用 preview,_attach_citations 中会用完整内容覆盖
|
||||||
|
"chunk_type": meta.get('chunk_type', 'text'),
|
||||||
|
}
|
||||||
|
|
||||||
|
doc_type = meta.get('doc_type', 'other')
|
||||||
|
|
||||||
|
if doc_type == 'pdf':
|
||||||
|
bbox_raw = meta.get('bbox')
|
||||||
|
bbox = None
|
||||||
|
if bbox_raw:
|
||||||
|
try:
|
||||||
|
bbox = json.loads(bbox_raw) if isinstance(bbox_raw, str) else bbox_raw
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
bbox = bbox_raw
|
||||||
|
|
||||||
|
citation.update({
|
||||||
|
"page": meta.get('page'),
|
||||||
|
"page_end": meta.get('page_end'),
|
||||||
|
"bbox": bbox,
|
||||||
|
"bbox_mode": meta.get('bbox_mode'),
|
||||||
|
})
|
||||||
|
elif doc_type == 'word':
|
||||||
|
citation.update({
|
||||||
|
"section_chunk_id": meta.get('section_chunk_id'),
|
||||||
|
})
|
||||||
|
elif doc_type == 'excel':
|
||||||
|
citation.update({
|
||||||
|
"page": meta.get('page'),
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
bbox_raw = meta.get('bbox')
|
||||||
|
bbox = None
|
||||||
|
if bbox_raw:
|
||||||
|
try:
|
||||||
|
bbox = json.loads(bbox_raw) if isinstance(bbox_raw, str) else bbox_raw
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
bbox = bbox_raw
|
||||||
|
|
||||||
|
citation.update({
|
||||||
|
"page": meta.get('page'),
|
||||||
|
"page_end": meta.get('page_end'),
|
||||||
|
"bbox": bbox,
|
||||||
|
"bbox_mode": meta.get('bbox_mode'),
|
||||||
|
})
|
||||||
|
|
||||||
|
return citation
|
||||||
|
|
||||||
|
def _attach_citations(self, answer: str, contexts: list) -> dict:
|
||||||
|
"""将引用信息附加到答案"""
|
||||||
|
citations = []
|
||||||
|
|
||||||
|
for c in contexts:
|
||||||
|
meta = c.get('meta', {})
|
||||||
|
full_content = c.get('doc', '')
|
||||||
|
citation = self._build_citation(meta)
|
||||||
|
# 用上下文中的完整文档内容覆盖 content 字段
|
||||||
|
if full_content:
|
||||||
|
citation['content'] = full_content[:300]
|
||||||
|
citations.append(citation)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"answer": answer,
|
||||||
|
"citations": citations,
|
||||||
|
"sources": self._extract_sources(contexts)
|
||||||
|
}
|
||||||
111
core/agentic_context.py
Normal file
111
core/agentic_context.py
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 上下文处理 Mixin
|
||||||
|
|
||||||
|
包含上下文压缩、去重、Token 控制等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger, MAX_CONTEXT_TOKENS, RERANK_THRESHOLD
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ContextMixin:
|
||||||
|
"""上下文处理方法"""
|
||||||
|
|
||||||
|
def _compress_contexts(self, query: str, contexts: list) -> list:
|
||||||
|
"""上下文压缩三步走:Rerank 过滤 → 去重 → Token 控制"""
|
||||||
|
if not contexts:
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
# Step 1: Rerank 过滤
|
||||||
|
filtered = self._rerank_filter(contexts)
|
||||||
|
|
||||||
|
# Step 2: 去重
|
||||||
|
deduped = self._deduplicate_contexts(filtered)
|
||||||
|
|
||||||
|
# Step 3: Token 控制
|
||||||
|
result = self._truncate_to_tokens(deduped, self.MAX_CONTEXT_TOKENS)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _rerank_filter(self, contexts: list) -> list:
|
||||||
|
"""Rerank 过滤 - 保留相关性分数 >= 阈值的上下文"""
|
||||||
|
scored_contexts = [c for c in contexts if c.get('score') is not None]
|
||||||
|
|
||||||
|
if scored_contexts:
|
||||||
|
filtered = [c for c in contexts if c.get('score', 0) >= self.RERANK_THRESHOLD]
|
||||||
|
return filtered if filtered else contexts
|
||||||
|
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
def _deduplicate_contexts(self, contexts: list, threshold: float = 0.9) -> list:
|
||||||
|
"""去重 - 基于内容相似度去重"""
|
||||||
|
if len(contexts) <= 1:
|
||||||
|
return contexts
|
||||||
|
|
||||||
|
result = []
|
||||||
|
seen_keys = set()
|
||||||
|
|
||||||
|
for c in contexts:
|
||||||
|
doc = c.get('doc', '')
|
||||||
|
key = doc[:100] if doc else ''
|
||||||
|
|
||||||
|
meta = c.get('meta', {})
|
||||||
|
source = meta.get('source', '')
|
||||||
|
page = meta.get('page', '')
|
||||||
|
|
||||||
|
composite_key = f"{source}|{page}|{key}"
|
||||||
|
|
||||||
|
if composite_key not in seen_keys:
|
||||||
|
seen_keys.add(composite_key)
|
||||||
|
result.append(c)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _truncate_to_tokens(self, contexts: list, max_tokens: int) -> list:
|
||||||
|
"""Token 控制 - 截断到最大 Token 数"""
|
||||||
|
result = []
|
||||||
|
total_tokens = 0
|
||||||
|
|
||||||
|
for c in contexts:
|
||||||
|
doc = c.get('doc', '')
|
||||||
|
# 简单估算:1 token ≈ 1.5 中文字符
|
||||||
|
tokens = len(doc) // 1.5
|
||||||
|
|
||||||
|
if total_tokens + tokens <= max_tokens:
|
||||||
|
result.append(c)
|
||||||
|
total_tokens += tokens
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _merge_and_deduplicate(self, old_contexts: list, new_contexts: list) -> list:
|
||||||
|
"""合并并去重两个上下文列表"""
|
||||||
|
result = list(old_contexts)
|
||||||
|
seen_keys = set()
|
||||||
|
|
||||||
|
# 记录已有上下文的 key
|
||||||
|
for c in old_contexts:
|
||||||
|
doc = c.get('doc', '')
|
||||||
|
key = doc[:100] if doc else ''
|
||||||
|
meta = c.get('meta', {})
|
||||||
|
source = meta.get('source', '')
|
||||||
|
composite_key = f"{source}|{key}"
|
||||||
|
seen_keys.add(composite_key)
|
||||||
|
|
||||||
|
# 添加新上下文(去重)
|
||||||
|
for c in new_contexts:
|
||||||
|
doc = c.get('doc', '')
|
||||||
|
key = doc[:100] if doc else ''
|
||||||
|
meta = c.get('meta', {})
|
||||||
|
source = meta.get('source', '')
|
||||||
|
composite_key = f"{source}|{key}"
|
||||||
|
|
||||||
|
if composite_key not in seen_keys:
|
||||||
|
seen_keys.add(composite_key)
|
||||||
|
result.append(c)
|
||||||
|
|
||||||
|
return result
|
||||||
200
core/agentic_media.py
Normal file
200
core/agentic_media.py
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 富媒体处理 Mixin
|
||||||
|
|
||||||
|
包含图表查找、图片提取、富媒体附加等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class RichMediaMixin:
|
||||||
|
"""富媒体处理方法"""
|
||||||
|
|
||||||
|
def _find_figure(self, query: str, contexts: list, source: str = None) -> dict:
|
||||||
|
"""精确查找图表,带 fallback"""
|
||||||
|
patterns = [
|
||||||
|
r'图\s*(\d+[\.\-]\d+)',
|
||||||
|
r'Fig\.?\s*(\d+[\.\-]\d+)',
|
||||||
|
r'Figure\s*(\d+[\.\-]\d+)',
|
||||||
|
]
|
||||||
|
|
||||||
|
target_figure = None
|
||||||
|
for pattern in patterns:
|
||||||
|
match = re.search(pattern, query, re.IGNORECASE)
|
||||||
|
if match:
|
||||||
|
target_figure = match.group(1).replace('-', '.')
|
||||||
|
break
|
||||||
|
|
||||||
|
if not target_figure:
|
||||||
|
return {"found": False}
|
||||||
|
|
||||||
|
# 从 contexts 中查找
|
||||||
|
for ctx in contexts:
|
||||||
|
meta = ctx.get('meta', {})
|
||||||
|
fig_num = meta.get('figure_number', '')
|
||||||
|
if fig_num == target_figure:
|
||||||
|
if not source or meta.get('source') == source:
|
||||||
|
return {
|
||||||
|
"found": True,
|
||||||
|
"chunk_id": meta.get('chunk_id'),
|
||||||
|
"source": meta.get('source'),
|
||||||
|
"page": meta.get('page'),
|
||||||
|
"caption": meta.get('caption'),
|
||||||
|
"image_path": meta.get('image_path'),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fallback: 直接查向量库
|
||||||
|
try:
|
||||||
|
from knowledge.manager import get_kb_manager
|
||||||
|
kb_mgr = get_kb_manager()
|
||||||
|
coll = kb_mgr.get_collection('public_kb')
|
||||||
|
|
||||||
|
if coll:
|
||||||
|
where_conditions = [{'chunk_type': {'$in': ['image', 'chart']}}]
|
||||||
|
if source:
|
||||||
|
where_conditions.append({'source': source})
|
||||||
|
|
||||||
|
result = coll.get(
|
||||||
|
where={'$and': where_conditions} if len(where_conditions) > 1 else where_conditions[0],
|
||||||
|
include=['metadatas', 'documents']
|
||||||
|
)
|
||||||
|
|
||||||
|
for meta, doc in zip(result.get('metadatas', []), result.get('documents', [])):
|
||||||
|
if meta.get('figure_number') == target_figure:
|
||||||
|
return {
|
||||||
|
"found": True,
|
||||||
|
"chunk_id": meta.get('chunk_id'),
|
||||||
|
"source": meta.get('source'),
|
||||||
|
"page": meta.get('page'),
|
||||||
|
"caption": meta.get('caption'),
|
||||||
|
"image_path": meta.get('image_path'),
|
||||||
|
}
|
||||||
|
caption = meta.get('caption', '') or (doc if doc else '')
|
||||||
|
if f"图{target_figure}" in caption or f"图 {target_figure}" in caption:
|
||||||
|
return {
|
||||||
|
"found": True,
|
||||||
|
"chunk_id": meta.get('chunk_id'),
|
||||||
|
"source": meta.get('source'),
|
||||||
|
"page": meta.get('page'),
|
||||||
|
"caption": meta.get('caption'),
|
||||||
|
"image_path": meta.get('image_path'),
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"_find_figure fallback 查询失败: {e}")
|
||||||
|
|
||||||
|
return {"found": False}
|
||||||
|
|
||||||
|
def _get_images_for_source(self, source: str, collections: list = None) -> list:
|
||||||
|
"""直接从向量库获取指定文件的所有图片"""
|
||||||
|
try:
|
||||||
|
from knowledge.manager import get_kb_manager
|
||||||
|
kb_mgr = get_kb_manager()
|
||||||
|
except ImportError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
images = []
|
||||||
|
seen_ids = set()
|
||||||
|
|
||||||
|
target_collections = collections or ['public_kb']
|
||||||
|
|
||||||
|
for kb_name in target_collections:
|
||||||
|
try:
|
||||||
|
coll = kb_mgr.get_collection(kb_name)
|
||||||
|
if not coll:
|
||||||
|
continue
|
||||||
|
|
||||||
|
result = coll.get(
|
||||||
|
where={'source': source},
|
||||||
|
include=['metadatas']
|
||||||
|
)
|
||||||
|
|
||||||
|
for meta in result.get('metadatas', []):
|
||||||
|
images_json = meta.get('images_json')
|
||||||
|
if images_json:
|
||||||
|
try:
|
||||||
|
imgs = json.loads(images_json)
|
||||||
|
for img in imgs:
|
||||||
|
img_id = img.get('id')
|
||||||
|
if img_id and img_id not in seen_ids:
|
||||||
|
seen_ids.add(img_id)
|
||||||
|
images.append({
|
||||||
|
"id": img_id,
|
||||||
|
"caption": img.get("caption", ""),
|
||||||
|
"url": f"/images/{img_id}",
|
||||||
|
"page": img.get("page") or meta.get("page"),
|
||||||
|
"source": source,
|
||||||
|
"width": img.get("width"),
|
||||||
|
"height": img.get("height")
|
||||||
|
})
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"从 {kb_name} 获取图片失败: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
return images
|
||||||
|
|
||||||
|
def _extract_rich_media(self, contexts: list, sources_filter: list = None, max_images: int = 10,
|
||||||
|
max_tables: int = 5) -> dict:
|
||||||
|
"""从检索结果中提取富媒体(图片、表格)"""
|
||||||
|
images = []
|
||||||
|
tables = []
|
||||||
|
seen_image_ids = set()
|
||||||
|
seen_table_ids = set()
|
||||||
|
|
||||||
|
for ctx in contexts:
|
||||||
|
meta = ctx.get('meta', {})
|
||||||
|
source = meta.get('source', '')
|
||||||
|
|
||||||
|
# 过滤来源
|
||||||
|
if sources_filter and source not in sources_filter:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 提取图片
|
||||||
|
images_json = meta.get('images_json')
|
||||||
|
if images_json:
|
||||||
|
try:
|
||||||
|
imgs = json.loads(images_json)
|
||||||
|
for img in imgs:
|
||||||
|
img_id = img.get('id')
|
||||||
|
if img_id and img_id not in seen_image_ids:
|
||||||
|
seen_image_ids.add(img_id)
|
||||||
|
images.append({
|
||||||
|
"id": img_id,
|
||||||
|
"caption": img.get("caption", ""),
|
||||||
|
"url": f"/images/{img_id}",
|
||||||
|
"page": img.get("page") or meta.get("page"),
|
||||||
|
"source": source,
|
||||||
|
"type": img.get("type", "image")
|
||||||
|
})
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 提取表格
|
||||||
|
table_json = meta.get('table_json')
|
||||||
|
if table_json:
|
||||||
|
try:
|
||||||
|
tbl = json.loads(table_json)
|
||||||
|
tbl_id = tbl.get('id') or meta.get('chunk_id')
|
||||||
|
if tbl_id and tbl_id not in seen_table_ids:
|
||||||
|
seen_table_ids.add(tbl_id)
|
||||||
|
tables.append({
|
||||||
|
"id": tbl_id,
|
||||||
|
"caption": tbl.get("caption", ""),
|
||||||
|
"markdown": tbl.get("markdown", ""),
|
||||||
|
"page": meta.get("page"),
|
||||||
|
"source": source
|
||||||
|
})
|
||||||
|
except (json.JSONDecodeError, TypeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"images": images[:max_images],
|
||||||
|
"tables": tables[:max_tables]
|
||||||
|
}
|
||||||
133
core/agentic_meta.py
Normal file
133
core/agentic_meta.py
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 元问题处理 Mixin
|
||||||
|
|
||||||
|
包含元问题判断和知识库元数据回答方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class MetaQuestionMixin:
|
||||||
|
"""元问题处理方法"""
|
||||||
|
|
||||||
|
def _is_meta_question(self, query: str) -> bool:
|
||||||
|
"""判断是否为元问题(关于知识库本身的问题)"""
|
||||||
|
meta_patterns = [
|
||||||
|
"有哪些文件", "什么文件", "哪些文件", "文件列表", "文件目录",
|
||||||
|
"可以查看", "能查看", "有权限查看", "权限查看",
|
||||||
|
"能访问", "可以访问", "有权限访问",
|
||||||
|
"我的权限", "用户权限", "查看权限", "访问权限",
|
||||||
|
"权限能", "权限可以", "有什么权限", "有哪些权限",
|
||||||
|
"我能看", "我可以看", "我能查", "我可以查",
|
||||||
|
"能看到什么", "能查到什么", "可以看什么", "可以查什么",
|
||||||
|
"知识库有哪些", "库里有", "文档有哪些", "有哪些文档",
|
||||||
|
"有什么文档", "有什么文件", "包含什么", "包含哪些",
|
||||||
|
"你知道什么", "你都知道", "你能回答什么",
|
||||||
|
"系统里有什么", "库里有什么",
|
||||||
|
"public_kb", "dept_tech", "dept_hr", "dept_finance", "dept_operation",
|
||||||
|
"kb里", "向量库", "有哪些库", "库列表", "kb有哪些"
|
||||||
|
]
|
||||||
|
query_lower = query.lower()
|
||||||
|
return any(kw in query_lower for kw in meta_patterns)
|
||||||
|
|
||||||
|
def _answer_meta_question(self, query: str, allowed_levels: list = None,
|
||||||
|
role: str = None, department: str = None) -> str:
|
||||||
|
"""回答元问题(关于知识库本身的问题)"""
|
||||||
|
try:
|
||||||
|
source_map = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
from knowledge.manager import get_kb_manager
|
||||||
|
from auth.gateway import get_accessible_collections as _get_accessible
|
||||||
|
|
||||||
|
kb_mgr = get_kb_manager()
|
||||||
|
accessible = _get_accessible(role or 'user', department or '', 'read')
|
||||||
|
|
||||||
|
for kb_name in accessible:
|
||||||
|
coll = kb_mgr.get_collection(kb_name)
|
||||||
|
if not coll:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result = coll.get(include=['metadatas'])
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"获取{kb_name}元数据失败: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
for meta in result.get('metadatas', []):
|
||||||
|
source = meta.get('source', '未知')
|
||||||
|
level = meta.get('security_level', 'public')
|
||||||
|
page = meta.get('page')
|
||||||
|
|
||||||
|
if source not in source_map:
|
||||||
|
source_map[source] = {
|
||||||
|
'count': 0, 'levels': set(),
|
||||||
|
'pages': set(), 'collections': set()
|
||||||
|
}
|
||||||
|
|
||||||
|
source_map[source]['count'] += 1
|
||||||
|
source_map[source]['levels'].add(level)
|
||||||
|
source_map[source]['collections'].add(kb_name)
|
||||||
|
if page:
|
||||||
|
source_map[source]['pages'].add(page)
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
from core.engine import get_engine
|
||||||
|
all_docs = get_engine().collection.get(include=['metadatas'])
|
||||||
|
for meta in all_docs.get('metadatas', []):
|
||||||
|
source = meta.get('source', '未知')
|
||||||
|
level = meta.get('security_level', 'public')
|
||||||
|
page = meta.get('page')
|
||||||
|
|
||||||
|
if source not in source_map:
|
||||||
|
source_map[source] = {
|
||||||
|
'count': 0, 'levels': set(),
|
||||||
|
'pages': set(), 'collections': set()
|
||||||
|
}
|
||||||
|
|
||||||
|
source_map[source]['count'] += 1
|
||||||
|
source_map[source]['levels'].add(level)
|
||||||
|
if page:
|
||||||
|
source_map[source]['pages'].add(page)
|
||||||
|
|
||||||
|
# 根据安全级别过滤
|
||||||
|
if allowed_levels:
|
||||||
|
allowed_set = set(allowed_levels)
|
||||||
|
filtered_sources = {}
|
||||||
|
for source, info in source_map.items():
|
||||||
|
if info['levels'] & allowed_set:
|
||||||
|
filtered_sources[source] = info
|
||||||
|
source_map = filtered_sources
|
||||||
|
|
||||||
|
if not source_map:
|
||||||
|
return "抱歉,您当前没有权限查看任何文档,或者知识库为空。"
|
||||||
|
|
||||||
|
sorted_sources = sorted(source_map.items(), key=lambda x: x[1]['count'], reverse=True)
|
||||||
|
|
||||||
|
answer_parts = [f"📚 **知识库文档列表**(共 {len(sorted_sources)} 个文档)\n"]
|
||||||
|
|
||||||
|
for i, (source, info) in enumerate(sorted_sources, 1):
|
||||||
|
colls = info.get('collections', set())
|
||||||
|
coll_str = f",所属: {', '.join(sorted(colls))}" if colls else ""
|
||||||
|
pages_str = ''
|
||||||
|
if info['pages']:
|
||||||
|
pages_list = sorted(info['pages'])
|
||||||
|
if len(pages_list) <= 5:
|
||||||
|
pages_str = f",页码: {', '.join(map(str, pages_list))}"
|
||||||
|
else:
|
||||||
|
pages_str = f",共 {len(info['pages'])} 页"
|
||||||
|
|
||||||
|
answer_parts.append(f"{i}. **{source}** ({info['count']} 条片段{coll_str}{pages_str})")
|
||||||
|
|
||||||
|
answer_parts.append(f"\n**总计**: {sum(s[1]['count'] for s in sorted_sources)} 条知识片段")
|
||||||
|
answer_parts.append(f"\n**您的权限级别**: {', '.join(allowed_levels) if allowed_levels else '全部'}")
|
||||||
|
|
||||||
|
answer_parts.append("\n\n💡 **提示**: 您可以直接提问关于这些文档内容的问题。")
|
||||||
|
|
||||||
|
return '\n'.join(answer_parts)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
return f"获取文档列表时出错: {str(e)}\n\n您可以直接提问,我会尝试从知识库中检索相关信息。"
|
||||||
137
core/agentic_quality.py
Normal file
137
core/agentic_quality.py
Normal file
@@ -0,0 +1,137 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 质量评估 Mixin
|
||||||
|
|
||||||
|
包含置信度门控、质量评估、推理反思等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class QualityMixin:
|
||||||
|
"""质量评估方法"""
|
||||||
|
|
||||||
|
def _check_confidence_gate(self, query: str, docs: list, verbose: bool = True,
|
||||||
|
precomputed_scores: list = None):
|
||||||
|
"""检查置信度门控
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: 用户查询
|
||||||
|
docs: 文档列表
|
||||||
|
verbose: 是否详细输出
|
||||||
|
precomputed_scores: 预计算的 Rerank 分数(可选,避免重复推理)
|
||||||
|
"""
|
||||||
|
if not self.confidence_gate:
|
||||||
|
return {"passed": True, "reason": "no_gate"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = self.confidence_gate.evaluate(query, docs,
|
||||||
|
precomputed_scores=precomputed_scores)
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"置信度门控检查失败: {e}")
|
||||||
|
return {"passed": True, "reason": "error"}
|
||||||
|
|
||||||
|
def _assess_quality(self, query: str, docs: list, metas: list = None,
|
||||||
|
verbose: bool = True) -> dict:
|
||||||
|
"""多维质量评估"""
|
||||||
|
if not self.quality_assessor:
|
||||||
|
return {"overall_score": 0.5, "dimensions": {}}
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = self.quality_assessor.assess(query, docs, metas)
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"质量评估失败: {e}")
|
||||||
|
return {"overall_score": 0.5, "dimensions": {}}
|
||||||
|
|
||||||
|
def _reflect_on_answer(self, query: str, answer: str, contexts: list,
|
||||||
|
verbose: bool = True) -> dict:
|
||||||
|
"""推理反思"""
|
||||||
|
if not self.reasoning_reflector:
|
||||||
|
return {"needs_reflection": False, "issues": []}
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = self.reasoning_reflector.reflect(query, answer, contexts)
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"推理反思失败: {e}")
|
||||||
|
return {"needs_reflection": False, "issues": []}
|
||||||
|
|
||||||
|
def _think(self, original_query: str, current_query: str,
|
||||||
|
iteration: int, contexts: list, verbose: bool = True) -> dict:
|
||||||
|
"""
|
||||||
|
Agent 思考:决定下一步行动
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
"action": "answer" | "rewrite" | "search_web" | "decompose",
|
||||||
|
"reason": "...",
|
||||||
|
"rewrite_query": "..." # 如果 action == "rewrite"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
from core.llm_utils import call_llm, parse_json_from_response
|
||||||
|
from .agentic_base import MODEL
|
||||||
|
|
||||||
|
# 构建思考提示
|
||||||
|
context_summary = ""
|
||||||
|
if contexts:
|
||||||
|
for i, ctx in enumerate(contexts[:3], 1):
|
||||||
|
meta = ctx.get('meta', {})
|
||||||
|
source = meta.get('source', '未知')
|
||||||
|
doc_preview = ctx.get('doc', '')[:100]
|
||||||
|
context_summary += f"{i}. [{source}] {doc_preview}...\n"
|
||||||
|
|
||||||
|
prompt = f"""你是一个 RAG 系统的决策 Agent,需要判断下一步行动。
|
||||||
|
|
||||||
|
【原始问题】
|
||||||
|
{original_query}
|
||||||
|
|
||||||
|
【当前问题】
|
||||||
|
{current_query}
|
||||||
|
|
||||||
|
【迭代轮次】
|
||||||
|
{iteration} / {self.max_iterations}
|
||||||
|
|
||||||
|
【已检索到的上下文】
|
||||||
|
{context_summary if context_summary else "(无)"}
|
||||||
|
|
||||||
|
【可选行动】
|
||||||
|
1. answer - 已有足够信息,可以回答
|
||||||
|
2. rewrite - 查询不够清晰,需要重写
|
||||||
|
3. search_web - 知识库信息不足,需要网络搜索
|
||||||
|
4. decompose - 问题太复杂,需要分解
|
||||||
|
|
||||||
|
【决策要求】
|
||||||
|
- 如果上下文足够回答问题,选择 answer
|
||||||
|
- 如果上下文不足且迭代未超限,选择 search_web 或 rewrite
|
||||||
|
- 返回 JSON 格式
|
||||||
|
|
||||||
|
请决策:"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = call_llm(
|
||||||
|
self.client, prompt, MODEL,
|
||||||
|
temperature=0.3,
|
||||||
|
max_tokens=200
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = parse_json_from_response(result) if result else {}
|
||||||
|
|
||||||
|
# 默认决策
|
||||||
|
if not decision or "action" not in decision:
|
||||||
|
if contexts and len(contexts) >= 2:
|
||||||
|
decision = {"action": "answer", "reason": "有足够上下文"}
|
||||||
|
else:
|
||||||
|
decision = {"action": "rewrite", "reason": "上下文不足"}
|
||||||
|
|
||||||
|
return decision
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Agent 思考失败: {e}")
|
||||||
|
if contexts:
|
||||||
|
return {"action": "answer", "reason": "默认回答"}
|
||||||
|
return {"action": "rewrite", "reason": "默认重写"}
|
||||||
271
core/agentic_query.py
Normal file
271
core/agentic_query.py
Normal file
@@ -0,0 +1,271 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 查询重写 Mixin
|
||||||
|
|
||||||
|
包含查询改写、实体补全、专业术语映射等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .agentic_base import logger, MODEL
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class QueryRewriteMixin:
|
||||||
|
"""查询重写方法"""
|
||||||
|
|
||||||
|
def _rewrite_query(self, query: str, history: list = None,
|
||||||
|
strategy: str = "professional") -> str:
|
||||||
|
"""
|
||||||
|
增强版查询重写:将口语化表达转为专业术语
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: 原始查询
|
||||||
|
history: 对话历史(用于实体补全)
|
||||||
|
strategy: 重写策略
|
||||||
|
- professional: 口语化→专业术语
|
||||||
|
- expand: 扩展关键词
|
||||||
|
- clarify: 消歧义
|
||||||
|
- entity: 实体补全
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: 重写后的查询
|
||||||
|
"""
|
||||||
|
# 尝试多种策略组合
|
||||||
|
rewritten = query
|
||||||
|
|
||||||
|
# 策略1: 口语化→专业术语映射
|
||||||
|
if strategy in ["professional", "all"]:
|
||||||
|
rewritten = self._apply_professional_mapping(rewritten)
|
||||||
|
|
||||||
|
# 策略2: 实体补全(利用对话历史)
|
||||||
|
if strategy in ["entity", "all"] and history:
|
||||||
|
rewritten = self._complete_entities(rewritten, history)
|
||||||
|
|
||||||
|
# 策略3: LLM 深度重写(仅在需要时调用)
|
||||||
|
if strategy in ["professional", "all"]:
|
||||||
|
llm_rewritten = self._llm_rewrite(rewritten)
|
||||||
|
if llm_rewritten and len(llm_rewritten) > len(rewritten) * 0.5:
|
||||||
|
rewritten = llm_rewritten
|
||||||
|
|
||||||
|
return rewritten
|
||||||
|
|
||||||
|
def _apply_professional_mapping(self, query: str) -> str:
|
||||||
|
"""应用口语化→专业术语映射"""
|
||||||
|
TERM_MAPPING = {
|
||||||
|
"报销": "差旅报销 费用报销 报销审批",
|
||||||
|
"请假": "休假申请 请假审批 考勤管理",
|
||||||
|
"加班": "加班申请 工时管理 加班审批",
|
||||||
|
"工资": "薪酬管理 工资发放 薪资结构",
|
||||||
|
"合同": "合同管理 合同签署 合同审批",
|
||||||
|
"流程": "审批流程 业务流程 工作流",
|
||||||
|
"制度": "管理制度 规章制度 企业规范",
|
||||||
|
"规定": "管理规定 制度规定 政策要求",
|
||||||
|
"几天": "时限 审批时限 办理时限",
|
||||||
|
"多久": "处理时效 审批周期 办理周期",
|
||||||
|
"多少": "标准 额度 限额 标准",
|
||||||
|
"能不能": "是否允许 是否可以 权限",
|
||||||
|
"人事": "人力资源 HR 人力部门",
|
||||||
|
"财务": "财务部 财务部门 财务管理",
|
||||||
|
"技术": "技术部 研发部 IT部门",
|
||||||
|
}
|
||||||
|
|
||||||
|
result = query
|
||||||
|
for colloquial, professional in TERM_MAPPING.items():
|
||||||
|
if colloquial in query:
|
||||||
|
result = result.replace(colloquial, f"{colloquial} {professional.split()[0]}")
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _complete_entities(self, query: str, history: list) -> str:
|
||||||
|
"""实体补全:利用对话历史补充缺失的实体"""
|
||||||
|
if not history:
|
||||||
|
return query
|
||||||
|
|
||||||
|
# 图片指代识别
|
||||||
|
image_reference = self._detect_image_reference(query, history)
|
||||||
|
if image_reference:
|
||||||
|
return image_reference
|
||||||
|
|
||||||
|
# 获取最近用户消息
|
||||||
|
last_user_msg = None
|
||||||
|
for msg in reversed(history):
|
||||||
|
if msg.get("role") == "user":
|
||||||
|
last_user_msg = msg.get("content", "")
|
||||||
|
break
|
||||||
|
|
||||||
|
if not last_user_msg:
|
||||||
|
return query
|
||||||
|
|
||||||
|
# 检查当前查询是否缺少主语
|
||||||
|
BUSINESS_KEYWORDS = ["报销", "出差", "请假", "工资", "合同", "审批", "流程",
|
||||||
|
"制度", "规定", "标准", "金额", "时间"]
|
||||||
|
|
||||||
|
has_subject = any(kw in query for kw in BUSINESS_KEYWORDS)
|
||||||
|
|
||||||
|
if not has_subject:
|
||||||
|
try:
|
||||||
|
import jieba
|
||||||
|
entities = []
|
||||||
|
for word in jieba.cut(last_user_msg):
|
||||||
|
word = word.strip()
|
||||||
|
if len(word) >= 2 and any(kw in word for kw in BUSINESS_KEYWORDS):
|
||||||
|
entities.append(word)
|
||||||
|
|
||||||
|
if entities:
|
||||||
|
return f"{entities[0]} {query}"
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return query
|
||||||
|
|
||||||
|
def _detect_image_reference(self, query: str, history: list) -> str:
|
||||||
|
"""检测图片指代查询并重写"""
|
||||||
|
IMAGE_REFERENCE_PATTERNS = [
|
||||||
|
r'这[张些]图片', r'那[张些]图片', r'上面的图片', r'刚才的图片',
|
||||||
|
r'这[张些]图', r'那[张些]图', r'上面的图', r'刚才的图',
|
||||||
|
r'解释一下这[张些]图', r'说明一下这[张些]图',
|
||||||
|
r'这[张些]是什么图', r'图[里内]是什么', r'图片[里内]是什么',
|
||||||
|
]
|
||||||
|
|
||||||
|
is_image_reference = False
|
||||||
|
for pattern in IMAGE_REFERENCE_PATTERNS:
|
||||||
|
if re.search(pattern, query):
|
||||||
|
is_image_reference = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if not is_image_reference:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
last_images = []
|
||||||
|
for msg in reversed(history):
|
||||||
|
if msg.get("role") == "assistant":
|
||||||
|
metadata = msg.get("metadata", {})
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
images = metadata.get("images", [])
|
||||||
|
if images:
|
||||||
|
for img in images[:5]:
|
||||||
|
if isinstance(img, dict):
|
||||||
|
desc = img.get("description", "")
|
||||||
|
img_type = img.get("type", "图片")
|
||||||
|
if desc:
|
||||||
|
last_images.append(f"{img_type}:{desc}")
|
||||||
|
elif isinstance(img, str):
|
||||||
|
last_images.append(f"图片:{img}")
|
||||||
|
|
||||||
|
if not last_images:
|
||||||
|
content = msg.get("content", "")
|
||||||
|
if "图片" in content or "图表" in content or "图" in content:
|
||||||
|
sentences = content.split("。")
|
||||||
|
for sentence in sentences:
|
||||||
|
if "图片" in sentence or "图表" in sentence:
|
||||||
|
last_images.append(sentence.strip())
|
||||||
|
|
||||||
|
if last_images:
|
||||||
|
break
|
||||||
|
|
||||||
|
if last_images:
|
||||||
|
image_context = " ".join(last_images[:3])
|
||||||
|
question_intent = re.sub(
|
||||||
|
r'这[张些]图片?|那[张些]图片?|上面的图片?|刚才的图片?|解释一下|说明一下',
|
||||||
|
'', query
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
if question_intent:
|
||||||
|
return f"{image_context} {question_intent}"
|
||||||
|
else:
|
||||||
|
return f"详细解释:{image_context}"
|
||||||
|
|
||||||
|
return query
|
||||||
|
|
||||||
|
def _extract_image_context_from_history(self, history: list) -> str:
|
||||||
|
"""从对话历史中提取图片上下文"""
|
||||||
|
if not history:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
for msg in reversed(history):
|
||||||
|
if msg.get("role") == "assistant":
|
||||||
|
metadata = msg.get("metadata", {})
|
||||||
|
images = metadata.get("images", [])
|
||||||
|
content = msg.get("content", "")
|
||||||
|
|
||||||
|
image_descriptions = []
|
||||||
|
|
||||||
|
if images:
|
||||||
|
for i, img in enumerate(images[:5], 1):
|
||||||
|
if isinstance(img, dict):
|
||||||
|
desc = img.get("description", "")
|
||||||
|
img_type = img.get("type", "图片")
|
||||||
|
source = img.get("source", "")
|
||||||
|
page = img.get("page", "")
|
||||||
|
|
||||||
|
img_info = f"图片{i}:{img_type}"
|
||||||
|
if desc:
|
||||||
|
img_info += f",描述:{desc}"
|
||||||
|
if source:
|
||||||
|
img_info += f",来源:{source}"
|
||||||
|
if page:
|
||||||
|
img_info += f",第{page}页"
|
||||||
|
image_descriptions.append(img_info)
|
||||||
|
|
||||||
|
if not image_descriptions:
|
||||||
|
if "图片" in content or "图表" in content:
|
||||||
|
sentences = content.split("。")
|
||||||
|
for sentence in sentences:
|
||||||
|
if "图片" in sentence or "图表" in sentence:
|
||||||
|
image_descriptions.append(sentence.strip())
|
||||||
|
if len(image_descriptions) >= 3:
|
||||||
|
break
|
||||||
|
|
||||||
|
if image_descriptions:
|
||||||
|
return "\n".join(image_descriptions)
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def _answer_image_reference(self, enhanced_query: str, history: list) -> str:
|
||||||
|
"""回答图片引用问题"""
|
||||||
|
from core.llm_utils import call_llm
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": "你是一个专业的助手,请根据提供的图片信息回答用户的问题。"}
|
||||||
|
]
|
||||||
|
|
||||||
|
for h in history[-4:]:
|
||||||
|
if h.get("role") in ["user", "assistant"]:
|
||||||
|
messages.append({"role": h["role"], "content": h.get("content", "")})
|
||||||
|
|
||||||
|
messages.append({"role": "user", "content": enhanced_query})
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = call_llm(
|
||||||
|
self.client, "", MODEL,
|
||||||
|
temperature=0.3,
|
||||||
|
max_tokens=1000,
|
||||||
|
messages=messages
|
||||||
|
)
|
||||||
|
return result or ""
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"图片引用回答失败: {e}")
|
||||||
|
return f"抱歉,回答图片问题时出现错误:{str(e)}"
|
||||||
|
|
||||||
|
def _llm_rewrite(self, query: str) -> str:
|
||||||
|
"""LLM 深度重写查询"""
|
||||||
|
from core.llm_utils import call_llm
|
||||||
|
|
||||||
|
prompt = f"""请将以下用户问题改写为更专业、更清晰的表达,保持原意不变。
|
||||||
|
|
||||||
|
原问题:{query}
|
||||||
|
|
||||||
|
改写后的问题:"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
rewritten = call_llm(
|
||||||
|
self.client, prompt, MODEL,
|
||||||
|
temperature=0.3,
|
||||||
|
max_tokens=100
|
||||||
|
)
|
||||||
|
return rewritten.strip() if rewritten else query
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"LLM 重写失败: {e}")
|
||||||
|
return query
|
||||||
152
core/agentic_search.py
Normal file
152
core/agentic_search.py
Normal file
@@ -0,0 +1,152 @@
|
|||||||
|
"""
|
||||||
|
Agentic RAG - 检索 Mixin
|
||||||
|
|
||||||
|
包含知识库检索、网络搜索等方法
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .agentic_base import (
|
||||||
|
logger, HAS_SERPER, SERPER_API_KEY,
|
||||||
|
SOURCE_KB, SOURCE_WEB
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class SearchMixin:
|
||||||
|
"""检索功能方法"""
|
||||||
|
|
||||||
|
def _web_search(self, query: str, top_k: int = 5) -> list:
|
||||||
|
"""网络搜索(使用Serper API)"""
|
||||||
|
if not HAS_SERPER:
|
||||||
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
url = "https://google.serper.dev/search"
|
||||||
|
payload = json.dumps({
|
||||||
|
"q": query,
|
||||||
|
"gl": "cn",
|
||||||
|
"hl": "zh-cn",
|
||||||
|
"num": top_k
|
||||||
|
})
|
||||||
|
headers = {
|
||||||
|
'X-API-KEY': SERPER_API_KEY,
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
}
|
||||||
|
|
||||||
|
response = requests.post(url, headers=headers, data=payload, timeout=10)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for item in data.get('organic', [])[:top_k]:
|
||||||
|
results.append({
|
||||||
|
'title': item.get('title', ''),
|
||||||
|
'link': item.get('link', ''),
|
||||||
|
'snippet': item.get('snippet', ''),
|
||||||
|
'date': item.get('date', '')
|
||||||
|
})
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"网络搜索失败: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _should_web_search(self, query: str) -> bool:
|
||||||
|
"""判断是否需要网络搜索"""
|
||||||
|
realtime_keywords = [
|
||||||
|
"今天", "最新", "今日", "当前", "现在",
|
||||||
|
"天气", "新闻", "股价", "行情", "汇率",
|
||||||
|
"最近", "近期", "这周", "本月", "今年",
|
||||||
|
"实时", "动态", "热点", "发生"
|
||||||
|
]
|
||||||
|
|
||||||
|
query_lower = query.lower()
|
||||||
|
return any(kw in query_lower for kw in realtime_keywords)
|
||||||
|
|
||||||
|
def _web_search_flow(self, query: str, log_trace: list, emit_log, verbose: bool,
|
||||||
|
allowed_levels: list = None) -> list:
|
||||||
|
"""
|
||||||
|
网络搜索流程
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: 查询
|
||||||
|
log_trace: 日志追踪列表
|
||||||
|
emit_log: 日志发射函数
|
||||||
|
verbose: 是否详细输出
|
||||||
|
allowed_levels: 允许的安全级别
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
网络搜索结果列表
|
||||||
|
"""
|
||||||
|
if not self.enable_web_search or not HAS_SERPER:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if emit_log:
|
||||||
|
emit_log("🌐 触发网络搜索...")
|
||||||
|
|
||||||
|
web_results = self._web_search(query, top_k=5)
|
||||||
|
|
||||||
|
if not web_results:
|
||||||
|
if emit_log:
|
||||||
|
emit_log("⚠️ 网络搜索未返回结果")
|
||||||
|
return []
|
||||||
|
|
||||||
|
# 转换为统一上下文格式
|
||||||
|
web_contexts = []
|
||||||
|
for item in web_results:
|
||||||
|
web_contexts.append({
|
||||||
|
'doc': f"{item.get('title', '')}\n{item.get('snippet', '')}",
|
||||||
|
'meta': {
|
||||||
|
'source': self.SOURCE_WEB,
|
||||||
|
'link': item.get('link', ''),
|
||||||
|
'date': item.get('date', '')
|
||||||
|
},
|
||||||
|
'source_type': self.SOURCE_WEB,
|
||||||
|
'query': query
|
||||||
|
})
|
||||||
|
|
||||||
|
log_trace.append({
|
||||||
|
'phase': 'web_search',
|
||||||
|
'query': query,
|
||||||
|
'results_count': len(web_contexts)
|
||||||
|
})
|
||||||
|
|
||||||
|
if emit_log:
|
||||||
|
emit_log(f"✅ 网络搜索返回 {len(web_contexts)} 条结果")
|
||||||
|
|
||||||
|
return web_contexts
|
||||||
|
|
||||||
|
def _is_kb_result_sufficient(self, query: str, docs: list) -> bool:
|
||||||
|
"""判断知识库检索结果是否充分"""
|
||||||
|
if not docs:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# 结果数量检查
|
||||||
|
if len(docs) >= 3:
|
||||||
|
# 至少3条结果,检查相关性
|
||||||
|
high_rel_count = 0
|
||||||
|
for doc in docs:
|
||||||
|
score = doc.get('score', 0) or doc.get('distance', 1)
|
||||||
|
# cosine 距离转相似度
|
||||||
|
if isinstance(score, (int, float)):
|
||||||
|
sim = 1 - score if score <= 1 else score
|
||||||
|
if sim >= 0.6:
|
||||||
|
high_rel_count += 1
|
||||||
|
|
||||||
|
if high_rel_count >= 2:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# 有高质量结果
|
||||||
|
for doc in docs[:2]:
|
||||||
|
score = doc.get('score', 0) or doc.get('distance', 1)
|
||||||
|
if isinstance(score, (int, float)):
|
||||||
|
sim = 1 - score if score <= 1 else score
|
||||||
|
if sim >= 0.8:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
@@ -224,36 +224,39 @@ class RAGCacheManager:
|
|||||||
"""
|
"""
|
||||||
获取查询缓存结果
|
获取查询缓存结果
|
||||||
|
|
||||||
始终使用粗粒度 key(基于 kb_version),确保 GET/SET key 一致。
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
query: 查询文本
|
query: 查询文本
|
||||||
kb_name: 知识库名称
|
kb_name: 知识库名称
|
||||||
doc_ids: 保留参数以兼容调用方签名(当前未使用)
|
doc_ids: 相关文档 ID 列表(用于细粒度缓存 key)
|
||||||
"""
|
"""
|
||||||
kb_version = self.get_kb_version(kb_name)
|
kb_version = self.get_kb_version(kb_name)
|
||||||
|
|
||||||
# 使用粗粒度 key,与 SET 保持一致
|
# 计算文档哈希(如果提供了 doc_ids)
|
||||||
key = self._make_query_cache_key(query, kb_name, kb_version)
|
doc_hash = ""
|
||||||
|
if doc_ids:
|
||||||
|
doc_hash = self._compute_doc_hash(kb_name, doc_ids)
|
||||||
|
|
||||||
|
key = self._make_query_cache_key(query, kb_name, kb_version, doc_hash)
|
||||||
return self.query_cache.get(key)
|
return self.query_cache.get(key)
|
||||||
|
|
||||||
def set_query_result(self, query: str, kb_name: str, result: Dict, doc_ids: List[str] = None) -> None:
|
def set_query_result(self, query: str, kb_name: str, result: Dict, doc_ids: List[str] = None) -> None:
|
||||||
"""
|
"""
|
||||||
设置查询缓存结果
|
设置查询缓存结果
|
||||||
|
|
||||||
始终使用粗粒度 key(基于 kb_version),确保 GET/SET key 一致。
|
|
||||||
kb_version 在文档变更时自增,触发整个知识库的缓存失效。
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
query: 查询文本
|
query: 查询文本
|
||||||
kb_name: 知识库名称
|
kb_name: 知识库名称
|
||||||
result: 缓存结果
|
result: 缓存结果
|
||||||
doc_ids: 保留参数以兼容调用方签名(当前未使用)
|
doc_ids: 相关文档 ID 列表(用于细粒度失效)
|
||||||
"""
|
"""
|
||||||
kb_version = self.get_kb_version(kb_name)
|
kb_version = self.get_kb_version(kb_name)
|
||||||
|
|
||||||
# 使用与 GET 相同的粗粒度 key,确保缓存可命中
|
# 计算相关文档的版本哈希(细粒度失效)
|
||||||
key = self._make_query_cache_key(query, kb_name, kb_version)
|
doc_hash = ""
|
||||||
|
if doc_ids:
|
||||||
|
doc_hash = self._compute_doc_hash(kb_name, doc_ids)
|
||||||
|
|
||||||
|
key = self._make_query_cache_key(query, kb_name, kb_version, doc_hash)
|
||||||
self.query_cache.set(key, result, kb_version=kb_version)
|
self.query_cache.set(key, result, kb_version=kb_version)
|
||||||
|
|
||||||
def _compute_doc_hash(self, kb_name: str, doc_ids: List[str]) -> str:
|
def _compute_doc_hash(self, kb_name: str, doc_ids: List[str]) -> str:
|
||||||
|
|||||||
105
core/engine.py
105
core/engine.py
@@ -113,7 +113,7 @@ except ImportError:
|
|||||||
RERANK_DEVICE = "auto"
|
RERANK_DEVICE = "auto"
|
||||||
RERANK_USE_ONNX = False
|
RERANK_USE_ONNX = False
|
||||||
RERANK_BACKEND = "local"
|
RERANK_BACKEND = "local"
|
||||||
RERANK_CLOUD_MODEL = "xop3qwen8breranker"
|
RERANK_CLOUD_MODEL = "qwen3-rerank"
|
||||||
RERANK_CLOUD_API_KEY = ""
|
RERANK_CLOUD_API_KEY = ""
|
||||||
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
||||||
RERANK_CLOUD_TIMEOUT = 15
|
RERANK_CLOUD_TIMEOUT = 15
|
||||||
@@ -273,7 +273,8 @@ class CloudReranker:
|
|||||||
body = {
|
body = {
|
||||||
"model": self.model,
|
"model": self.model,
|
||||||
"query": query,
|
"query": query,
|
||||||
"documents": documents
|
"documents": documents,
|
||||||
|
"top_n": len(documents)
|
||||||
}
|
}
|
||||||
|
|
||||||
session = self._get_session()
|
session = self._get_session()
|
||||||
@@ -1231,7 +1232,6 @@ class RAGEngine:
|
|||||||
logger.warning(f"扩展连续切片失败: {e}")
|
logger.warning(f"扩展连续切片失败: {e}")
|
||||||
return {'ids': [], 'documents': [], 'metadatas': []}
|
return {'ids': [], 'documents': [], 'metadatas': []}
|
||||||
|
|
||||||
# 扩展同 section 的 text 邻居
|
|
||||||
where_filter = {"$and": [{"source": source}, {"chunk_type": "text"}]}
|
where_filter = {"$and": [{"source": source}, {"chunk_type": "text"}]}
|
||||||
if section:
|
if section:
|
||||||
where_filter["$and"].append({"section": section})
|
where_filter["$and"].append({"section": section})
|
||||||
@@ -2037,7 +2037,104 @@ class RAGEngine:
|
|||||||
reranked[key] = results[key]
|
reranked[key] = results[key]
|
||||||
return reranked
|
return reranked
|
||||||
|
|
||||||
# ---------------- 流式生成 ----------------
|
# ---------------- 安全与工具 ----------------
|
||||||
|
|
||||||
|
def check_restricted_documents(self, query, allowed_levels, top_k=3, role=None, department=None):
|
||||||
|
if not self._initialized:
|
||||||
|
self.initialize()
|
||||||
|
|
||||||
|
if USE_MULTI_KB and self.kb_manager and role and department:
|
||||||
|
from auth.gateway import get_accessible_collections
|
||||||
|
all_colls = [c.name for c in self.kb_manager.list_collections()]
|
||||||
|
accessible = set(get_accessible_collections(role, department, 'read'))
|
||||||
|
restricted = set(all_colls) - accessible
|
||||||
|
|
||||||
|
if not restricted:
|
||||||
|
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": []}
|
||||||
|
|
||||||
|
query_vector = self.embedding_model.encode(query).tolist()
|
||||||
|
found_sources = set()
|
||||||
|
top_score = 0.0
|
||||||
|
|
||||||
|
for coll_name in restricted:
|
||||||
|
try:
|
||||||
|
coll = self.kb_manager.get_collection(coll_name)
|
||||||
|
if not coll: continue
|
||||||
|
res = coll.query(query_embeddings=[query_vector], n_results=top_k)
|
||||||
|
if res['metadatas'] and res['metadatas'][0]:
|
||||||
|
for meta in res['metadatas'][0]:
|
||||||
|
found_sources.add(meta.get('source', '未知'))
|
||||||
|
for dist in (res.get('distances', [[]])[0] or []):
|
||||||
|
if dist > top_score: top_score = dist
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"权限检查遍历失败: {e}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"has_restricted": len(found_sources) > 0,
|
||||||
|
"restricted_levels": [c.replace('dept_', '') for c in restricted if True][:3],
|
||||||
|
"restricted_sources": list(found_sources)[:3],
|
||||||
|
"top_restricted_score": top_score
|
||||||
|
}
|
||||||
|
|
||||||
|
if not allowed_levels:
|
||||||
|
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
|
||||||
|
|
||||||
|
restricted_levels = {"public", "internal", "confidential", "secret"} - set(allowed_levels)
|
||||||
|
if not restricted_levels:
|
||||||
|
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
|
||||||
|
|
||||||
|
query_vector = self.embedding_model.encode(query).tolist()
|
||||||
|
try:
|
||||||
|
res = self.collection.query(
|
||||||
|
query_embeddings=[query_vector],
|
||||||
|
n_results=top_k,
|
||||||
|
where={"security_level": {"$in": list(restricted_levels)}}
|
||||||
|
)
|
||||||
|
docs = res.get('documents', [[]])[0]
|
||||||
|
if not docs:
|
||||||
|
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
|
||||||
|
|
||||||
|
metas = res.get('metadatas', [[]])[0]
|
||||||
|
dists = res.get('distances', [[]])[0]
|
||||||
|
found_levels, found_sources, top_score = set(), set(), 0.0
|
||||||
|
|
||||||
|
for meta, dist in zip(metas, dists):
|
||||||
|
found_levels.add(meta.get('security_level', 'public'))
|
||||||
|
found_sources.add(meta.get('source', '未知'))
|
||||||
|
if dist > top_score: top_score = dist
|
||||||
|
|
||||||
|
return {
|
||||||
|
"has_restricted": True,
|
||||||
|
"restricted_levels": list(found_levels),
|
||||||
|
"restricted_sources": list(found_sources)[:3],
|
||||||
|
"top_restricted_score": top_score
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"受限内容检查失败: {e}")
|
||||||
|
return {"has_restricted": False, "restricted_levels": [], "restricted_sources": [], "top_restricted_score": 0.0}
|
||||||
|
|
||||||
|
def generate_answer(self, query, context):
|
||||||
|
"""底层生成答复能力"""
|
||||||
|
prompt = f"""你是一个严谨的智能助手,请根据以下参考资料回答用户的问题。
|
||||||
|
...
|
||||||
|
参考资料:
|
||||||
|
{context}
|
||||||
|
|
||||||
|
用户问题:{query}
|
||||||
|
|
||||||
|
请回答:"""
|
||||||
|
try:
|
||||||
|
from core.llm_utils import call_llm
|
||||||
|
result = call_llm(
|
||||||
|
self.llm_client,
|
||||||
|
prompt,
|
||||||
|
MODEL,
|
||||||
|
temperature=LLM_TEMPERATURE,
|
||||||
|
max_tokens=LLM_MAX_TOKENS
|
||||||
|
)
|
||||||
|
return result or f"调用大模型失败: 返回结果为空"
|
||||||
|
except Exception as e:
|
||||||
|
return f"调用大模型失败: {str(e)}"
|
||||||
|
|
||||||
def generate_answer_stream(self, query, context, history=None):
|
def generate_answer_stream(self, query, context, history=None):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -83,16 +83,9 @@ class IntentAnalyzer:
|
|||||||
根据对话历史和当前用户消息,输出一个 JSON 对象,包含以下字段:
|
根据对话历史和当前用户消息,输出一个 JSON 对象,包含以下字段:
|
||||||
|
|
||||||
1. **rewritten_query**: 改写后的完整问题
|
1. **rewritten_query**: 改写后的完整问题
|
||||||
- **指代消解**:如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
- 如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
||||||
- 例如:"分析一下这两张图片" → "分析一下对话历史中提到的图片"
|
- 例如:"分析一下这两张图片" → "分析一下对话历史中提到的图片"
|
||||||
- **追问补全**:如果问题是省略式追问(省略了上一轮讨论的主题实体),必须补全为完整问题
|
- 如果问题本身已经完整,直接返回原文
|
||||||
- 判断方法:当前问题缺少主语/宾语,且对话历史中可以推断出省略的实体
|
|
||||||
- 补全方法:从上一轮用户问题中提取主题实体,与追问组合成完整问题
|
|
||||||
- 例如:
|
|
||||||
- 上一轮问"吸烟点C1类是什么区?",追问"有完整表格吗?" → "吸烟点C1类有完整表格吗?"
|
|
||||||
- 上一轮问"三峡工程的投资情况",追问"建设地点在哪?" → "三峡工程的建设地点在哪?"
|
|
||||||
- 上一轮问"货源投放有哪些原则?",追问"具体内容是什么?" → "货源投放原则的具体内容是什么?"
|
|
||||||
- 如果问题本身已经完整且独立,直接返回原文
|
|
||||||
|
|
||||||
2. **use_context**: 布尔值
|
2. **use_context**: 布尔值
|
||||||
- true: 问题依赖历史对话中的信息,答案已经在历史回答中
|
- true: 问题依赖历史对话中的信息,答案已经在历史回答中
|
||||||
@@ -112,9 +105,7 @@ class IntentAnalyzer:
|
|||||||
- 推理类(intent="reasoning"):生成最多2个子查询
|
- 推理类(intent="reasoning"):生成最多2个子查询
|
||||||
* 原问题的检索查询
|
* 原问题的检索查询
|
||||||
* 一个补充角度的检索查询(如原因、背景、影响等),帮助获取更全面的上下文
|
* 一个补充角度的检索查询(如原因、背景、影响等),帮助获取更全面的上下文
|
||||||
- 其他类(factual/instruction/other):严格只生成1个子查询
|
- 其他类(factual/instruction/other):严格只生成1个子查询(原问题)
|
||||||
* 子查询应基于 rewritten_query(改写后的完整问题),而非用户原始输入
|
|
||||||
* 例如:追问"有完整表格吗?"改写为"吸烟点C1类有完整表格吗?"后,子查询应为"吸烟点C1类的完整表格内容"
|
|
||||||
- 不要为同一实体生成语义重叠的查询
|
- 不要为同一实体生成语义重叠的查询
|
||||||
- 子查询应保持原问题的关键词,长度20-60字符为宜
|
- 子查询应保持原问题的关键词,长度20-60字符为宜
|
||||||
|
|
||||||
@@ -316,8 +307,7 @@ class IntentAnalyzer:
|
|||||||
|
|
||||||
if cache_emb is not None:
|
if cache_emb is not None:
|
||||||
cached = cache.get(cache_emb)
|
cached = cache.get(cache_emb)
|
||||||
# 确保缓存条目是意图分析结果(非 RAG 回答缓存)
|
if cached:
|
||||||
if cached and cached.get("cache_type") != "rag_answer":
|
|
||||||
logger.info(f"意图分析缓存命中: {cached.get('reason', '')[:50]}")
|
logger.info(f"意图分析缓存命中: {cached.get('reason', '')[:50]}")
|
||||||
return IntentAnalysis.from_dict(cached)
|
return IntentAnalysis.from_dict(cached)
|
||||||
else:
|
else:
|
||||||
@@ -387,11 +377,9 @@ class IntentAnalyzer:
|
|||||||
intent=intent_type
|
intent=intent_type
|
||||||
)
|
)
|
||||||
|
|
||||||
# 存入语义缓存(标记类型,避免与 RAG 回答缓存混淆)
|
# 存入语义缓存
|
||||||
if cache and cache_emb is not None:
|
if cache and cache_emb is not None:
|
||||||
cache_data = analysis.to_dict()
|
cache.set(cache_emb, analysis.to_dict())
|
||||||
cache_data["cache_type"] = "intent_analysis"
|
|
||||||
cache.set(cache_emb, cache_data)
|
|
||||||
|
|
||||||
# 存入精确匹配缓存
|
# 存入精确匹配缓存
|
||||||
if len(self._exact_cache) < self._exact_cache_max:
|
if len(self._exact_cache) < self._exact_cache_max:
|
||||||
@@ -449,7 +437,6 @@ class IntentAnalyzer:
|
|||||||
if not history:
|
if not history:
|
||||||
return "(无历史对话)"
|
return "(无历史对话)"
|
||||||
|
|
||||||
import re
|
|
||||||
parts = []
|
parts = []
|
||||||
|
|
||||||
# 提取最近 3 轮对话
|
# 提取最近 3 轮对话
|
||||||
@@ -458,7 +445,6 @@ class IntentAnalyzer:
|
|||||||
for msg in recent_history:
|
for msg in recent_history:
|
||||||
role = "用户" if msg.get("role") == "user" else "助手"
|
role = "用户" if msg.get("role") == "user" else "助手"
|
||||||
content = msg.get("content", "")
|
content = msg.get("content", "")
|
||||||
original_content = content # 保留原始内容用于结构化提取
|
|
||||||
|
|
||||||
# 截断过长的内容
|
# 截断过长的内容
|
||||||
if len(content) > 500:
|
if len(content) > 500:
|
||||||
@@ -466,10 +452,9 @@ class IntentAnalyzer:
|
|||||||
|
|
||||||
parts.append(f"【{role}】{content}")
|
parts.append(f"【{role}】{content}")
|
||||||
|
|
||||||
# 提取结构化信息(从 assistant 消息中提取章节、表格、来源等)
|
# 提取图片信息
|
||||||
metadata = msg.get("metadata", {})
|
metadata = msg.get("metadata", {})
|
||||||
if isinstance(metadata, dict):
|
if isinstance(metadata, dict):
|
||||||
# 已有的图片提取
|
|
||||||
images = metadata.get("images", [])
|
images = metadata.get("images", [])
|
||||||
if images:
|
if images:
|
||||||
for img in images[:3]:
|
for img in images[:3]:
|
||||||
@@ -478,43 +463,6 @@ class IntentAnalyzer:
|
|||||||
img_type = img.get("type", "图片")
|
img_type = img.get("type", "图片")
|
||||||
parts.append(f" └─ {img_type}: {desc}")
|
parts.append(f" └─ {img_type}: {desc}")
|
||||||
|
|
||||||
# 来源文件提取
|
|
||||||
sources = metadata.get("sources", [])
|
|
||||||
if sources:
|
|
||||||
source_names = []
|
|
||||||
for s in sources[:3]:
|
|
||||||
if isinstance(s, dict):
|
|
||||||
name = s.get("source", "") or s.get("name", "")
|
|
||||||
if name:
|
|
||||||
source_names.append(name)
|
|
||||||
elif isinstance(s, str):
|
|
||||||
source_names.append(s)
|
|
||||||
if source_names:
|
|
||||||
parts.append(f" └─ 来源文件: {', '.join(source_names)}")
|
|
||||||
|
|
||||||
# collections(检索知识库)提取
|
|
||||||
colls = metadata.get("collections", [])
|
|
||||||
if colls:
|
|
||||||
parts.append(f" └─ 检索知识库: {', '.join(colls)}")
|
|
||||||
|
|
||||||
# 从 assistant 原始内容中提取章节路径和表格结构
|
|
||||||
if role == "助手" and original_content:
|
|
||||||
# 提取章节路径:━ xxx ━ 格式
|
|
||||||
sections = re.findall(r'━\s*(.+?)\s*━', original_content)
|
|
||||||
if sections:
|
|
||||||
unique_sections = list(dict.fromkeys(sections)) # 去重保序
|
|
||||||
parts.append(f" └─ 涉及章节: {'; '.join(unique_sections[:3])}")
|
|
||||||
|
|
||||||
# 提取表格列名:| A | B | C | 格式的表头行
|
|
||||||
table_headers = re.findall(r'^\|\s*(.+?)\s*\|', original_content, re.MULTILINE)
|
|
||||||
if table_headers:
|
|
||||||
# 取第一个表格的列名
|
|
||||||
first_header = table_headers[0]
|
|
||||||
cols = [c.strip() for c in first_header.split('|') if c.strip()]
|
|
||||||
# 排除分隔符行(--- 格式)
|
|
||||||
if cols and not all(re.match(r'^[-:]+$', c) for c in cols):
|
|
||||||
parts.append(f" └─ 含表格,列名: {', '.join(cols[:6])}")
|
|
||||||
|
|
||||||
# 添加图片上下文
|
# 添加图片上下文
|
||||||
if context_images:
|
if context_images:
|
||||||
parts.append("\n【上下文中的图片】")
|
parts.append("\n【上下文中的图片】")
|
||||||
|
|||||||
@@ -352,7 +352,7 @@ def quick_yes_no(
|
|||||||
if keywords is None:
|
if keywords is None:
|
||||||
keywords = ["是", "需要", "yes", "true"]
|
keywords = ["是", "需要", "yes", "true"]
|
||||||
|
|
||||||
result = call_llm(client, prompt, model, temperature=0, max_tokens=128)
|
result = call_llm(client, prompt, model, temperature=0, max_tokens=10)
|
||||||
if result is None:
|
if result is None:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|||||||
81
core/mmr.py
81
core/mmr.py
@@ -108,22 +108,44 @@ def mmr_rerank(
|
|||||||
return selected
|
return selected
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenize_words(text: str) -> set:
|
||||||
|
"""
|
||||||
|
使用 jieba 分词并过滤噪声,返回有意义的词集合。
|
||||||
|
|
||||||
|
过滤规则:
|
||||||
|
- 去除单字符词(如 "的", "了", "在")—— 这些是停用词,对区分文档无意义
|
||||||
|
- 去除纯数字 / 纯标点
|
||||||
|
- 保留 2 字及以上的实词
|
||||||
|
"""
|
||||||
|
import jieba
|
||||||
|
words = set()
|
||||||
|
for w in jieba.cut(text):
|
||||||
|
w = w.strip()
|
||||||
|
if len(w) >= 2 and not w.isdigit():
|
||||||
|
words.add(w)
|
||||||
|
return words
|
||||||
|
|
||||||
|
|
||||||
def mmr_filter_by_content(
|
def mmr_filter_by_content(
|
||||||
candidates: List[Dict],
|
candidates: List[Dict],
|
||||||
top_k: int = 30,
|
top_k: int = 30,
|
||||||
similarity_threshold: float = 0.9
|
similarity_threshold: float = 0.85
|
||||||
) -> List[Dict]:
|
) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
基于内容相似度的去重(简化版,不需要 embedding)
|
基于 jieba 词级 Jaccard 相似度的去重(不需要 embedding)
|
||||||
|
|
||||||
|
与旧版字符级 set(text) 的区别:
|
||||||
|
- 旧版:set("安全生产管理制度") → {'安','全','生','产',...},中文文档间字符集合高度重叠
|
||||||
|
- 新版:jieba 分词 → {"安全生产", "管理制度", ...},词级集合区分度高
|
||||||
|
|
||||||
适用于:
|
适用于:
|
||||||
- 没有 embedding 的情况
|
- MMR_USE_EMBEDDING=False 时的快速去重
|
||||||
- 快速去重场景
|
- 避免 CPU 编码 100+ 文档的 50 秒开销
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
candidates: 候选文档列表
|
candidates: 候选文档列表
|
||||||
top_k: 返回数量
|
top_k: 返回数量
|
||||||
similarity_threshold: 相似度阈值,超过则视为重复
|
similarity_threshold: 相似度阈值,超过则视为重复(默认 0.85)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
去重后的候选文档列表
|
去重后的候选文档列表
|
||||||
@@ -134,35 +156,42 @@ def mmr_filter_by_content(
|
|||||||
if len(candidates) <= top_k:
|
if len(candidates) <= top_k:
|
||||||
return candidates
|
return candidates
|
||||||
|
|
||||||
selected = []
|
# 预分词:对所有候选文档一次性分词,避免重复调用 jieba.cut
|
||||||
remaining = candidates.copy()
|
word_sets = []
|
||||||
|
for c in candidates:
|
||||||
|
content = c.get('content', c.get('document', ''))[:500]
|
||||||
|
word_sets.append(_tokenize_words(content))
|
||||||
|
|
||||||
while len(selected) < top_k and remaining:
|
selected_indices = []
|
||||||
current = remaining.pop(0)
|
|
||||||
|
for i in range(len(candidates)):
|
||||||
|
if len(selected_indices) >= top_k:
|
||||||
|
break
|
||||||
|
|
||||||
|
current_words = word_sets[i]
|
||||||
|
if not current_words:
|
||||||
|
# 空内容直接保留
|
||||||
|
selected_indices.append(i)
|
||||||
|
continue
|
||||||
|
|
||||||
# 检查是否与已选内容重复
|
|
||||||
is_duplicate = False
|
is_duplicate = False
|
||||||
current_content = current.get('content', current.get('document', ''))[:200]
|
for j in selected_indices:
|
||||||
|
selected_words = word_sets[j]
|
||||||
|
if not selected_words:
|
||||||
|
continue
|
||||||
|
|
||||||
for s in selected:
|
intersection = len(current_words & selected_words)
|
||||||
s_content = s.get('content', s.get('document', ''))[:200]
|
union = len(current_words | selected_words)
|
||||||
|
similarity = intersection / union if union > 0 else 0
|
||||||
|
|
||||||
# 简单的 Jaccard 相似度
|
if similarity > similarity_threshold:
|
||||||
words1 = set(current_content)
|
is_duplicate = True
|
||||||
words2 = set(s_content)
|
break
|
||||||
if words1 and words2:
|
|
||||||
intersection = len(words1 & words2)
|
|
||||||
union = len(words1 | words2)
|
|
||||||
similarity = intersection / union if union > 0 else 0
|
|
||||||
|
|
||||||
if similarity > similarity_threshold:
|
|
||||||
is_duplicate = True
|
|
||||||
break
|
|
||||||
|
|
||||||
if not is_duplicate:
|
if not is_duplicate:
|
||||||
selected.append(current)
|
selected_indices.append(i)
|
||||||
|
|
||||||
return selected
|
return [candidates[i] for i in selected_indices]
|
||||||
|
|
||||||
|
|
||||||
# ==================== 测试 ====================
|
# ==================== 测试 ====================
|
||||||
|
|||||||
@@ -1,30 +1,115 @@
|
|||||||
# RAG 系统完整指南
|
# Agentic RAG 完整指南
|
||||||
|
|
||||||
> **版本**: v4.0(统一编排 + 四层缓存修复)
|
> **版本**: v3.2(模型/Reranker/管线更新)
|
||||||
> **生产入口**: `api/chat_routes.py::rag()` → `generate()` → `core/engine.py`
|
> **生产入口**: `api/chat_routes.py::rag()` → `core/engine.py`(轻量编排,当前启用)
|
||||||
> **最后更新**: 2026-06-05
|
> **备用编排**: `core/agentic.py::AgenticRAG.process()` + 8 个 Mixin(完整决策循环,未接线)
|
||||||
|
> **最后更新**: 2026-06-04
|
||||||
>
|
>
|
||||||
> 本次更新:删除未使用的 AgenticRAG 备用编排路径(10 个文件 ~2050 行),修复 Query Cache 键不匹配与阈值问题,将语义缓存集成至生产 `/rag` 端点。
|
> ⚠️ 项目存在两套编排,生产 `/rag` 走的不是 `AgenticRAG`——详见下方「一·五、两套编排路径」。
|
||||||
|
|
||||||
## 一、功能概述
|
## 一、功能概述
|
||||||
|
|
||||||
本系统是一个检索增强生成(RAG)问答系统,采用**单一统一编排路径**,由 `api/chat_routes.py` 的 `generate()` 函数直接编排全流程。核心能力包括:
|
Agentic RAG 是一个智能问答系统,基于 Mixin 模式组合 8 个功能模块,具备以下核心能力:
|
||||||
|
|
||||||
| 功能 | 说明 | 实现位置 |
|
| 功能 | 说明 | Mixin 模块 |
|
||||||
|------|------|----------|
|
|------|------|-----------|
|
||||||
| **意图分析** | LLM 驱动的双层判断(是否需要检索)+ 查询改写 | `core/intent_analyzer.py` |
|
| **意图分析** | LLM 驱动的查询改写 + 双层判断(是否需要检索) | `IntentAnalyzer`(独立模块) |
|
||||||
| **混合检索** | 向量检索 + BM25 + RRF 融合 + Rerank 重排 | `core/engine.py` |
|
| **查询重写** | 口语化→专业术语、实体补全、指代消解 | `QueryRewriteMixin` |
|
||||||
| **四层缓存** | Query + Embedding + Rerank(LRU)+ 语义缓存(FAISS) | `core/cache.py` + `core/semantic_cache.py` |
|
| **混合检索** | 向量检索 + BM25 + RRF 融合 + Rerank 重排 | `SearchMixin` → `RAGEngine` |
|
||||||
| **流式生成** | SSE 流式答案输出,逐 token 推送 | `core/engine.py::generate_answer_stream()` |
|
| **多源融合** | 知识库 + 网络搜索,智能处理冲突 | `AnswerMixin` |
|
||||||
| **引用标注** | 自动标注信息来源和引用编号 | `api/chat_routes.py::_attach_citations()` |
|
| **幻觉验证** | 基于参考信息验证答案,防止 LLM 编造 | `AnswerMixin` |
|
||||||
| **富媒体** | 图片/表格的智能提取与展示 | `api/chat_routes.py` |
|
| **引用标注** | 自动标注信息来源和引用编号 | `CitationMixin` |
|
||||||
| **查询理解** | 查询分解、扩展、MMR 去重、自适应 TopK | `core/` 各独立模块 |
|
| **富媒体提取** | 图片/表格的智能提取与展示 | `RichMediaMixin` |
|
||||||
| **安全护栏** | 敏感信息过滤、Prompt 安全守卫 | `api/response_utils.py`、`core/prompt_guard.py` |
|
| **质量评估** | 多维度质量评估(相关性/完整性/准确性/覆盖面) | `QualityMixin` |
|
||||||
|
| **上下文压缩** | Rerank 阈值过滤 + Token 预算控制 | `ContextMixin` |
|
||||||
|
| **元问题处理** | 文件列表、权限查询等非知识类问题 | `MetaQuestionMixin` |
|
||||||
|
| **置信度门控** | 基于 Reranker 分数判断检索质量,低分触发补救 | `ConfidenceGate`(独立模块) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ⚠️ 一·五、两套编排路径(务必先读)
|
||||||
|
|
||||||
|
> **关键认知**:本项目存在**两套并存的编排(orchestration)**。生产 HTTP 接口 `/rag` 走的是**轻量编排**,而 `AgenticRAG.process()` 那套**完整决策循环目前处于备用状态、未接入任何 HTTP 路由**。
|
||||||
|
> 阅读下方所有架构图前请先理解这一点——下面 2.1 的「整体架构图」描绘的是**备用路径(AgenticRAG.process)**,不是当前生产实际跑的流程。
|
||||||
|
|
||||||
|
### 路径对比
|
||||||
|
|
||||||
|
| 维度 | 🟢 生产路径(当前启用) | 💤 备用路径(未接线) |
|
||||||
|
|------|----------------------|---------------------|
|
||||||
|
| 入口 | `api/chat_routes.py` → `rag()` → `generate()` | `core/agentic.py` → `AgenticRAG.process()` |
|
||||||
|
| 编排者 | `chat_routes` 自己的流程代码 | `AgenticRAG` 类(8 个 Mixin 组合) |
|
||||||
|
| 意图分析 | ✅ `intent_analyzer.analyze_intent()` | ✅ `IntentAnalyzer` / `QueryRewriteMixin` |
|
||||||
|
| 检索 | ✅ `search_hybrid()` → `engine.search_knowledge()` | ✅ `engine.search_knowledge()` |
|
||||||
|
| 查询分解/扩展/MMR/自适应TopK | ✅ 在 `engine` 内部执行 | ✅ 同左 |
|
||||||
|
| 答案生成 | ✅ `engine.generate_answer_stream()`(流式) | `AnswerMixin._generate_fused_answer()` |
|
||||||
|
| 引用标注 | ✅ `chat_routes._attach_citations()`(本地版) | `CitationMixin._attach_citations()` |
|
||||||
|
| 置信度门控 | ❌ 不调用 | `ConfidenceGate`(仅此路径用) |
|
||||||
|
| 多维质量评估 | ❌ 不调用 | `QualityMixin._assess_quality()` |
|
||||||
|
| 推理反思 | ❌ 不调用 | `QualityMixin._reflect_on_answer()` |
|
||||||
|
| 循环防护 | ❌ 不调用 | `LoopGuard`(仅此路径用) |
|
||||||
|
| 幻觉验证 | ❌ 不调用 | `AnswerMixin._verify_and_refine_answer()` |
|
||||||
|
|
||||||
|
### 重要结论
|
||||||
|
|
||||||
|
- **Agentic 的核心能力是活跃的**:意图分析+LLM改写、子查询拆分、查询扩展、自适应 TopK、MMR 去重、混合检索+Rerank——这些都在 `/rag` 中**真实运行**,只是由 `chat_routes` + `engine` 直接调用,而非通过 `AgenticRAG` 类。
|
||||||
|
- **休眠的只是「决策循环编排类」**:`AgenticRAG.process()` 及其独有组件(置信度门控 / 质量评估 / 推理反思 / 循环防护 / 幻觉验证)未接入 `/rag`。
|
||||||
|
- import 证据:`confidence_gate.py`、`quality_assessor.py`、`reasoning_reflector.py`、`loop_guard.py` 以及 8 个 `agentic_*` Mixin **只被 `core/agentic.py` import**;而 `AgenticRAG` 实例虽在 `api/__init__.py:90` 启动时创建,但其唯一读取入口 `_get_agentic_rag()` **零调用**。
|
||||||
|
- **这不是死代码可删**:`AgenticRAG` 在启动时被实例化(直接删会导致启动报错),且 `_extract_rich_media` 被 `scripts/test_rag_image_recall.py` 使用。它是「**一套更重、更完整、目前未启用的 Agentic 决策闭环**」,未来可选择接入。
|
||||||
|
|
||||||
|
### 🔬 如何验证「系统现在到底走哪套流程」
|
||||||
|
|
||||||
|
**方法 1:看开发环境 SSE 调试事件(最直接)**
|
||||||
|
|
||||||
|
`/rag` 在 `IS_DEV=True` 时会发出一串**只有 `chat_routes` 编排才会发**的调试事件,收到它们即证明走的是生产路径:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# UTF-8 payload 避免 Windows shell 编码问题
|
||||||
|
curl -s -N -X POST http://localhost:5001/rag \
|
||||||
|
-H "Content-Type: application/json; charset=utf-8" \
|
||||||
|
-H "Authorization: Bearer mock-token-admin" \
|
||||||
|
--data-binary @payload.json
|
||||||
|
```
|
||||||
|
|
||||||
|
观察 SSE 事件序列,**生产路径**会依次出现这些 `type`(`AgenticRAG.process` 不发这些):
|
||||||
|
|
||||||
|
| SSE 事件 `type` | 来源代码 | 含义 |
|
||||||
|
|----------------|---------|------|
|
||||||
|
| `start` | `chat_routes.py:1232` | 请求开始处理 |
|
||||||
|
| `intent_result` | `chat_routes.py:1228` | 意图分析结果(来自 `intent_analyzer`)[DEV] |
|
||||||
|
| `retrieval_debug` | `chat_routes.py:1311` | 检索管线各步骤(来自 `engine.search_knowledge` 的 `_debug`)[DEV] |
|
||||||
|
| `chunks_retrieved` | `chat_routes.py:1416` | 召回切片详情 [DEV] |
|
||||||
|
| `sources` | `chat_routes.py:1547` | 检索到的来源列表 |
|
||||||
|
| `images_selected` | `chat_routes.py:1574` | 图片选择详情 [DEV] |
|
||||||
|
| `context_built` | `chat_routes.py:1622` | 最终上下文构建 [DEV] |
|
||||||
|
| `chunk` | `chat_routes.py:1630` | 流式答案的每个 token |
|
||||||
|
| `finish` | `chat_routes.py:1699` | 含 `timing`、`sources`、`citations` |
|
||||||
|
| `error` | `chat_routes.py:1733` | 处理异常时的错误信息 |
|
||||||
|
|
||||||
|
> 标注 [DEV] 的事件仅在 `IS_DEV=True` 时发送,其余事件在生产环境也会发送。
|
||||||
|
|
||||||
|
**方法 2:看服务端日志**
|
||||||
|
|
||||||
|
- 启动时:出现一次 `Agentic RAG 引擎已初始化`(`api/__init__.py:95`,仅实例化,不代表被调用)。
|
||||||
|
- 每次 `/rag` 请求:出现 `[意图分析] use_context=... need_retrieval=...`(`chat_routes.py:1224`)。
|
||||||
|
- **不会**出现任何来自 `AgenticRAG.process()` 内部的日志(如查询重写 `📝 查询重写`、`🔍 知识库检索: N 条结果`)——若出现则说明走了备用路径。
|
||||||
|
|
||||||
|
**方法 3:埋点验证(最确定)**
|
||||||
|
|
||||||
|
临时在 `core/agentic.py` 的 `AgenticRAG.process()` 第一行加 `logger.warning("AgenticRAG.process CALLED")`,重启后发 `/rag` 请求——**该日志不会触发**,即证明生产不走 `AgenticRAG`。
|
||||||
|
|
||||||
|
**方法 4:静态确认调用链**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
grep -rn "_get_agentic_rag()" --include="*.py" . # 仅定义,无调用者 → AgenticRAG 实例未被请求使用
|
||||||
|
grep -rn "\.process(" --include="*.py" api/ # /rag、/chat 均无 .process() 调用
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 二、系统架构
|
## 二、系统架构
|
||||||
|
|
||||||
|
> ⚠️ 注意:下方 2.1「整体架构图」描绘的是**备用路径 `AgenticRAG.process()`** 的完整设计;当前生产 `/rag` 的实际流程见上方「一·五」及本节 2.3「生产 /rag 实际流程」。
|
||||||
|
|
||||||
### 2.1 整体架构图
|
### 2.1 整体架构图
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -33,12 +118,6 @@
|
|||||||
└────────────────────────────┬────────────────────────────────────────┘
|
└────────────────────────────┬────────────────────────────────────────┘
|
||||||
↓
|
↓
|
||||||
┌─────────────────────────────────────────────────────────────────────┐
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
│ 语义缓存检查 (SemanticCache - FAISS) │
|
|
||||||
│ cosine ≥ 0.92 → 命中则直接返回缓存结果 │
|
|
||||||
│ 跳过检索 + 生成全流程(~100ms vs ~9s) │
|
|
||||||
└────────────────────────────┬────────────────────────────────────────┘
|
|
||||||
↓ 未命中
|
|
||||||
┌─────────────────────────────────────────────────────────────────────┐
|
|
||||||
│ 意图分析 (IntentAnalyzer) │
|
│ 意图分析 (IntentAnalyzer) │
|
||||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||||
│ │ 改写查询 │ │ 双层判断 │ │ 子查询拆分 │ │
|
│ │ 改写查询 │ │ 双层判断 │ │ 子查询拆分 │ │
|
||||||
@@ -47,24 +126,26 @@
|
|||||||
└─────────┼──────────────────┼──────────────────┼─────────────────────┘
|
└─────────┼──────────────────┼──────────────────┼─────────────────────┘
|
||||||
↓ ↓ ↓
|
↓ ↓ ↓
|
||||||
┌──────────┐ ┌──────────────────────────────────────────┐
|
┌──────────┐ ┌──────────────────────────────────────────┐
|
||||||
│ 直接回答 │ │ 统一编排流程 │
|
│ 直接回答 │ │ AgenticRAG.process() │
|
||||||
│ (LLM) │ │ 1. 混合检索 (engine.search_knowledge) │
|
│ (LLM) │ │ 1. 元问题检查 │
|
||||||
└──────────┘ │ 2. 上下文提取 + 来源去重 │
|
└──────────┘ │ 2. 查询重写 (QueryRewriteMixin) │
|
||||||
│ 3. 图片补充检索 + 打分选择 │
|
│ 3. 知识库检索 (RAGEngine.search_knowledge)│
|
||||||
│ 4. 构建上下文 │
|
│ 4. 上下文压缩 (ContextMixin) │
|
||||||
│ 5. 流式答案生成 (engine.generate_stream) │
|
│ 5. 网络搜索 (SearchMixin, 可选) │
|
||||||
│ 6. 答案图号对齐 + 引用标注 │
|
│ 6. (图谱检索已废弃,graph/ 目录已清空) │
|
||||||
│ 7. 敏感信息过滤 │
|
│ 7. 融合答案生成 (AnswerMixin) │
|
||||||
│ 8. 语义缓存写入 + SSE finish │
|
│ 8. 幻觉验证 (AnswerMixin) │
|
||||||
|
│ 9. 富媒体提取 (RichMediaMixin) │
|
||||||
|
│ 10. 引用标注 (CitationMixin) │
|
||||||
└────────────────────┬─────────────────────┘
|
└────────────────────┬─────────────────────┘
|
||||||
↓
|
↓
|
||||||
┌─────────────────────────────────────────────────────────────────────┐
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
│ 检索层 (RAGEngine) │
|
│ 检索层 (RAGEngine) │
|
||||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||||
│ │ 查询缓存 │ │ 向量检索 │ │ BM25 检索 │ │
|
│ │ 向量检索 │ │ BM25 检索 │ │ FAQ 独立召回 │ │
|
||||||
│ │ (LRU 500) │ │ (语义匹配) │ │ (关键词匹配) │ │
|
│ │ (语义匹配) │ │ (关键词匹配) │ │ (精准命中) │ │
|
||||||
│ │ 命中直接返回│ └──────┬───────┘ └──────┬───────┘ │
|
│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │
|
||||||
│ └──────────────┘ └─────────────────┘ │
|
│ └─────────────────┼─────────────────┘ │
|
||||||
│ ↓ │
|
│ ↓ │
|
||||||
│ ┌──────────────┐ │
|
│ ┌──────────────┐ │
|
||||||
│ │ RRF 融合 │ ← 动态权重(查询类型/长度驱动)│
|
│ │ RRF 融合 │ ← 动态权重(查询类型/长度驱动)│
|
||||||
@@ -86,133 +167,78 @@
|
|||||||
┌─────────────────────────────────────────────────────────────────────┐
|
┌─────────────────────────────────────────────────────────────────────┐
|
||||||
│ 答案生成 (LLM 流式) │
|
│ 答案生成 (LLM 流式) │
|
||||||
│ ┌────────────────────────────────────────────────────────────────┐ │
|
│ ┌────────────────────────────────────────────────────────────────┐ │
|
||||||
│ │ 整合多源信息 + 标注来源 + 引用编号 + SSE 流式输出 │ │
|
│ │ 整合多源信息 + 标注来源 + 处理冲突 + 引用编号 + SSE 流式输出 │ │
|
||||||
│ └────────────────────────────────────────────────────────────────┘ │
|
│ └────────────────────────────────────────────────────────────────┘ │
|
||||||
└─────────────────────────────────────────────────────────────────────┘
|
└─────────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2.2 编排方式
|
### 2.2 Mixin 组合架构
|
||||||
|
|
||||||
系统采用**函数式编排**而非类组合模式。整个 RAG 流程由 `api/chat_routes.py` 的 `generate()` 函数直接控制,按步骤调用各独立模块:
|
|
||||||
|
|
||||||
- **意图分析**:`core/intent_analyzer.py` 的 `analyze_intent()`
|
|
||||||
- **检索管线**:`core/engine.py` 的 `search_knowledge()`
|
|
||||||
- **流式生成**:`core/engine.py` 的 `generate_answer_stream()`
|
|
||||||
- **引用标注**:`api/chat_routes.py` 内的 `_attach_citations()`
|
|
||||||
- **缓存系统**:`core/cache.py` 的 `RAGCacheManager` 单例 + `core/semantic_cache.py` 的 `SemanticCache` 单例
|
|
||||||
|
|
||||||
各模块通过 `get_engine()`、`get_cache_manager()`、`get_semantic_cache()` 等工厂函数获取全局单例实例。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 三、四层缓存架构
|
|
||||||
|
|
||||||
### 3.1 缓存层次概览
|
|
||||||
|
|
||||||
| 层次 | 缓存类型 | 存储结构 | 容量 | TTL | 作用 |
|
|
||||||
|------|----------|----------|------|-----|------|
|
|
||||||
| L1 | Query Cache | LRU (OrderedDict) | 500 条 | 1 小时 | 缓存完整问答结果,命中后跳过整个检索+生成 |
|
|
||||||
| L2 | Embedding Cache | LRU (OrderedDict) | 2000 条 | 24 小时 | 缓存向量化结果,避免重复调用 embedding 模型 |
|
|
||||||
| L3 | Rerank Cache | LRU (OrderedDict) | 1000 条 | 1 小时 | 缓存 Rerank 分数,避免重复调用 Reranker |
|
|
||||||
| L4 | Semantic Cache | FAISS IndexFlatIP | 10000 条 | 无过期 | 语义级缓存,相似查询也能命中 |
|
|
||||||
|
|
||||||
### 3.2 Query Cache
|
|
||||||
|
|
||||||
Query Cache 是最外层的完整问答结果缓存。命中后直接返回缓存的 `answer + sources + citations`,跳过检索和生成全流程。
|
|
||||||
|
|
||||||
**缓存键设计**:`{query_hash}:{kb_name}:{kb_version}`
|
|
||||||
|
|
||||||
- 基于查询文本哈希 + 知识库名称 + 知识库版本号
|
|
||||||
- 知识库版本变更时(如文档更新/重新索引),相关缓存自动失效
|
|
||||||
|
|
||||||
**已修复的问题**:
|
|
||||||
|
|
||||||
1. **键不匹配问题(已修复)**:此前 `set_query_result()` 在有 `doc_ids` 参数时使用 `doc_hash` 分支生成键,而 `get_query_result()` 始终使用 `kb_version` 分支——导致 GET 和 SET 的键永远不匹配,命中率始终为 0%。修复后两端统一使用 `kb_version` 分支。
|
|
||||||
|
|
||||||
2. **写入阈值问题(已修复)**:`CACHE_MIN_SCORE` 原值为 `0.3`,但 ChromaDB 余弦距离经 `1 - dist` 计算后得分通常在 0.03-0.06 之间,远低于阈值,导致几乎不写入缓存。修复后设为 `0.0`。
|
|
||||||
|
|
||||||
### 3.3 Embedding Cache
|
|
||||||
|
|
||||||
缓存文本向量化结果,由 `RAGEngine` 在调用 embedding 模型前后自动读写。键为查询文本哈希,避免对相同文本重复调用 embedding 模型(如 DashScope text-embedding-v3)。
|
|
||||||
|
|
||||||
### 3.4 Rerank Cache
|
|
||||||
|
|
||||||
缓存 Rerank 重排序的分数结果。键为 `query + sorted(doc_ids)` 的精确匹配。注意:由于 RRF 融合产出差异,命中率可能偏低。
|
|
||||||
|
|
||||||
### 3.5 语义缓存(Semantic Cache)
|
|
||||||
|
|
||||||
语义缓存基于 FAISS 向量索引实现语义级匹配——即使查询文字不完全相同,只要语义足够相似(cosine similarity ≥ 0.92),就能命中缓存。
|
|
||||||
|
|
||||||
**工作机制**:
|
|
||||||
|
|
||||||
```
|
|
||||||
用户查询 → embedding 编码 → FAISS 向量检索
|
|
||||||
→ cosine ≥ 0.92 → 命中:返回缓存的 answer + sources + citations(~100ms)
|
|
||||||
→ cosine < 0.92 → 未命中:执行完整 RAG 流程后写入缓存
|
|
||||||
```
|
|
||||||
|
|
||||||
**集成位置**:
|
|
||||||
|
|
||||||
- **读取**:在 `generate()` 函数的意图分析之后、混合检索之前(跳过整个检索+生成流程)
|
|
||||||
- **写入**:在 `generate()` 函数生成完整答案后、发送 `finish` 事件之前
|
|
||||||
- **缓存内容**:answer、sources、citations、images、tables
|
|
||||||
|
|
||||||
**验证结果**:语义缓存命中率约 66.7%,平均响应从 ~9.2 秒降至 ~100 毫秒(约 92 倍加速)。
|
|
||||||
|
|
||||||
### 3.6 缓存失效机制
|
|
||||||
|
|
||||||
所有基于 LRU 的缓存(L1-L3)均支持基于知识库版本号(`kb_version`)的自动失效:
|
|
||||||
|
|
||||||
- 每个缓存条目关联 `kb_version`
|
|
||||||
- 知识库文档变更时 `kb_version` 递增
|
|
||||||
- 读取时检查 `kb_version` 是否匹配,不匹配则视为过期
|
|
||||||
|
|
||||||
语义缓存(L4)当前无 TTL 过期机制,仅受 `max_size=10000` 容量限制。
|
|
||||||
|
|
||||||
### 3.7 缓存配置
|
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# config.py / config.example.py
|
class AgenticRAG(
|
||||||
|
QueryRewriteMixin, # 查询重写:口语化→专业术语、实体补全
|
||||||
# 查询结果缓存
|
SearchMixin, # 检索功能:网络搜索
|
||||||
QUERY_CACHE_ENABLED = True
|
AnswerMixin, # 答案生成:融合回答、幻觉验证
|
||||||
QUERY_CACHE_SIZE = 500
|
CitationMixin, # 引用处理:来源标注、引用编号
|
||||||
QUERY_CACHE_TTL = 3600 # 秒
|
RichMediaMixin, # 富媒体:图片/表格提取
|
||||||
|
QualityMixin, # 质量评估:多维评估
|
||||||
# Embedding 缓存
|
ContextMixin, # 上下文处理:压缩、过滤
|
||||||
EMBEDDING_CACHE_ENABLED = True
|
MetaQuestionMixin # 元问题:文件列表、权限查询
|
||||||
EMBEDDING_CACHE_SIZE = 2000
|
):
|
||||||
EMBEDDING_CACHE_TTL = 86400 # 24小时
|
...
|
||||||
|
|
||||||
# Rerank 缓存
|
|
||||||
RERANK_CACHE_ENABLED = True
|
|
||||||
RERANK_CACHE_SIZE = 1000
|
|
||||||
RERANK_CACHE_TTL = 3600
|
|
||||||
|
|
||||||
# 语义缓存
|
|
||||||
SEMANTIC_CACHE_ENABLED = True
|
|
||||||
SEMANTIC_CACHE_THRESHOLD = 0.92 # cosine 相似度阈值
|
|
||||||
|
|
||||||
# 缓存写入最低置信度
|
|
||||||
CACHE_MIN_SCORE = 0.0 # ChromaDB 余弦距离经 1-dist 后得分约 0.03-0.06,须设为 0
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3.8 部署注意事项
|
> 注:以上 2.1 / 2.2 是 `AgenticRAG`(备用路径)的设计。**当前生产 `/rag` 不实例化走这条链**,实际流程见下方 2.3。
|
||||||
|
|
||||||
当前所有缓存均为**进程内内存存储**(LRU 使用 `OrderedDict`,语义缓存使用 FAISS 内存索引),有以下部署影响:
|
### 2.3 生产 /rag 实际流程(当前启用)
|
||||||
|
|
||||||
- **单 Worker**:Gunicorn 默认 1 个 worker,所有请求共享同一缓存实例,缓存有效
|
入口 `api/chat_routes.py::rag() → generate()`,**不经过 `AgenticRAG`**:
|
||||||
- **多 Worker**:每个 worker 有独立缓存,不共享,缓存效率降低
|
|
||||||
- **冷启动**:进程重启后缓存全部丢失,需重新预热
|
|
||||||
- **`max_requests=1000`**:Gunicorn worker 定期重启会导致缓存周期性清空
|
|
||||||
|
|
||||||
对于生产环境多实例部署场景,已规划 Redis 外部缓存迁移方案(见 `reports/redis_migration_plan.md`)。
|
```
|
||||||
|
POST /rag (SSE 流式)
|
||||||
|
↓
|
||||||
|
[chat_routes.generate()] ← 轻量编排,不实例化 AgenticRAG
|
||||||
|
│
|
||||||
|
├─ 发 SSE: start
|
||||||
|
│
|
||||||
|
├─ 1. 意图分析 intent_analyzer.analyze_intent() # chat_routes:1222
|
||||||
|
│ ├─ need_retrieval=False → 直接 LLM 回答(流式发 SSE: chunk),结束
|
||||||
|
│ │ └─ use_context=True 时带历史上下文,use_context=False 时纯闲聊
|
||||||
|
│ └─ 否则继续;sub_queries 传入检索
|
||||||
|
│ └─[DEV] 发 SSE: intent_result
|
||||||
|
│
|
||||||
|
├─ 2. 混合检索 search_hybrid() → engine.search_knowledge() # chat_routes:1300
|
||||||
|
│ (内部:向量+BM25+RRF+废止过滤+章节过滤
|
||||||
|
│ +云端Rerank+MMR去重+FAQ加权+黑名单+时间衰减
|
||||||
|
│ +上下文扩展+自适应TopK)
|
||||||
|
│ └─[DEV] 发 SSE: retrieval_debug
|
||||||
|
│
|
||||||
|
├─ 3. 提取上下文/来源(按 source 去重,doc_type 驱动溯源展示) # chat_routes:1362
|
||||||
|
│ └─[DEV] 发 SSE: chunks_retrieved
|
||||||
|
│ └─ 发 SSE: sources
|
||||||
|
│
|
||||||
|
├─ 4. 图片补充检索 + 图片打分选择 (select_images)
|
||||||
|
│ └─[DEV] 发 SSE: images_selected
|
||||||
|
├─ 5. 构建上下文 (_order_text_contexts_for_prompt)
|
||||||
|
│ └─[DEV] 发 SSE: context_built
|
||||||
|
│
|
||||||
|
├─ 6. 流式答案生成 engine.generate_answer_stream() # chat_routes:1628
|
||||||
|
│ └─ 逐 token 发 SSE: chunk
|
||||||
|
│
|
||||||
|
├─ 7. 答案图号对齐过滤
|
||||||
|
├─ 8. 引用标注 chat_routes._attach_citations()(本地版,非 CitationMixin) # chat_routes:1668
|
||||||
|
├─ 9. 敏感信息过滤 filter_response()
|
||||||
|
├─ 10. 发 SSE: finish(answer + sources + citations + images + timing)
|
||||||
|
└─[异常] 发 SSE: error
|
||||||
|
```
|
||||||
|
|
||||||
|
**与备用路径(AgenticRAG.process)的差异**:生产路径**没有**置信度门控、多维质量评估、推理反思、循环防护、幻觉验证这几步——它们只存在于 `AgenticRAG.process()`。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 四、意图分析流程
|
## 三、意图分析流程
|
||||||
|
|
||||||
### 4.1 IntentAnalyzer 双层判断
|
### 3.1 IntentAnalyzer 双层判断
|
||||||
|
|
||||||
意图分析由 `core/intent_analyzer.py` 的 `IntentAnalyzer` 类完成,采用 **LLM 驱动** 的双层判断:
|
意图分析由 `core/intent_analyzer.py` 的 `IntentAnalyzer` 类完成,采用 **LLM 驱动** 的双层判断:
|
||||||
|
|
||||||
@@ -243,7 +269,7 @@ CACHE_MIN_SCORE = 0.0 # ChromaDB 余弦距离经 1-dist 后得分
|
|||||||
- intent: factual/comparison/reasoning/instruction/other
|
- intent: factual/comparison/reasoning/instruction/other
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4.2 QueryClassifier 规则分类
|
### 3.2 QueryClassifier 规则分类
|
||||||
|
|
||||||
`core/query_classifier.py` 提供无 LLM 调用的快速规则分类:
|
`core/query_classifier.py` 提供无 LLM 调用的快速规则分类:
|
||||||
|
|
||||||
@@ -260,9 +286,9 @@ CACHE_MIN_SCORE = 0.0 # ChromaDB 余弦距离经 1-dist 后得分
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 五、检索管线详解
|
## 四、检索管线详解
|
||||||
|
|
||||||
### 5.1 完整检索流程
|
### 4.1 完整检索流程
|
||||||
|
|
||||||
```
|
```
|
||||||
search_knowledge(query, top_k=30)
|
search_knowledge(query, top_k=30)
|
||||||
@@ -309,7 +335,7 @@ search_knowledge(query, top_k=30)
|
|||||||
└─ 15. 缓存写入 → 返回结果
|
└─ 15. 缓存写入 → 返回结果
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5.2 混合检索代码示例
|
### 4.2 混合检索代码示例
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# 向量检索(语义相似)
|
# 向量检索(语义相似)
|
||||||
@@ -322,10 +348,7 @@ bm25_results = bm25_index.search(query, top_k=recall_k)
|
|||||||
faq_results = faq_collection.query(query_embeddings=[query_vector], n_results=3)
|
faq_results = faq_collection.query(query_embeddings=[query_vector], n_results=3)
|
||||||
|
|
||||||
# 图片独立召回(P0 通道)
|
# 图片独立召回(P0 通道)
|
||||||
image_results = collection.query(
|
image_results = collection.query(query_embeddings=[query_vector], n_results=5, where={"chunk_type": {"$in": ["image", "chart", "table"]}})
|
||||||
query_embeddings=[query_vector], n_results=5,
|
|
||||||
where={"chunk_type": {"$in": ["image", "chart", "table"]}}
|
|
||||||
)
|
|
||||||
|
|
||||||
# RRF 融合(动态权重)
|
# RRF 融合(动态权重)
|
||||||
fused = reciprocal_rank_fusion([vector_results, bm25_results], weights=[vector_w, bm25_w])
|
fused = reciprocal_rank_fusion([vector_results, bm25_results], weights=[vector_w, bm25_w])
|
||||||
@@ -337,7 +360,7 @@ reranked = rerank_results(query, fused, top_k=15)
|
|||||||
mmr_results = mmr_rerank(query_emb, reranked, top_k=30, lambda_param=0.5)
|
mmr_results = mmr_rerank(query_emb, reranked, top_k=30, lambda_param=0.5)
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5.3 RRF 融合算法
|
### 4.3 RRF 融合算法
|
||||||
|
|
||||||
```
|
```
|
||||||
RRF分数 = Σ (权重 / (k + 排名位置))
|
RRF分数 = Σ (权重 / (k + 排名位置))
|
||||||
@@ -353,9 +376,9 @@ RRF分数 = Σ (权重 / (k + 排名位置))
|
|||||||
- 查询类型驱动: FACT→BM25优先, PROCESS→向量优先
|
- 查询类型驱动: FACT→BM25优先, PROCESS→向量优先
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5.4 Rerank 重排
|
### 4.4 Rerank 重排
|
||||||
|
|
||||||
**后端**:支持三种模式,由 `RERANK_BACKEND` 环境变量控制
|
**后端**: 支持三种模式,由 `RERANK_BACKEND` 环境变量控制
|
||||||
|
|
||||||
| RERANK_BACKEND | 说明 |
|
| RERANK_BACKEND | 说明 |
|
||||||
|----------------|------|
|
|----------------|------|
|
||||||
@@ -363,20 +386,20 @@ RRF分数 = Σ (权重 / (k + 排名位置))
|
|||||||
| `"local"` | 仅使用本地 `BAAI/bge-reranker-base`(CrossEncoder / ONNX) |
|
| `"local"` | 仅使用本地 `BAAI/bge-reranker-base`(CrossEncoder / ONNX) |
|
||||||
| `"fallback"` | 优先云端,失败时自动回退本地(推荐生产环境) |
|
| `"fallback"` | 优先云端,失败时自动回退本地(推荐生产环境) |
|
||||||
|
|
||||||
**云端 Reranker(推荐)**:
|
**云端 Reranker(推荐)**:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# config.py
|
# config.py
|
||||||
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "local")
|
RERANK_BACKEND = os.getenv("RERANK_BACKEND", "local") # local / cloud / fallback
|
||||||
RERANK_CLOUD_MODEL = "qwen3-rerank"
|
RERANK_CLOUD_MODEL = "qwen3-rerank" # DashScope 云端 Rerank 模型
|
||||||
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", DASHSCOPE_API_KEY)
|
RERANK_CLOUD_API_KEY = os.getenv("RERANK_CLOUD_API_KEY", DASHSCOPE_API_KEY)
|
||||||
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
RERANK_CLOUD_BASE_URL = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks"
|
||||||
RERANK_CLOUD_TIMEOUT = 15
|
RERANK_CLOUD_TIMEOUT = 15 # 云端请求超时(秒)
|
||||||
```
|
```
|
||||||
|
|
||||||
`CloudReranker` 类(`core/engine.py`)封装 DashScope 的 `/compatible-api/v1/reranks` 接口,提供与本地 `CrossEncoder.predict()` / `ONNXReranker.predict()` 一致的调用接口。
|
`CloudReranker` 类(`core/engine.py`)封装 DashScope 的 `/compatible-api/v1/reranks` 接口,提供与本地 `CrossEncoder.predict()` / `ONNXReranker.predict()` 一致的调用接口。
|
||||||
|
|
||||||
**本地 Reranker(备选)**:
|
**本地 Reranker(备选)**:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
def rerank_results(self, query, results, top_k=5):
|
def rerank_results(self, query, results, top_k=5):
|
||||||
@@ -386,73 +409,69 @@ def rerank_results(self, query, results, top_k=5):
|
|||||||
# 返回 top_k 个最高分结果
|
# 返回 top_k 个最高分结果
|
||||||
```
|
```
|
||||||
|
|
||||||
**调用位置**:`core/engine.py` 的 `search_knowledge()` 和 `_search_multi_kb()` 中,RRF 融合 + 废止/章节过滤之后、MMR 去重之前执行。
|
**调用位置**: `core/engine.py` 的 `search_knowledge()` 和 `_search_multi_kb()` 中,RRF 融合 + 废止/章节过滤之后、MMR 去重之前执行。
|
||||||
|
|
||||||
|
**引擎初始化顺序**: `RAGEngine.__init__()` 中按 `RERANK_BACKEND` 决定加载策略:
|
||||||
|
- `cloud` / `fallback`:先尝试创建 `CloudReranker`,需要 `RERANK_CLOUD_API_KEY`
|
||||||
|
- `local` / `fallback`(云端失败时):加载本地 `BAAI/bge-reranker-base`,支持 ONNX 加速
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 六、生产 /rag 完整流程
|
## 五、置信度门控
|
||||||
|
|
||||||
入口 `api/chat_routes.py::rag() → generate()`:
|
`core/confidence_gate.py` 基于 Reranker 分数判断检索结果质量:
|
||||||
|
|
||||||
```
|
```
|
||||||
POST /rag (SSE 流式)
|
检索结果 → Reranker 计算置信度 → 阈值判断 → 决策
|
||||||
↓
|
│
|
||||||
[chat_routes.generate()]
|
┌─────────────────┼─────────────────┐
|
||||||
│
|
↓ ↓ ↓
|
||||||
├─ 发 SSE: start
|
PASS (≥0.4) REWRITE (0.2~0.4) WEB_SEARCH (<0.2)
|
||||||
│
|
继续生成 查询重写 网络搜索补救
|
||||||
├─ 1. 语义缓存检查 SemanticCache.get() # chat_routes
|
|
||||||
│ ├─ 命中 → 直接流式发 SSE: chunk + finish,结束(~100ms)
|
|
||||||
│ └─ 未命中 → 继续;记录 embedding 供后续写入
|
|
||||||
│
|
|
||||||
├─ 2. 意图分析 intent_analyzer.analyze_intent()
|
|
||||||
│ ├─ need_retrieval=False → 直接 LLM 回答(流式发 SSE: chunk),结束
|
|
||||||
│ │ └─ use_context=True 时带历史上下文,use_context=False 时纯闲聊
|
|
||||||
│ └─ 否则继续;sub_queries 传入检索
|
|
||||||
│ └─[DEV] 发 SSE: intent_result
|
|
||||||
│
|
|
||||||
├─ 3. 混合检索 search_hybrid() → engine.search_knowledge()
|
|
||||||
│ (内部:查询缓存检查 → 向量+BM25+RRF+废止过滤+章节过滤
|
|
||||||
│ +云端Rerank+MMR去重+FAQ加权+黑名单+时间衰减
|
|
||||||
│ +上下文扩展+自适应TopK)
|
|
||||||
│ └─[DEV] 发 SSE: retrieval_debug
|
|
||||||
│
|
|
||||||
├─ 4. 提取上下文/来源(按 source 去重,doc_type 驱动溯源展示)
|
|
||||||
│ └─[DEV] 发 SSE: chunks_retrieved
|
|
||||||
│ └─ 发 SSE: sources
|
|
||||||
│
|
|
||||||
├─ 5. 图片补充检索 + 图片打分选择 (select_images)
|
|
||||||
│ └─[DEV] 发 SSE: images_selected
|
|
||||||
├─ 6. 构建上下文 (_order_texts_for_prompt)
|
|
||||||
│ └─[DEV] 发 SSE: context_built
|
|
||||||
│
|
|
||||||
├─ 7. 流式答案生成 engine.generate_answer_stream()
|
|
||||||
│ └─ 逐 token 发 SSE: chunk
|
|
||||||
│
|
|
||||||
├─ 8. 答案图号对齐过滤
|
|
||||||
├─ 9. 引用标注 _attach_citations()
|
|
||||||
├─ 10. 敏感信息过滤 filter_response()
|
|
||||||
├─ 11. 语义缓存写入 SemanticCache.set() # 写入缓存供后续命中
|
|
||||||
├─ 12. 发 SSE: finish(answer + sources + citations + images + timing)
|
|
||||||
└─[异常] 发 SSE: error
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### SSE 事件序列
|
**阈值配置**:
|
||||||
|
- `PASS_THRESHOLD = 0.2`: 通过阈值(低于此值需要补救)
|
||||||
|
- `GOOD_THRESHOLD = 0.4`: 良好阈值(高质量结果)
|
||||||
|
- `EXCELLENT_THRESHOLD = 0.7`: 优秀阈值
|
||||||
|
|
||||||
| SSE 事件 `type` | 含义 |
|
---
|
||||||
|----------------|------|
|
|
||||||
| `start` | 请求开始处理 |
|
|
||||||
| `intent_result` | 意图分析结果 [DEV] |
|
|
||||||
| `retrieval_debug` | 检索管线各步骤 [DEV] |
|
|
||||||
| `chunks_retrieved` | 召回切片详情 [DEV] |
|
|
||||||
| `sources` | 检索到的来源列表 |
|
|
||||||
| `images_selected` | 图片选择详情 [DEV] |
|
|
||||||
| `context_built` | 最终上下文构建 [DEV] |
|
|
||||||
| `chunk` | 流式答案的每个 token |
|
|
||||||
| `finish` | 含 `timing`、`sources`、`citations`、`images` |
|
|
||||||
| `error` | 处理异常时的错误信息 |
|
|
||||||
|
|
||||||
> 标注 [DEV] 的事件仅在 `IS_DEV=True` 时发送。
|
## 六、AgenticRAG 主流程
|
||||||
|
|
||||||
|
### 6.1 process() 方法
|
||||||
|
|
||||||
|
```python
|
||||||
|
def process(self, query, verbose=True, history=None,
|
||||||
|
allowed_levels=None, role=None, department=None,
|
||||||
|
emit_log=None) -> dict:
|
||||||
|
"""
|
||||||
|
返回:
|
||||||
|
{
|
||||||
|
"answer": str, # 最终答案
|
||||||
|
"sources": list, # 来源列表
|
||||||
|
"images": list, # 图片列表
|
||||||
|
"tables": list, # 表格列表
|
||||||
|
"citations": list, # 引用列表
|
||||||
|
"log_trace": list # 推理过程追踪
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6.2 流程步骤
|
||||||
|
|
||||||
|
| 步骤 | 方法 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| 1 | `_is_meta_question()` | 检查元问题(文件列表、权限等) |
|
||||||
|
| 2 | `should_rewrite()` + `_rewrite_query()` | 查询重写(口语化→专业术语、实体补全) |
|
||||||
|
| 3 | `engine.search_knowledge()` / `engine.search_multiple()` | 知识库检索(含向量+BM25+RRF+MMR+Rerank) |
|
||||||
|
| 4 | `_compress_contexts()` | 上下文压缩(Rerank 阈值过滤) |
|
||||||
|
| 5 | `_web_search_flow()` | 网络搜索(可选,需 SERPER_API_KEY) |
|
||||||
|
| 6 | ~~`_graph_search()`~~ | ~~图谱检索(已废弃,graph/ 目录已清空)~~ |
|
||||||
|
| 7 | `_generate_fused_answer()` | 融合答案生成(多源信息+冲突处理) |
|
||||||
|
| 8 | `_verify_and_refine_answer()` | 幻觉验证(防止 LLM 编造) |
|
||||||
|
| 9 | `_extract_rich_media()` | 富媒体提取(图片/表格) |
|
||||||
|
| 10 | `_attach_citations()` | 引用标注 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -470,7 +489,7 @@ curl -X POST http://localhost:5001/rag \
|
|||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
**响应格式**:SSE(Server-Sent Events)流式返回
|
**响应格式**: SSE(Server-Sent Events)流式返回
|
||||||
|
|
||||||
```
|
```
|
||||||
event: token
|
event: token
|
||||||
@@ -482,7 +501,7 @@ data: {"text": "规定"}
|
|||||||
...
|
...
|
||||||
|
|
||||||
event: finish
|
event: finish
|
||||||
data: {"answer": "完整答案", "sources": [...], "citations": [...], "images": [...], "duration_ms": 200}
|
data: {"answer": "完整答案", "sources": [...], "citations": [...], "images": [...], "duration_ms": 3200}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 7.2 代码调用
|
### 7.2 代码调用
|
||||||
@@ -501,27 +520,20 @@ for token in engine.generate_answer_stream(query, context, history=history):
|
|||||||
print(token, end="", flush=True)
|
print(token, end="", flush=True)
|
||||||
```
|
```
|
||||||
|
|
||||||
### 7.3 缓存统计查询
|
### 7.3 AgenticRAG 调用
|
||||||
|
|
||||||
```bash
|
```python
|
||||||
# 查看各层缓存命中率和统计
|
from core.agentic import AgenticRAG
|
||||||
curl http://localhost:5001/cache/stats \
|
|
||||||
-H "Authorization: Bearer mock-token-admin"
|
rag = AgenticRAG(max_iterations=3, enable_web_search=True)
|
||||||
|
result = rag.process("出差补助标准是什么?")
|
||||||
|
|
||||||
|
print(f"答案: {result['answer']}")
|
||||||
|
print(f"来源: {result['sources']}")
|
||||||
|
print(f"图片: {result['images']}")
|
||||||
|
print(f"引用: {result['citations']}")
|
||||||
```
|
```
|
||||||
|
|
||||||
返回示例:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"query_cache": {"total_entries": 50, "hits": 10, "misses": 6, "hit_rate": 0.625},
|
|
||||||
"embedding_cache": {"total_entries": 200, "hits": 0, "misses": 0, "hit_rate": 0},
|
|
||||||
"rerank_cache": {"total_entries": 100, "hits": 0, "misses": 0, "hit_rate": 0},
|
|
||||||
"semantic_cache": {"total_entries": 15, "hits": 6, "misses": 3, "hit_rate": 0.667}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
> 注:当 Query Cache 或 Semantic Cache 在外层拦截了重复查询时,Embedding Cache 和 Rerank Cache 的命中率为 0 是正常现象——重复查询根本不会到达这些层。
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 八、配置说明
|
## 八、配置说明
|
||||||
@@ -553,10 +565,10 @@ RECALL_MULTIPLIER = 3 # 候选池最小倍数
|
|||||||
# 重排序
|
# 重排序
|
||||||
USE_RERANK = True # 启用重排序
|
USE_RERANK = True # 启用重排序
|
||||||
RERANK_BACKEND = "local" # "local"=本地模型, "cloud"=云端API, "fallback"=优先云端失败回退本地
|
RERANK_BACKEND = "local" # "local"=本地模型, "cloud"=云端API, "fallback"=优先云端失败回退本地
|
||||||
RERANK_CLOUD_MODEL = "qwen3-rerank"
|
RERANK_CLOUD_MODEL = "qwen3-rerank" # 云端 Rerank 模型(DashScope API)
|
||||||
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
RERANK_CANDIDATES = 20 # 送入重排序的候选数
|
||||||
RERANK_TOP_K = 15 # 重排序后保留数
|
RERANK_TOP_K = 15 # 重排序后保留数
|
||||||
RERANK_USE_ONNX = True # ONNX 加速(仅本地模式,环境变量控制)
|
RERANK_USE_ONNX = True # ONNX 加速(仅本地模式,环境变量控制,默认开启)
|
||||||
|
|
||||||
# RRF 融合
|
# RRF 融合
|
||||||
RRF_K = 60 # RRF 常数
|
RRF_K = 60 # RRF 常数
|
||||||
@@ -569,7 +581,30 @@ MMR_TOP_K = 30 # MMR 保留数
|
|||||||
MMR_LAMBDA = 0.5 # 相关性 vs 多样性权衡
|
MMR_LAMBDA = 0.5 # 相关性 vs 多样性权衡
|
||||||
```
|
```
|
||||||
|
|
||||||
### 8.3 设备配置
|
### 8.3 缓存配置
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 查询结果缓存
|
||||||
|
QUERY_CACHE_ENABLED = True
|
||||||
|
QUERY_CACHE_SIZE = 500
|
||||||
|
QUERY_CACHE_TTL = 3600 # 1小时
|
||||||
|
|
||||||
|
# Embedding 缓存
|
||||||
|
EMBEDDING_CACHE_ENABLED = True
|
||||||
|
EMBEDDING_CACHE_SIZE = 2000
|
||||||
|
EMBEDDING_CACHE_TTL = 86400 # 24小时
|
||||||
|
|
||||||
|
# Rerank 缓存
|
||||||
|
RERANK_CACHE_ENABLED = True
|
||||||
|
RERANK_CACHE_SIZE = 1000
|
||||||
|
RERANK_CACHE_TTL = 3600 # 1小时
|
||||||
|
|
||||||
|
# 语义缓存
|
||||||
|
SEMANTIC_CACHE_ENABLED = True
|
||||||
|
SEMANTIC_CACHE_THRESHOLD = 0.92 # 相似度阈值
|
||||||
|
```
|
||||||
|
|
||||||
|
### 8.4 设备配置
|
||||||
|
|
||||||
```python
|
```python
|
||||||
DEVICE = "auto" # auto / cuda / cpu / cuda:0
|
DEVICE = "auto" # auto / cuda / cpu / cuda:0
|
||||||
@@ -583,9 +618,17 @@ RERANK_DEVICE = DEVICE # Rerank 模型设备
|
|||||||
|
|
||||||
```
|
```
|
||||||
core/ # RAG 核心引擎
|
core/ # RAG 核心引擎
|
||||||
├── engine.py # RAGEngine 单例(检索主流程、Rerank、RRF、流式生成)
|
├── engine.py # RAGEngine 单例(检索主流程、Rerank、RRF)
|
||||||
├── cache.py # 三层 LRU 缓存管理器(Query/Embedding/Rerank)
|
├── agentic.py # AgenticRAG 主类(Mixin 组合)
|
||||||
├── semantic_cache.py # 语义缓存(FAISS IndexFlatIP)
|
├── agentic_base.py # 基础常量与条件导入
|
||||||
|
├── agentic_query.py # QueryRewriteMixin(查询重写)
|
||||||
|
├── agentic_search.py # SearchMixin(网络搜索)
|
||||||
|
├── agentic_answer.py # AnswerMixin(答案生成、幻觉验证)
|
||||||
|
├── agentic_citation.py # CitationMixin(引用标注)
|
||||||
|
├── agentic_media.py # RichMediaMixin(富媒体提取)
|
||||||
|
├── agentic_quality.py # QualityMixin(质量评估)
|
||||||
|
├── agentic_context.py # ContextMixin(上下文压缩)
|
||||||
|
├── agentic_meta.py # MetaQuestionMixin(元问题处理)
|
||||||
├── bm25_index.py # BM25Index(关键词检索)
|
├── bm25_index.py # BM25Index(关键词检索)
|
||||||
├── chunker.py # 文本分块器
|
├── chunker.py # 文本分块器
|
||||||
├── mmr.py # MMR 去重(语义向量版 + 文本 Jaccard 版)
|
├── mmr.py # MMR 去重(语义向量版 + 文本 Jaccard 版)
|
||||||
@@ -594,13 +637,14 @@ core/ # RAG 核心引擎
|
|||||||
├── query_decomposer.py # QueryDecomposer(复杂查询拆分)
|
├── query_decomposer.py # QueryDecomposer(复杂查询拆分)
|
||||||
├── query_expansion.py # 查询扩展
|
├── query_expansion.py # 查询扩展
|
||||||
├── adaptive_topk.py # AdaptiveTopK(自适应 TopK)
|
├── adaptive_topk.py # AdaptiveTopK(自适应 TopK)
|
||||||
├── confidence_gate.py # ConfidenceGate(置信度门控,当前未接入生产流程)
|
├── confidence_gate.py # ConfidenceGate(置信度门控)
|
||||||
├── quality_assessor.py # 多维质量评估(当前未接入生产流程)
|
├── quality_assessor.py # 多维质量评估
|
||||||
├── reasoning_reflector.py # 推理反思(当前未接入生产流程)
|
├── reasoning_reflector.py # 推理反思
|
||||||
├── loop_guard.py # 循环防护(当前未接入生产流程)
|
├── loop_guard.py # 循环防护
|
||||||
├── prompt_guard.py # Prompt 安全守卫
|
|
||||||
├── llm_budget.py # LLM 调用预算控制
|
├── llm_budget.py # LLM 调用预算控制
|
||||||
├── llm_utils.py # LLM 调用工具函数
|
├── llm_utils.py # LLM 调用工具函数
|
||||||
|
├── semantic_cache.py # 语义缓存
|
||||||
|
├── cache.py # 三层缓存管理器(Query/Embedding/Rerank)
|
||||||
├── status_codes.py # 状态码定义
|
├── status_codes.py # 状态码定义
|
||||||
└── constants.py # 公共常量
|
└── constants.py # 公共常量
|
||||||
|
|
||||||
@@ -622,7 +666,7 @@ knowledge/ # 知识库管理
|
|||||||
|
|
||||||
api/ # API 路由层
|
api/ # API 路由层
|
||||||
├── __init__.py # create_app() 工厂
|
├── __init__.py # create_app() 工厂
|
||||||
├── chat_routes.py # /chat, /rag(SSE), /search(核心编排入口)
|
├── chat_routes.py # /chat, /rag(SSE), /search
|
||||||
├── kb_routes.py # /collections
|
├── kb_routes.py # /collections
|
||||||
├── document_routes.py # /documents/*
|
├── document_routes.py # /documents/*
|
||||||
├── sync_routes.py # /sync
|
├── sync_routes.py # /sync
|
||||||
@@ -682,10 +726,6 @@ deploy/ # 部署配置
|
|||||||
├── gunicorn.conf.py # Gunicorn WSGI 配置
|
├── gunicorn.conf.py # Gunicorn WSGI 配置
|
||||||
└── wsgi.py # WSGI 入口
|
└── wsgi.py # WSGI 入口
|
||||||
|
|
||||||
reports/ # 分析报告
|
|
||||||
├── cache_performance_report.md # 缓存性能验证报告
|
|
||||||
└── redis_migration_plan.md # Redis 缓存迁移方案(规划中)
|
|
||||||
|
|
||||||
config/ # 运行时配置
|
config/ # 运行时配置
|
||||||
└── banned_words.txt # 敏感词库
|
└── banned_words.txt # 敏感词库
|
||||||
```
|
```
|
||||||
@@ -694,8 +734,8 @@ config/ # 运行时配置
|
|||||||
|
|
||||||
## 十、与传统 RAG 对比
|
## 十、与传统 RAG 对比
|
||||||
|
|
||||||
| 特性 | 传统 RAG | 本系统 |
|
| 特性 | 传统 RAG | Agentic RAG (当前) |
|
||||||
|------|---------|--------|
|
|------|---------|-------------------|
|
||||||
| 意图判断 | 无 | IntentAnalyzer LLM 双层判断 |
|
| 意图判断 | 无 | IntentAnalyzer LLM 双层判断 |
|
||||||
| 查询改写 | 无 | 口语化→专业术语 + 实体补全 + 指代消解 |
|
| 查询改写 | 无 | 口语化→专业术语 + 实体补全 + 指代消解 |
|
||||||
| 检索方式 | 单一向量检索 | 向量 + BM25 + FAQ + 图片独立召回 |
|
| 检索方式 | 单一向量检索 | 向量 + BM25 + FAQ + 图片独立召回 |
|
||||||
@@ -704,11 +744,14 @@ config/ # 运行时配置
|
|||||||
| 重排序 | 无 | 云端 qwen3-rerank API(支持本地 BGE 回退) |
|
| 重排序 | 无 | 云端 qwen3-rerank API(支持本地 BGE 回退) |
|
||||||
| 问题分解 | 无 | 自动拆分对比/推理类查询 |
|
| 问题分解 | 无 | 自动拆分对比/推理类查询 |
|
||||||
| 闲聊处理 | 无 | 意图分析自动判断 |
|
| 闲聊处理 | 无 | 意图分析自动判断 |
|
||||||
| 缓存体系 | 无 | 四层缓存(Query + Embedding + Rerank + 语义缓存) |
|
| 网络搜索 | 无 | 可选支持(Serper API) |
|
||||||
| 语义缓存 | 无 | FAISS 向量索引,相似查询复用(92x 加速) |
|
| 知识图谱 | 无 | ~~可选支持(Neo4j)~~(已废弃,graph/ 目录已清空) |
|
||||||
|
| 幻觉验证 | 无 | 基于参考信息的答案验证 |
|
||||||
|
| 置信度门控 | 无 | Reranker 分数驱动,低分触发补救 |
|
||||||
|
| 缓存 | 无 | 三层缓存 + 语义缓存 |
|
||||||
| 自适应 TopK | 固定 top_k | 根据置信度动态调整 |
|
| 自适应 TopK | 固定 top_k | 根据置信度动态调整 |
|
||||||
| 上下文理解 | 无 | 多轮对话 + 历史上下文 |
|
| 上下文理解 | 无 | 多轮对话 + 历史上下文 |
|
||||||
| 响应时间 | ~2秒 | 首次 ~3-8秒 / 缓存命中 ~100-200毫秒 |
|
| 响应时间 | ~2秒 | ~3-8秒(取决于 Rerank + LLM) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -716,71 +759,71 @@ config/ # 运行时配置
|
|||||||
|
|
||||||
### 11.1 Rerank 调用路径
|
### 11.1 Rerank 调用路径
|
||||||
|
|
||||||
Rerank 在生产流程中有 **一个调用路径**:
|
Rerank 在系统中有 **两个独立调用路径**:
|
||||||
|
|
||||||
| 路径 | 位置 | 说明 |
|
| 路径 | 位置 | 说明 |
|
||||||
|------|------|------|
|
|------|------|------|
|
||||||
| 主检索管线 | `engine.rerank_results()` | RRF 融合后、MMR 去重前执行,对候选重排取 top_k |
|
| 主检索管线 | `engine.rerank_results()` | RRF 融合后、MMR 去重前执行,对候选重排取 top_k |
|
||||||
|
| 置信度门控 | `confidence_gate._compute_scores()` | 直接调用 `reranker.predict()`,可能重复推理 |
|
||||||
|
|
||||||
### 11.2 性能特征
|
### 11.2 性能瓶颈
|
||||||
|
|
||||||
| 项目 | 说明 |
|
| 瓶颈 | 严重程度 | 说明 |
|
||||||
|------|------|
|
|------|---------|------|
|
||||||
| Rerank 缓存命中率 | 偏低——键基于 `query + sorted(doc_ids)` 精确匹配,RRF 融合产出稍有不同就无法命中 |
|
| Rerank 缓存命中率偏低 | 🟡 中 | `rerank_results()` 已正确调用缓存读写,但缓存 key 基于 `query + sorted(doc_ids)` 精确匹配,RRF 融合产出稍有不同就无法命中 |
|
||||||
| 性能计时 | `rerank_results()` 返回 `_rerank_time_ms` 计时字段 |
|
| 置信度门控重复推理 | 🟡 中 | 同一 query+documents 可能被 Rerank 两次(当前仅备用路径使用,暂未影响生产) |
|
||||||
| Query Cache 拦截 | 重复查询被 Query Cache 在外层拦截,不会到达 Rerank 层(正确行为) |
|
| ~~无性能计时~~ | ~~🟡 中~~ | 已修复:`rerank_results()` 现返回 `_rerank_time_ms` 计时字段 |
|
||||||
|
| 查询分类器策略未生效 | 🟢 低 | `QueryClassifier` 定义的差异化 rerank 参数未传递到引擎 |
|
||||||
|
|
||||||
### 11.3 Rerank 配置参数
|
### 11.3 Rerank 配置参数
|
||||||
|
|
||||||
| 配置项 | 默认值 | 说明 |
|
| 配置项 | 默认值 | 说明 |
|
||||||
|--------|--------|------|
|
|--------|--------|------|
|
||||||
| `USE_RERANK` | `True` | 总开关 |
|
| `USE_RERANK` | `True` | 总开关 |
|
||||||
| `RERANK_BACKEND` | `"local"` | 后端选择 |
|
| `RERANK_BACKEND` | `"local"` | 后端选择:`local`=本地模型, `cloud`=云端API, `fallback`=优先云端失败回退本地 |
|
||||||
| `RERANK_CLOUD_MODEL` | `"qwen3-rerank"` | 云端模型名称 |
|
| `RERANK_CLOUD_MODEL` | `"qwen3-rerank"` | 云端 Rerank 模型名称(DashScope API) |
|
||||||
| `RERANK_CLOUD_API_KEY` | 同 `DASHSCOPE_API_KEY` | 云端 API 密钥 |
|
| `RERANK_CLOUD_API_KEY` | 同 `DASHSCOPE_API_KEY` | 云端 API 密钥 |
|
||||||
| `RERANK_CLOUD_BASE_URL` | `https://dashscope.aliyuncs.com/compatible-api/v1/reranks` | 云端 API 地址 |
|
| `RERANK_CLOUD_BASE_URL` | `https://dashscope.aliyuncs.com/compatible-api/v1/reranks` | 云端 API 地址 |
|
||||||
| `RERANK_CLOUD_TIMEOUT` | `15` | 云端请求超时(秒) |
|
| `RERANK_CLOUD_TIMEOUT` | `15` | 云端请求超时(秒) |
|
||||||
| `RERANK_MODEL_PATH` | `models/bge-reranker-base` | 本地模型路径 |
|
| `RERANK_MODEL_PATH` | `models/bge-reranker-base` | 本地模型路径(仅 local/fallback 模式) |
|
||||||
| `RERANK_CANDIDATES` | `20` | 送入 Rerank 的候选数 |
|
| `RERANK_CANDIDATES` | `20` | 送入 Rerank 的候选数 |
|
||||||
| `RERANK_TOP_K` | `15` | Rerank 后保留数 |
|
| `RERANK_TOP_K` | `15` | Rerank 后保留数 |
|
||||||
| `RERANK_USE_ONNX` | `True` | ONNX 加速开关 |
|
| `RERANK_USE_ONNX` | `True`(环境变量默认) | ONNX 加速开关(仅本地模式) |
|
||||||
| `RERANK_DEVICE` | 跟随 `DEVICE` | 设备选择 |
|
| `RERANK_DEVICE` | 跟随 `DEVICE` | 设备选择(仅本地模式) |
|
||||||
| `RERANK_THRESHOLD` | `0.3` | 上下文过滤阈值 |
|
| `RERANK_THRESHOLD` | `0.3` | 上下文过滤阈值 |
|
||||||
| `RERANK_CACHE_ENABLED` | `True` | 缓存开关 |
|
| `RERANK_CACHE_ENABLED` | `True` | 缓存开关(已在 `rerank_results()` 中使用) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 十二、最佳实践
|
## 十二、最佳实践
|
||||||
|
|
||||||
### 12.1 何时使用 /rag 接口
|
### 12.1 何时使用 Agentic RAG
|
||||||
|
|
||||||
**推荐使用**:
|
✅ **推荐使用**:
|
||||||
|
- 复杂问题需要多轮检索
|
||||||
- 复杂问题需要检索知识库
|
|
||||||
- 用户表达模糊需要改写
|
- 用户表达模糊需要改写
|
||||||
- 需要区分闲聊和知识问答
|
- 需要区分闲聊和知识问答
|
||||||
- 需要多轮对话记忆
|
- 需要多轮对话记忆
|
||||||
- 需要引用来源和证据
|
- 需要引用来源和幻觉验证
|
||||||
|
|
||||||
**不推荐使用**(改用 `/search` 接口更快):
|
❌ **不推荐使用**:
|
||||||
|
- 简单明确的问题(用 `/search` 接口更快)
|
||||||
- 简单明确的问题,只需返回原始检索结果
|
- 对响应时间极度敏感的场景
|
||||||
- 对响应时间极度敏感且不需要 LLM 生成答案的场景
|
|
||||||
|
|
||||||
### 12.2 性能优化
|
### 12.2 性能优化
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
# 减少迭代次数
|
||||||
|
rag = AgenticRAG(max_iterations=2)
|
||||||
|
|
||||||
|
# 禁用网络搜索
|
||||||
|
rag = AgenticRAG(enable_web_search=False)
|
||||||
|
|
||||||
# ONNX 加速默认已开启;如有兼容性问题可关闭(环境变量)
|
# ONNX 加速默认已开启;如有兼容性问题可关闭(环境变量)
|
||||||
# RERANK_USE_ONNX=false
|
# RERANK_USE_ONNX=false
|
||||||
|
|
||||||
# 使用轻量 MMR(文本相似度代替语义向量)
|
# 使用轻量 MMR(文本相似度代替语义向量)
|
||||||
# config.py: MMR_USE_EMBEDDING = False
|
# config.py: MMR_USE_EMBEDDING = False
|
||||||
|
|
||||||
# 调整语义缓存阈值(降低阈值可提高命中率,但可能降低准确性)
|
|
||||||
# config.py: SEMANTIC_CACHE_THRESHOLD = 0.90
|
|
||||||
|
|
||||||
# 调整缓存容量
|
|
||||||
# config.py: QUERY_CACHE_SIZE = 1000 # 增大查询缓存容量
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 12.3 调试技巧
|
### 12.3 调试技巧
|
||||||
@@ -791,62 +834,73 @@ result = engine.search_knowledge("问题", top_k=10)
|
|||||||
debug = result.get('_debug', {})
|
debug = result.get('_debug', {})
|
||||||
for step in debug.get('steps', []):
|
for step in debug.get('steps', []):
|
||||||
print(f"步骤: {step['name']}, 详情: {step}")
|
print(f"步骤: {step['name']}, 详情: {step}")
|
||||||
|
|
||||||
# 查看缓存统计
|
|
||||||
from core.cache import get_cache_manager
|
|
||||||
cm = get_cache_manager()
|
|
||||||
print(cm.get_stats())
|
|
||||||
|
|
||||||
# 查看语义缓存统计
|
|
||||||
from core.semantic_cache import get_semantic_cache
|
|
||||||
sc = get_semantic_cache()
|
|
||||||
print({"hits": sc.hits, "misses": sc.misses, "total": sc.total_entries})
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 十三、演进记录
|
## 附加篇:Agentic RAG 深入优化与工作机制
|
||||||
|
|
||||||
### v4.0(2026-06-05)— 统一编排 + 四层缓存修复
|
### 一、Agentic RAG 的核心架构
|
||||||
|
|
||||||
**删除未使用的备用编排路径**:移除了 `core/agentic.py` 及 8 个 Mixin 文件(共 10 个文件 ~2050 行)。这些文件实现了完整的决策循环编排(含置信度门控、质量评估、推理反思等),但从未接入任何 HTTP 路由。
|
Agentic RAG 构建了动态的决策闭环,核心组件包括:
|
||||||
|
|
||||||
**修复 Query Cache**:
|
- **意图分析器**:LLM 驱动的双层判断,替代硬编码规则
|
||||||
|
- **查询重写器**:口语化→专业术语、实体补全、指代消解
|
||||||
|
- **混合检索引擎**:向量 + BM25 + FAQ + 图片独立召回 + RRF 融合
|
||||||
|
- **MMR 去重**:平衡相关性与多样性,Rerank 后进一步精炼结果
|
||||||
|
- **Rerank 重排**:云端 qwen3-rerank API 精确排序,支持本地 BGE 回退
|
||||||
|
- **置信度门控**:Reranker 分数驱动,低分触发补救流程
|
||||||
|
- **幻觉验证**:基于参考信息验证答案,防止 LLM 编造
|
||||||
|
|
||||||
1. 修复 GET/SET 键不匹配——此前 SET 使用 `doc_hash` 分支,GET 使用 `kb_version` 分支,两端永远不匹配,命中率始终为 0%
|
### 二、分阶段优化策略
|
||||||
2. 修复 `CACHE_MIN_SCORE = 0.3` 阈值过高——ChromaDB 余弦距离经 `1-dist` 后得分约 0.03-0.06,远低于 0.3,导致几乎不写入缓存
|
|
||||||
|
|
||||||
**集成语义缓存**:将 FAISS 语义缓存从已删除的备用路径移植到生产 `/rag` 端点,在意图分析后、混合检索前检查,命中时跳过整个检索+生成流程。验证结果:命中率 66.7%,92 倍加速。
|
#### 1. 检索前:优化查询质量
|
||||||
|
|
||||||
### v3.2 — 模型/Reranker/管线更新
|
- **智能查询重写**:口语化表述 → 精准检索术语
|
||||||
|
- **复杂问题分解**:对比/推理类查询自动拆分为子查询
|
||||||
|
- **意图分析**:LLM 双层判断,避免不必要的检索
|
||||||
|
|
||||||
引入云端 qwen3-rerank、ONNX 加速、动态 RRF 权重等。
|
#### 2. 检索中:提升召回精准度
|
||||||
|
|
||||||
---
|
- **多路召回与融合**:向量 + BM25 + FAQ + 图片独立召回
|
||||||
|
- **动态 RRF 权重**:查询类型/长度驱动的权重调整
|
||||||
|
- **MMR 去重**:Rerank 后进一步精炼,平衡相关性与多样性(召回100 → Rerank取15 → MMR精炼)
|
||||||
|
- **Rerank 重排**:云端 qwen3-rerank 精排,置信度门控过滤低质量结果
|
||||||
|
|
||||||
## 十四、未来规划
|
#### 3. 检索后:质量评估与自我迭代
|
||||||
|
|
||||||
### Redis 缓存外部化
|
- **多维质量评估**:相关性/完整性/准确性/覆盖面
|
||||||
|
- **推理反思**:检查推理过程中未验证的假设
|
||||||
|
- **分层补救**:低置信度 → 查询重写 → 网络搜索
|
||||||
|
|
||||||
当前四层缓存均为进程内内存存储,在多 Worker / 多实例部署时无法共享。已规划 Redis 迁移方案(详见 `reports/redis_migration_plan.md`),核心设计:
|
### 三、系统级优化
|
||||||
|
|
||||||
- `RedisCacheManager` 提供与 `RAGCacheManager` 相同的接口
|
#### 1. 避免"循环检索"陷阱
|
||||||
- 通过 `REDIS_CACHE_URL` 环境变量启用,向后兼容
|
|
||||||
- 语义缓存采用混合方案:FAISS 索引保持在进程内,缓存结果存储到 Redis
|
|
||||||
- Query Cache、Embedding Cache、Rerank Cache 全部迁移到 Redis
|
|
||||||
|
|
||||||
### 可选能力接入
|
- 循环防护器(`loop_guard.py`):最多允许 N 次重写检索
|
||||||
|
- 置信度递增检查:连续两次无提升则终止
|
||||||
|
|
||||||
`core/` 目录下仍保留以下独立模块,当前未接入生产流程,可按需启用:
|
#### 2. 平衡智能性与效率
|
||||||
|
|
||||||
- `confidence_gate.py`:置信度门控,基于 Reranker 分数判断检索质量
|
- 轻量级决策模型:意图分析使用低温度、少 token 的 LLM 调用
|
||||||
- `quality_assessor.py`:多维质量评估(相关性/完整性/准确性/覆盖面)
|
- 三层缓存:Query Cache + Embedding Cache + Rerank Cache
|
||||||
- `reasoning_reflector.py`:推理反思,检查未验证的假设
|
- 语义缓存:相似查询复用结果(threshold=0.92)
|
||||||
- `loop_guard.py`:循环防护,防止重复检索
|
- LLM 预算控制:`MAX_LLM_CALLS_PER_QUERY = 2`
|
||||||
|
|
||||||
---
|
#### 3. 安全与可解释性
|
||||||
|
|
||||||
## 参考资料
|
- 证据溯源:引用标注 + 来源编号
|
||||||
|
- 思维链展示:`log_trace` 记录推理过程
|
||||||
|
- 安全护栏:输入验证 + 输出过滤 + 权限控制
|
||||||
|
|
||||||
|
### 四、学术前沿
|
||||||
|
|
||||||
|
1. **RAG-Gym**:三维度系统优化(提示工程 + 执行器调优 + 评判器训练)
|
||||||
|
2. **过程监督 vs 结果监督**:细粒度过程奖励显著提升训练效率
|
||||||
|
3. **Re2Search**:推理反思机制,F1 score 提升 10%+
|
||||||
|
|
||||||
|
### 参考资料
|
||||||
|
|
||||||
1. Xiong, G., et al. (2025). RAG-Gym: Systematic Optimization of Language Agents for Retrieval-Augmented Generation. arXiv:2502.13957
|
1. Xiong, G., et al. (2025). RAG-Gym: Systematic Optimization of Language Agents for Retrieval-Augmented Generation. arXiv:2502.13957
|
||||||
2. Zhang, W., et al. (2025). Process vs. Outcome Reward: Which is Better for Agentic RAG Reinforcement Learning. arXiv:2505.14069
|
2. Zhang, W., et al. (2025). Process vs. Outcome Reward: Which is Better for Agentic RAG Reinforcement Learning. arXiv:2505.14069
|
||||||
|
3. Agentic RAG 实战指南:从查询重写到多步重查全掌握。火山引擎 ADG 社区
|
||||||
@@ -444,7 +444,7 @@ Query Rewriting: "它" → "出差补助"
|
|||||||
|
|
||||||
- [后端对接规范.md](./后端对接规范.md) - API 接口规范(主要)
|
- [后端对接规范.md](./后端对接规范.md) - API 接口规范(主要)
|
||||||
- [数据库设计文档.md](./数据库设计文档.md) - 数据库结构
|
- [数据库设计文档.md](./数据库设计文档.md) - 数据库结构
|
||||||
- [RAG系统完整指南.md](./RAG系统完整指南.md) - RAG 系统架构与缓存详解
|
- [Agentic_RAG完整指南.md](./Agentic_RAG完整指南.md) - Agentic RAG 详解
|
||||||
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -134,6 +134,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
from .base import BM25Index
|
from .base import BM25Index
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if not kb_name or not kb_name.replace('_', '').isalnum():
|
if not kb_name or not kb_name.replace('_', '').isalnum():
|
||||||
return False, "向量库名称只能包含字母、数字和下划线"
|
return False, "向量库名称只能包含字母、数字和下划线"
|
||||||
|
|
||||||
@@ -190,6 +193,9 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
更新成功返回 True,向量库不存在返回 False
|
更新成功返回 True,向量库不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
collections = self._metadata.get("collections", {})
|
collections = self._metadata.get("collections", {})
|
||||||
if kb_name not in collections:
|
if kb_name not in collections:
|
||||||
return False
|
return False
|
||||||
@@ -228,6 +234,9 @@ class CollectionMixin:
|
|||||||
"""
|
"""
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
|
|
||||||
if kb_name == PUBLIC_KB_NAME:
|
if kb_name == PUBLIC_KB_NAME:
|
||||||
return False, "公开知识库不能删除"
|
return False, "公开知识库不能删除"
|
||||||
|
|
||||||
@@ -348,50 +357,36 @@ class CollectionMixin:
|
|||||||
- department: 所属部门
|
- department: 所属部门
|
||||||
- description: 描述
|
- description: 描述
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
# 扫描 base_path 下的所有子目录作为向量库
|
# 以元数据为唯一真相源,不再扫描磁盘自动补充
|
||||||
# 每个向量库使用独立目录:base_path/my_ky, base_path/public_kb 等
|
# (扫描磁盘会导致其他 worker 刚创建/删除的集合被误操作)
|
||||||
actual_collections = []
|
|
||||||
try:
|
|
||||||
if os.path.exists(self.base_path):
|
|
||||||
for item in os.listdir(self.base_path):
|
|
||||||
item_path = os.path.join(self.base_path, item)
|
|
||||||
if os.path.isdir(item_path) and not item.startswith('.'):
|
|
||||||
# 检查是否包含 chroma.sqlite3(有效的向量库目录)
|
|
||||||
if os.path.exists(os.path.join(item_path, 'chroma.sqlite3')):
|
|
||||||
actual_collections.append(item)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"扫描向量库目录失败: {e}")
|
|
||||||
|
|
||||||
# 如果扫描失败,回退到元数据中的集合列表
|
|
||||||
if not actual_collections:
|
|
||||||
actual_collections = list(self._metadata.get("collections", {}).keys())
|
|
||||||
|
|
||||||
for name in actual_collections:
|
|
||||||
if name not in self._metadata.get("collections", {}):
|
|
||||||
if "collections" not in self._metadata:
|
|
||||||
self._metadata["collections"] = {}
|
|
||||||
self._metadata["collections"][name] = {
|
|
||||||
"display_name": name,
|
|
||||||
"department": "",
|
|
||||||
"description": "",
|
|
||||||
"created_at": datetime.now().isoformat()
|
|
||||||
}
|
|
||||||
logger.info(f"自动补充向量库元数据: {name}")
|
|
||||||
|
|
||||||
self._save_metadata()
|
|
||||||
|
|
||||||
for name, info in self._metadata.get("collections", {}).items():
|
for name, info in self._metadata.get("collections", {}).items():
|
||||||
collection = self.get_collection(name)
|
try:
|
||||||
result.append(CollectionInfo(
|
collection = self.get_collection(name)
|
||||||
name=name,
|
result.append(CollectionInfo(
|
||||||
display_name=info.get("display_name", name),
|
name=name,
|
||||||
document_count=collection.count() if collection else 0,
|
display_name=info.get("display_name", name),
|
||||||
created_at=info.get("created_at", ""),
|
document_count=collection.count() if collection else 0,
|
||||||
department=info.get("department", ""),
|
created_at=info.get("created_at", ""),
|
||||||
description=info.get("description", "")
|
department=info.get("department", ""),
|
||||||
))
|
description=info.get("description", "")
|
||||||
|
))
|
||||||
|
except Exception as e:
|
||||||
|
# 只跳过不修改元数据(可能是其他 worker 刚创建的集合)
|
||||||
|
logger.warning(
|
||||||
|
f"跳过异常向量库 '{name}': {e}"
|
||||||
|
)
|
||||||
|
result.append(CollectionInfo(
|
||||||
|
name=name,
|
||||||
|
display_name=info.get("display_name", name),
|
||||||
|
document_count=0,
|
||||||
|
created_at=info.get("created_at", ""),
|
||||||
|
department=info.get("department", ""),
|
||||||
|
description=info.get("description", "")
|
||||||
|
))
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -405,4 +400,6 @@ class CollectionMixin:
|
|||||||
Returns:
|
Returns:
|
||||||
存在返回 True,不存在返回 False
|
存在返回 True,不存在返回 False
|
||||||
"""
|
"""
|
||||||
|
# 从磁盘重新加载元数据,确保多 worker 进程间状态一致
|
||||||
|
self._metadata = self._load_metadata()
|
||||||
return kb_name in self._metadata.get("collections", {})
|
return kb_name in self._metadata.get("collections", {})
|
||||||
|
|||||||
@@ -29,6 +29,11 @@
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import threading
|
import threading
|
||||||
|
try:
|
||||||
|
import fcntl
|
||||||
|
_HAS_FCNTL = True
|
||||||
|
except ImportError:
|
||||||
|
_HAS_FCNTL = False
|
||||||
from typing import List, Dict, Optional, Tuple
|
from typing import List, Dict, Optional, Tuple
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import logging
|
import logging
|
||||||
@@ -134,22 +139,36 @@ class KnowledgeBaseManager(
|
|||||||
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
logger.info(f"知识库管理器初始化完成,路径: {self.base_path},发现 {len(existing_kbs)} 个向量库: {existing_kbs}")
|
||||||
|
|
||||||
def _load_metadata(self) -> dict:
|
def _load_metadata(self) -> dict:
|
||||||
"""加载元数据"""
|
"""加载元数据(带文件锁)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
if os.path.exists(metadata_path):
|
if os.path.exists(metadata_path):
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||||
return json.load(f)
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
|
||||||
|
try:
|
||||||
|
return json.load(f)
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"加载元数据失败: {e}")
|
logger.error(f"加载元数据失败: {e}")
|
||||||
return {"collections": {}}
|
return {"collections": {}}
|
||||||
|
|
||||||
def _save_metadata(self):
|
def _save_metadata(self):
|
||||||
"""保存元数据"""
|
"""保存元数据(带文件锁,防止并发写入覆盖)"""
|
||||||
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
metadata_path = os.path.join(self.base_path, KB_METADATA_FILE)
|
||||||
try:
|
try:
|
||||||
with open(metadata_path, 'w', encoding='utf-8') as f:
|
with open(metadata_path, 'w', encoding='utf-8') as f:
|
||||||
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
json.dump(self._metadata, f, ensure_ascii=False, indent=2)
|
||||||
|
f.flush()
|
||||||
|
os.fsync(f.fileno())
|
||||||
|
finally:
|
||||||
|
if _HAS_FCNTL:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"保存元数据失败: {e}")
|
logger.error(f"保存元数据失败: {e}")
|
||||||
|
|
||||||
|
|||||||
@@ -616,7 +616,7 @@ class FeedbackService:
|
|||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
model=self.model,
|
model=self.model,
|
||||||
temperature=0.7,
|
temperature=0.7,
|
||||||
max_tokens=512
|
max_tokens=200
|
||||||
)
|
)
|
||||||
|
|
||||||
if not response:
|
if not response:
|
||||||
|
|||||||
@@ -159,7 +159,7 @@ class SessionManager:
|
|||||||
SELECT id, role, content, metadata, created_at
|
SELECT id, role, content, metadata, created_at
|
||||||
FROM messages
|
FROM messages
|
||||||
WHERE session_id = ?
|
WHERE session_id = ?
|
||||||
ORDER BY created_at DESC, id DESC
|
ORDER BY created_at DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
''', (session_id, limit))
|
''', (session_id, limit))
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user