init: RAG 知识库服务初始提交
- 后端 API（Flask + Gunicorn） - RAG 引擎（混合检索 + 云端 Reranker + 引用溯源） - 文档解析（MinerU + 多格式支持） - Docker 生产部署配置 - 排除前端项目、敏感配置、模型文件
This commit is contained in:
lacerate551
390
core/agentic.py
Normal file
390
core/agentic.py
Normal file
@@ -0,0 +1,390 @@
|
||||
"""
|
||||
Agentic RAG - 知识库智能问答系统
|
||||
|
||||
核心能力：
|
||||
1. 知识库检索 - 向量检索 + BM25 + Rerank
|
||||
2. 网络搜索 - 当知识库不足时自动搜索（需配置SERPER_API_KEY）
|
||||
3. 图谱检索 - 实体关系推理（需配置Neo4j）
|
||||
4. 多源融合 - 智能处理知识库和网络内容
|
||||
5. Agent决策 - 动态决定检索、改写、分解等操作
|
||||
|
||||
使用方式：
|
||||
from core.agentic import AgenticRAG
|
||||
|
||||
rag = AgenticRAG()
|
||||
result = rag.process("你的问题")
|
||||
print(result["answer"])
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from openai import OpenAI
|
||||
|
||||
# 配置日志
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 导入基础模块
|
||||
from core.engine import get_engine
|
||||
from core.llm_utils import call_llm, quick_yes_no, parse_json_from_response
|
||||
|
||||
# 导入基础常量和配置
|
||||
from .agentic_base import (
|
||||
API_KEY, BASE_URL, MODEL,
|
||||
HAS_SERPER,
|
||||
HAS_BUDGET, SEMANTIC_CACHE_ENABLED,
|
||||
MAX_CONTEXT_TOKENS, MAX_CONTEXT_COUNT, RERANK_THRESHOLD,
|
||||
SOURCE_KB, SOURCE_WEB,
|
||||
)
|
||||
|
||||
# 尝试导入语义缓存
|
||||
try:
|
||||
from core.semantic_cache import SemanticCache
|
||||
except ImportError:
|
||||
SemanticCache = None
|
||||
|
||||
# 导入 Mixin 类
|
||||
from .agentic_query import QueryRewriteMixin
|
||||
from .agentic_search import SearchMixin
|
||||
from .agentic_answer import AnswerMixin
|
||||
from .agentic_citation import CitationMixin
|
||||
from .agentic_media import RichMediaMixin
|
||||
from .agentic_quality import QualityMixin
|
||||
from .agentic_context import ContextMixin
|
||||
from .agentic_meta import MetaQuestionMixin
|
||||
|
||||
|
||||
class AgenticRAG(
|
||||
QueryRewriteMixin,
|
||||
SearchMixin,
|
||||
AnswerMixin,
|
||||
CitationMixin,
|
||||
RichMediaMixin,
|
||||
QualityMixin,
|
||||
ContextMixin,
|
||||
MetaQuestionMixin
|
||||
):
|
||||
"""
|
||||
Agentic RAG - 知识库智能问答
|
||||
|
||||
通过 Mixin 模式组合功能：
|
||||
- QueryRewriteMixin: 查询重写
|
||||
- SearchMixin: 检索功能
|
||||
- AnswerMixin: 答案生成
|
||||
- CitationMixin: 引用处理
|
||||
- RichMediaMixin: 富媒体处理
|
||||
- QualityMixin: 质量评估
|
||||
- ContextMixin: 上下文处理
|
||||
- MetaQuestionMixin: 元问题处理
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_iterations: int = 3,
|
||||
enable_web_search: bool = True,
|
||||
**kwargs
|
||||
):
|
||||
"""初始化"""
|
||||
self.max_iterations = max_iterations
|
||||
self.enable_web_search = enable_web_search and HAS_SERPER
|
||||
self.client = OpenAI(api_key=API_KEY, base_url=BASE_URL)
|
||||
|
||||
# 信息来源标记
|
||||
self.SOURCE_KB = SOURCE_KB
|
||||
self.SOURCE_WEB = SOURCE_WEB
|
||||
|
||||
# 初始化置信度门控
|
||||
try:
|
||||
from core.confidence_gate import create_gate
|
||||
self.confidence_gate = create_gate()
|
||||
except ImportError:
|
||||
self.confidence_gate = None
|
||||
|
||||
# 初始化多维质量评估器
|
||||
try:
|
||||
from core.quality_assessor import create_assessor
|
||||
self.quality_assessor = create_assessor()
|
||||
except ImportError:
|
||||
self.quality_assessor = None
|
||||
|
||||
# 初始化推理反思器
|
||||
try:
|
||||
from core.reasoning_reflector import create_reflector
|
||||
self.reasoning_reflector = create_reflector()
|
||||
except ImportError:
|
||||
self.reasoning_reflector = None
|
||||
|
||||
# 初始化循环防护器
|
||||
try:
|
||||
from core.loop_guard import create_guard
|
||||
self.loop_guard = create_guard(max_iterations=max_iterations)
|
||||
except ImportError:
|
||||
self.loop_guard = None
|
||||
|
||||
# 初始化语义缓存
|
||||
self.semantic_cache = None
|
||||
self.embedding_model = None
|
||||
if SEMANTIC_CACHE_ENABLED and SemanticCache:
|
||||
try:
|
||||
engine = get_engine()
|
||||
if engine and hasattr(engine, 'embedding_model'):
|
||||
self.embedding_model = engine.embedding_model
|
||||
emb_dim = 768
|
||||
# 优先使用新 API，兼容旧版本
|
||||
if hasattr(self.embedding_model, 'get_embedding_dimension'):
|
||||
emb_dim = self.embedding_model.get_embedding_dimension()
|
||||
elif hasattr(self.embedding_model, 'get_sentence_embedding_dimension'):
|
||||
emb_dim = self.embedding_model.get_sentence_embedding_dimension()
|
||||
self.semantic_cache = SemanticCache(
|
||||
dim=emb_dim,
|
||||
threshold=0.92,
|
||||
max_size=5000
|
||||
)
|
||||
logger.info(f"语义缓存已启用，维度={emb_dim}")
|
||||
except Exception as e:
|
||||
logger.warning(f"语义缓存初始化失败: {e}")
|
||||
self.semantic_cache = None
|
||||
|
||||
# Context Compression 配置
|
||||
self.MAX_CONTEXT_TOKENS = MAX_CONTEXT_TOKENS
|
||||
self.MAX_CONTEXT_COUNT = MAX_CONTEXT_COUNT
|
||||
self.RERANK_THRESHOLD = RERANK_THRESHOLD
|
||||
|
||||
# Answer Grounding 配置
|
||||
self.MAX_GROUNDING_RETRY = 1
|
||||
self.grounding_retry_count = 0
|
||||
|
||||
def should_rewrite(self, query: str, history: list = None) -> bool:
|
||||
"""判断是否需要重写查询"""
|
||||
# 口语化表达模式
|
||||
colloquial_patterns = [
|
||||
"这个", "那个", "它", "这", "那",
|
||||
"上面", "下面", "刚才", "之前",
|
||||
"能不能", "可以吗", "行不行",
|
||||
"怎么办", "怎么弄", "咋整"
|
||||
]
|
||||
|
||||
for pattern in colloquial_patterns:
|
||||
if pattern in query:
|
||||
return True
|
||||
|
||||
# 查询太短
|
||||
if len(query) < 5:
|
||||
return True
|
||||
|
||||
# 有对话历史时，可能需要实体补全
|
||||
if history:
|
||||
for msg in reversed(history[-3:]):
|
||||
if msg.get("role") == "user":
|
||||
prev_query = msg.get("content", "")
|
||||
# 如果当前查询缺少主语，可能需要补全
|
||||
if any(kw in query for kw in ["标准", "规定", "流程", "制度"]):
|
||||
if not any(kw in query for kw in ["报销", "出差", "请假", "工资", "合同"]):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def process(self, query: str, verbose: bool = True, history: list = None,
|
||||
allowed_levels: list = None, role: str = None, department: str = None,
|
||||
emit_log=None) -> dict:
|
||||
"""
|
||||
主流程：智能问答
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
verbose: 是否打印详细过程
|
||||
history: 对话历史
|
||||
allowed_levels: 允许访问的安全级别
|
||||
role: 用户角色
|
||||
department: 用户部门
|
||||
emit_log: 日志发射函数（流式输出）
|
||||
|
||||
Returns:
|
||||
{
|
||||
"answer": str,
|
||||
"sources": list,
|
||||
"images": list,
|
||||
"tables": list,
|
||||
"citations": list,
|
||||
"log_trace": list
|
||||
}
|
||||
"""
|
||||
log_trace = []
|
||||
|
||||
# 1. 检查元问题
|
||||
if self._is_meta_question(query):
|
||||
answer = self._answer_meta_question(query, allowed_levels, role, department)
|
||||
return {
|
||||
"answer": answer,
|
||||
"sources": [],
|
||||
"images": [],
|
||||
"tables": [],
|
||||
"citations": [],
|
||||
"log_trace": [{"phase": "meta_question", "query": query}]
|
||||
}
|
||||
|
||||
# 2. 查询重写
|
||||
current_query = query
|
||||
if self.should_rewrite(query, history):
|
||||
current_query = self._rewrite_query(query, history)
|
||||
log_trace.append({"phase": "rewrite", "original": query, "rewritten": current_query})
|
||||
if emit_log:
|
||||
emit_log(f"📝 查询重写: {query} → {current_query}")
|
||||
|
||||
# 3. 知识库检索
|
||||
contexts = []
|
||||
try:
|
||||
engine = get_engine()
|
||||
if not engine._initialized:
|
||||
engine.initialize()
|
||||
|
||||
# 获取用户可访问的向量库
|
||||
from knowledge.manager import get_kb_manager
|
||||
kb_mgr = get_kb_manager()
|
||||
accessible = kb_mgr.get_accessible_collections(role or 'user', department or '', 'read')
|
||||
|
||||
# 统一使用 search_knowledge() — 生产路径的同一 API
|
||||
# search_knowledge() 返回 dict: {ids, documents, metadatas, distances}，每项为 list[list]
|
||||
# top_k 与生产路径对齐（30），确保 Rerank 后仍有足够结果
|
||||
results = engine.search_knowledge(
|
||||
query=current_query,
|
||||
top_k=30,
|
||||
collections=accessible if accessible else None,
|
||||
)
|
||||
|
||||
docs = results.get('documents', [[]])[0]
|
||||
metas = results.get('metadatas', [[]])[0]
|
||||
dists = results.get('distances', [[]])[0]
|
||||
|
||||
for doc, meta, score in zip(docs, metas, dists):
|
||||
contexts.append({
|
||||
'doc': doc,
|
||||
'meta': meta,
|
||||
'score': 1 - score if score <= 1 else 1 / (1 + score), # 距离→相似度
|
||||
'source_type': self.SOURCE_KB,
|
||||
'query': current_query
|
||||
})
|
||||
|
||||
log_trace.append({"phase": "kb_search", "query": current_query, "results": len(contexts)})
|
||||
if emit_log:
|
||||
emit_log(f"🔍 知识库检索: {len(contexts)} 条结果")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"知识库检索失败: {e}")
|
||||
log_trace.append({"phase": "kb_search", "error": str(e)})
|
||||
|
||||
# 3.5 图片独立检索 + 打分选择（与生产路径对齐）
|
||||
selected_images = []
|
||||
try:
|
||||
from api.chat_routes import select_images
|
||||
selected_images = select_images(contexts, current_query)
|
||||
if selected_images and emit_log:
|
||||
emit_log(f"🖼️ 图片选择: {len(selected_images)} 张相关图片")
|
||||
log_trace.append({"phase": "image_selection", "count": len(selected_images)})
|
||||
except Exception as e:
|
||||
logger.debug(f"图片选择失败: {e}")
|
||||
|
||||
# 4. 上下文压缩
|
||||
contexts = self._compress_contexts(current_query, contexts)
|
||||
|
||||
# 5. 网络搜索（如果需要）
|
||||
web_contexts = []
|
||||
if self.enable_web_search and (
|
||||
not contexts or
|
||||
not self._is_kb_result_sufficient(current_query, [c['doc'] for c in contexts]) or
|
||||
self._should_web_search(current_query)
|
||||
):
|
||||
web_contexts = self._web_search_flow(current_query, log_trace, emit_log, verbose, allowed_levels)
|
||||
contexts.extend(web_contexts)
|
||||
|
||||
# 7. 生成答案（注入图片描述到上下文）
|
||||
if contexts:
|
||||
# 将选中图片的描述注入上下文，让 LLM 能"看到"图片内容
|
||||
if selected_images:
|
||||
image_contexts = []
|
||||
for i, img in enumerate(selected_images, 1):
|
||||
full_desc = img.get('full_description', '') or img.get('description', '')
|
||||
if full_desc:
|
||||
img_source = img.get('source', '')
|
||||
img_page = img.get('page', '')
|
||||
source_info = f"（来源：{img_source} 第{img_page}页）" if img_source and img_page else ""
|
||||
image_contexts.append({
|
||||
'doc': f"【图片{i}】{full_desc}{source_info}",
|
||||
'meta': {'source': img_source, 'page': img_page, 'chunk_type': img.get('type', 'image')},
|
||||
'score': img.get('score', 0),
|
||||
'source_type': self.SOURCE_KB,
|
||||
'query': current_query
|
||||
})
|
||||
# 图片上下文追加到知识库上下文前面（让 LLM 优先看到图片信息）
|
||||
contexts = image_contexts + contexts
|
||||
|
||||
answer = self._generate_fused_answer(current_query, contexts, allowed_levels)
|
||||
|
||||
# 答案验证（防止幻觉）
|
||||
if self.grounding_retry_count < self.MAX_GROUNDING_RETRY:
|
||||
answer = self._verify_and_refine_answer(current_query, answer, contexts)
|
||||
else:
|
||||
answer = self._generate_no_context_answer(current_query, allowed_levels)
|
||||
|
||||
# 8. 构建引用
|
||||
citations = self._attach_citations(answer, contexts)
|
||||
|
||||
# 9. 图片结果：优先使用 select_images 的结构化结果（含 URL + 打分）
|
||||
# 回退到 _extract_rich_media（从 metadata 提取）
|
||||
if selected_images:
|
||||
images_result = selected_images
|
||||
else:
|
||||
rich_media = self._extract_rich_media(contexts)
|
||||
images_result = rich_media.get("images", [])
|
||||
|
||||
return {
|
||||
"answer": answer,
|
||||
"sources": citations.get("sources", []),
|
||||
"images": images_result,
|
||||
"tables": citations.get("tables", []) if isinstance(citations, dict) else [],
|
||||
"citations": citations.get("citations", []),
|
||||
"log_trace": log_trace
|
||||
}
|
||||
|
||||
def chat_search(self, query: str, history: list = None, role: str = None,
|
||||
department: str = None, allowed_levels: list = None) -> dict:
|
||||
"""聊天式检索接口"""
|
||||
return self.process(
|
||||
query,
|
||||
verbose=False,
|
||||
history=history,
|
||||
role=role,
|
||||
department=department,
|
||||
allowed_levels=allowed_levels
|
||||
)
|
||||
|
||||
def chat(self):
|
||||
"""命令行交互模式"""
|
||||
print("🤖 Agentic RAG 已启动，输入 'quit' 退出")
|
||||
print("-" * 50)
|
||||
|
||||
history = []
|
||||
while True:
|
||||
try:
|
||||
query = input("\n👤 你: ").strip()
|
||||
if not query:
|
||||
continue
|
||||
if query.lower() in ['quit', 'exit', 'q']:
|
||||
print("👋 再见！")
|
||||
break
|
||||
|
||||
result = self.process(query, verbose=True, history=history)
|
||||
print(f"\n🤖 AI: {result['answer']}")
|
||||
|
||||
if result['sources']:
|
||||
print("\n📚 来源:")
|
||||
for src in result['sources'][:3]:
|
||||
print(f" - {src['source']}")
|
||||
|
||||
history.append({"role": "user", "content": query})
|
||||
history.append({"role": "assistant", "content": result['answer']})
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n👋 再见！")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"❌ 错误: {e}")
|
||||
Reference in New Issue
Block a user