feat(rag): 子章节级图片过滤 + VLM 后台增强 + 意图分析优化

- 图片选择新增 section_path 字段,支持子章节级过滤
- _filter_images_by_answer 增加 primary_sections 参数和叶子节点匹配
- 检索发散检测:primary_leaf_names > 3 时全局阈值+1
- _FIGURE_ANSWER_KEYWORDS 移除单字"图""表",正则图号兜底防误触发
- lazy_enhance 后台增强流程优化
- 意图分析与 LLM 工具层改进

评测:图片选择 F1 从 52.4% 提升至 66.3%(+13.9pp),Precision +16.5pp
This commit is contained in:
lacerate551
2026-06-20 19:26:40 +08:00
parent ee5295cc97
commit 5ba0d782e2
8 changed files with 3402 additions and 3028 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -20,9 +20,13 @@ DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
INTENT_MODEL = os.getenv("INTENT_MODEL", "mimo-v2.5") # 意图分析模型
INTENT_MODEL = os.getenv("INTENT_MODEL", "deepseek-v4-flash") # 意图分析模型(百炼快速模型)
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型(图片描述)
# 百炼 API阿里云 DashScope用于意图分析等轻量任务
BAILIAN_API_KEY = os.getenv("BAILIAN_API_KEY", "")
BAILIAN_BASE_URL = os.getenv("BAILIAN_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1")
# 兼容旧变量名(逐步迁移到 DASHSCOPE_* 命名)
API_KEY = DASHSCOPE_API_KEY
BASE_URL = DASHSCOPE_BASE_URL
@@ -96,12 +100,12 @@ RERANK_DEVICE = os.getenv("RERANK_DEVICE", os.getenv("DEVICE", "auto"))
# ----- 通用问答 -----
LLM_TEMPERATURE = 0.7 # 生成温度0=确定性1=随机性)
LLM_MAX_TOKENS = 3000 # 最大输出 token 数
LLM_MAX_TOKENS = 3000 # 最大输出 token 数推理模型思考链占用部分预算3000 平衡速度与质量)
# ----- 意图分析(轻量、确定性高)-----
# INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置
INTENT_TEMPERATURE = 0.1
INTENT_MAX_TOKENS = 4096 # 推理模型思维链消耗大量 token2048 偶发截断导致意图分析失败
INTENT_MAX_TOKENS = 1024 # 推理模型无需思考链预算JSON 输出 1024 足够
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
# ==============================================================================
@@ -286,3 +290,16 @@ def get_llm_client():
"""获取 LLM 客户端实例"""
from openai import OpenAI
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
_intent_client = None
def get_intent_client():
"""获取意图分析专用 LLM 客户端(百炼快速模型)"""
global _intent_client
if _intent_client is None:
if not BAILIAN_API_KEY:
raise ValueError("BAILIAN_API_KEY 未配置,请在 .env 中设置")
from openai import OpenAI
_intent_client = OpenAI(api_key=BAILIAN_API_KEY, base_url=BAILIAN_BASE_URL)
return _intent_client

View File

@@ -233,10 +233,10 @@ class IntentAnalyzer:
self._exact_cache_max = 500
def _get_client(self):
"""获取 LLM 客户端"""
"""获取 LLM 客户端(百炼快速模型)"""
if self._client is None:
from config import get_llm_client
self._client = get_llm_client()
from config import get_intent_client
self._client = get_intent_client()
return self._client
def _get_cache(self):

View File

@@ -71,19 +71,38 @@ def call_llm(
return response
content = response.choices[0].message.content
# 推理模型兼容content 为空时尝试从 reasoning_content 提取
# 推理模型兼容mimo-v2.5 等):
# 推理模型思考链消耗大量 token~1000max_tokens 不足时 content 为空,
# 全部输出进入 reasoning_content。此处从思考链中提取有效内容。
if not content or not content.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip():
# 从思维链中提取 JSON 块作为内容
json_match = re.search(r'\{[\s\S]*\}', reasoning)
if json_match:
logger.info("LLM: content为空从reasoning_content提取JSON")
return json_match.group().strip()
logger.warning("LLM 返回空 content可能需要增大 max_tokens")
# 先去掉 <think>...</think> 标签
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if cleaned:
logger.info("LLM: content为空从reasoning_content提取内容")
# 尝试提取 JSON 对象(兼容结构化响应场景)
json_match = re.search(r'\{[\s\S]*\}', cleaned)
if json_match:
try:
json.loads(json_match.group())
return json_match.group().strip()
except (json.JSONDecodeError, ValueError):
pass
# 尝试提取 JSON 数组
bracket_match = re.search(r'\[[\s\S]*\]', cleaned)
if bracket_match:
try:
json.loads(bracket_match.group())
return bracket_match.group().strip()
except (json.JSONDecodeError, ValueError):
pass
# 纯文本响应:直接返回清理后的内容
return cleaned
logger.warning("LLM 返回空 content 且 reasoning_content 也无法提取(可能需要增大 max_tokens")
return None
return content.strip()
except Exception as e:
logger.warning(f"LLM 调用失败: {e}")
@@ -95,7 +114,7 @@ def call_llm_stream(
prompt: str,
model: str,
temperature: float = 0.3,
max_tokens: int = 1000,
max_tokens: int = 3000,
messages: List[dict] = None,
error_prefix: str = "[错误]",
**kwargs
@@ -104,13 +123,14 @@ def call_llm_stream(
流式 LLM 调用(生成器封装)
自动处理流式响应,逐块 yield 文本内容。
兼容推理模型mimo-v2.5 等):当 content 为空时回退到 reasoning_content。
Args:
client: OpenAI 客户端实例
prompt: 用户提示
model: 模型名称
temperature: 温度参数
max_tokens: 最大 token 数
max_tokens: 最大 token 数(推理模型需留足思考链预算)
messages: 完整消息列表
error_prefix: 错误时的前缀
**kwargs: 其他参数
@@ -135,9 +155,33 @@ def call_llm_stream(
**kwargs
)
content_yielded = False
reasoning_buffer = []
for chunk in stream:
if chunk.choices and chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
if not chunk.choices:
continue
delta = chunk.choices[0].delta
# 正常 content 输出
if hasattr(delta, 'content') and delta.content:
content_yielded = True
yield delta.content
continue
# 推理模型reasoning_content思考链
rc = getattr(delta, 'reasoning_content', None)
if rc:
reasoning_buffer.append(rc)
# 回退content 为空但 reasoning_content 有内容(推理模型 token 不足时)
if not content_yielded and reasoning_buffer:
reasoning_text = ''.join(reasoning_buffer)
# 去掉 <think>...</think> 标签
cleaned = re.sub(r'', '', reasoning_text, flags=re.DOTALL).strip()
if cleaned:
logger.info("流式 LLM: content为空从reasoning_content提取内容")
yield cleaned
except Exception as e:
logger.error(f"LLM 流式调用失败: {e}")

View File

@@ -331,6 +331,21 @@ class CollectionMixin:
except Exception as e:
logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
# 注意:此时 ChromaDB collection 已删除cleanup_image_orphans 会扫描
# 所有剩余 collection仅该 collection 引用的图片会被识别为孤儿
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self)
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存, "
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
if kb_name in self._metadata.get("collections", {}):
del self._metadata["collections"][kb_name]
self._save_metadata()

View File

@@ -72,6 +72,18 @@ class DocumentMixin:
except Exception as e:
logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
logger.info(f"{kb_name} 删除文档: {filename}, 片段数: {deleted}")
return deleted

View File

@@ -25,7 +25,20 @@ def compute_file_hash(file_path: str) -> str:
return hashlib.md5(file_path.encode()).hexdigest()
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None) -> str:
def _get_embedding_model():
"""从 RAGEngine 获取 embedding 模型KnowledgeBaseManager 上没有此属性)"""
try:
from core.engine import get_engine
engine = get_engine()
if not engine._initialized:
engine.initialize()
return engine.embedding_model
except Exception as e:
logger.warning(f"获取 embedding 模型失败: {e}")
return None
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None, defer_chromadb: bool = False) -> str:
"""
懒加载 VLM 描述
@@ -36,6 +49,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
image_path: 图片路径(相对路径或绝对路径)
kb_name: 知识库名称
metadata: 图片元数据(包含 section、page、caption、上下文等
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
Returns:
VLM 生成的图片描述
@@ -49,23 +63,45 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
else:
full_image_path = image_path
# 1. 检查缓存
# 1. 检查缓存(空缓存视为无效,需重新生成)
img_hash = compute_file_hash(full_image_path)
cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
if cache_file.exists():
logger.info(f"VLM 缓存命中: {image_path}")
return cache_file.read_text(encoding='utf-8')
cached = cache_file.read_text(encoding='utf-8')
if len(cached.strip()) >= 5:
logger.info(f"VLM 缓存命中: {image_path}")
return cached
else:
logger.warning(f"VLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {image_path}")
try:
cache_file.unlink()
except OSError:
pass
# 2. 调用 VLM传入元数据
logger.info(f"VLM 懒加载: {image_path}")
kb_manager = get_kb_manager()
description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
# 3. 写入缓存
# 3. 空描述保护VLM 返回内容过短时不写入缓存和向量库
if not description or len(description.strip()) < 5:
logger.warning(f"VLM 返回描述过短({len(description.strip()) if description else 0}字符),跳过缓存和向量库更新: {image_path}")
return description or ''
# 4. 写入缓存
VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
cache_file.write_text(description, encoding='utf-8')
# 4. 更新向量库metadata + embedding
# 5. 更新向量库metadata + embedding,需校验 chunk_id 非空
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
if defer_chromadb:
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
return description
if not chunk_id:
logger.warning("chunk_id 为空,跳过向量库更新")
return description
try:
collection = kb_manager.get_collection(kb_name)
result = collection.get(ids=[chunk_id], include=['metadatas'])
@@ -79,7 +115,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
# 更新 embedding使用 VLM 描述重新计算向量)
# 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索
embedding_model = kb_manager.embedding_model
embedding_model = _get_embedding_model()
if embedding_model:
new_vector = embedding_model.encode(description).tolist()
if isinstance(new_vector[0], list):
@@ -91,20 +127,21 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
embeddings=[new_vector],
documents=[description] # 同时更新 document 字段
)
logger.info(f"已更新向量库 embedding: {chunk_id}")
logger.info(f"已更新向量库(embedding+metadata): {chunk_id}")
else:
# 无 embedding 模型时只更新 metadata
collection.update(
ids=[chunk_id],
metadatas=[new_metadata]
)
logger.info(f"已更新向量库(仅metadata,无embedding模型): {chunk_id}")
except Exception as e:
logger.warning(f"更新向量库失败: {e}")
return description
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str, defer_chromadb: bool = False) -> str:
"""
懒加载表格摘要
@@ -114,50 +151,76 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
chunk_id: 切片 ID
table_md: 表格 Markdown 内容
kb_name: 知识库名称
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
Returns:
LLM 生成的表格摘要
"""
from knowledge.manager import get_kb_manager
# 1. 检查缓存
# 1. 检查缓存(空缓存视为无效)
table_hash = hashlib.md5(table_md.encode()).hexdigest()
cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
if cache_file.exists():
logger.info(f"LLM 缓存命中: {chunk_id}")
return cache_file.read_text(encoding='utf-8')
cached = cache_file.read_text(encoding='utf-8')
if len(cached.strip()) >= 5:
logger.info(f"LLM 缓存命中: {chunk_id}")
return cached
else:
logger.warning(f"LLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {chunk_id}")
try:
cache_file.unlink()
except OSError:
pass
# 2. 调用 LLM
logger.info(f"LLM 懒加载: {chunk_id}")
kb_manager = get_kb_manager()
summary = kb_manager._generate_table_summary(table_md, None)
# 空摘要保护
if not summary or len(summary.strip()) < 5:
logger.warning(f"LLM 返回摘要过短,跳过缓存和向量库更新: {chunk_id}")
return summary or ''
# 3. 写入缓存
LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
cache_file.write_text(summary, encoding='utf-8')
# 4. 更新向量库(可选)
# 4. 更新向量库,需校验 chunk_id 非空
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
if defer_chromadb:
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
return summary
if not chunk_id:
logger.warning("chunk_id 为空,跳过表格向量库更新")
return summary
try:
collection = kb_manager.get_collection(kb_name)
result = collection.get(ids=[chunk_id], include=['metadatas'])
if result['metadatas']:
# 新增摘要切片
embedding_model = kb_manager.embedding_model
vector = embedding_model.encode(summary).tolist()
if isinstance(vector[0], list):
vector = vector[0]
# 新增摘要切片(需要 embedding 模型)
embedding_model = _get_embedding_model()
if embedding_model:
vector = embedding_model.encode(summary).tolist()
if isinstance(vector[0], list):
vector = vector[0]
collection.add(
ids=[f"{chunk_id}_summary"],
embeddings=[vector],
documents=[summary],
metadatas=[{
**result['metadatas'][0],
'is_summary': True,
'original_doc_id': chunk_id
}]
)
# 更新原切片标记
collection.add(
ids=[f"{chunk_id}_summary"],
embeddings=[vector],
documents=[summary],
metadatas=[{
**result['metadatas'][0],
'is_summary': True,
'original_doc_id': chunk_id
}]
)
logger.info(f"已新增摘要切片(embedding): {chunk_id}_summary")
else:
logger.info(f"跳过摘要切片(无embedding模型): {chunk_id}")
# 更新原切片标记(不依赖 embedding 模型)
collection.update(
ids=[chunk_id],
metadatas=[{**result['metadatas'][0], 'has_summary': True}]
@@ -168,7 +231,7 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
return summary
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str, defer_chromadb: bool = False):
"""
检索后增强:按需调用 LLM/VLM
@@ -176,23 +239,24 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
contexts: 检索上下文列表
query: 用户查询
kb_name: 知识库名称
defer_chromadb: 为 True 时后台线程只写文件缓存,不更新 ChromaDB避免写锁竞争
"""
for ctx in contexts:
meta = ctx.get('meta', {})
chunk_type = meta.get('chunk_type', 'text')
image_path = meta.get('image_path', '')
import re
# 图片切片:懒加载 VLM 描述
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
if image_path:
try:
for ctx in contexts:
try:
meta = ctx.get('meta', {})
chunk_type = meta.get('chunk_type', 'text')
image_path = meta.get('image_path', '')
# 图片切片:懒加载 VLM 描述
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
if image_path:
# 从 doc 字段中提取图号(上下文可能包含"见图2.5"等)
doc_text = ctx.get('doc', '')
import re
# 提取图号(从前文/后文中)
figure_number = ""
# 匹配 "见图2.5"、"图2.5"、"见图 2.5" 等
fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
if fig_match:
figure_number = fig_match.group(1)
@@ -210,78 +274,73 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
'page': meta.get('page'),
'caption': meta.get('caption', ''),
'source': meta.get('source', ''),
'figure_number': figure_number, # 添加提取的图号
'doc_text': doc_text # 添加完整文档文本
'figure_number': figure_number,
'doc_text': doc_text
}
vlm_desc = await lazy_vlm_description(
meta.get('id', ''),
meta.get('chunk_id', ''),
image_path,
kb_name,
metadata=image_metadata
metadata=image_metadata,
defer_chromadb=defer_chromadb
)
ctx['doc'] = vlm_desc
ctx['vlm_enhanced'] = True
except Exception as e:
logger.warning(f"VLM 懒加载失败: {e}")
if vlm_desc:
ctx['doc'] = vlm_desc
ctx['vlm_enhanced'] = True
# 表格切片:同时处理摘要和关联图片的 VLM 描述
elif chunk_type == 'table':
doc_text = ctx.get('doc', '')
# 表格切片:同时处理摘要和关联图片的 VLM 描述
elif chunk_type == 'table':
doc_text = ctx.get('doc', '')
# 1. 懒加载表格摘要(高分切片)
if not meta.get('has_summary'):
score = meta.get('score', 0)
if score > 0.7: # 只对高相关表格生成摘要
try:
# 1. 懒加载表格摘要(高分切片)
if not meta.get('has_summary'):
score = ctx.get('score', 0)
if score > 0.7:
summary = await lazy_table_summary(
meta.get('id', ''),
meta.get('chunk_id', ''),
doc_text,
kb_name
kb_name,
defer_chromadb=defer_chromadb
)
# 摘要作为补充信息
ctx['summary'] = summary
ctx['llm_enhanced'] = True
except Exception as e:
logger.warning(f"表格摘要懒加载失败: {e}")
# 2. 表格有关联图片时,懒加载 VLM 描述
if image_path and not meta.get('has_vlm_desc'):
try:
import re
if summary:
ctx['summary'] = summary
ctx['llm_enhanced'] = True
# 2. 表格有关联图片时,懒加载 VLM 描述
if image_path and not meta.get('has_vlm_desc'):
# 提取表号(如 "表2.2"、"见表2.1"
table_number = ""
# 匹配 "表2.2"、"见表2.2"、"见表 2.2" 等
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
if table_match:
table_number = table_match.group(1)
# 如果 doc 中没有,尝试从 section 中提取
section = meta.get('section') or meta.get('section_path', '')
if not table_number and section:
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
if table_match:
table_number = table_match.group(1)
# 构建表格图片元数据
table_image_metadata = {
'section': section,
'page': meta.get('page'),
'caption': meta.get('caption', ''),
'source': meta.get('source', ''),
'table_number': table_number, # 表号
'figure_number': table_number, # 兼容字段
'table_number': table_number,
'figure_number': table_number,
'doc_text': doc_text,
'is_table': True # 标记为表格图片
'is_table': True
}
vlm_desc = await lazy_vlm_description(
meta.get('id', ''),
meta.get('chunk_id', ''),
image_path,
kb_name,
metadata=table_image_metadata
metadata=table_image_metadata,
defer_chromadb=defer_chromadb
)
# 表格图片描述作为补充信息
ctx['image_description'] = vlm_desc
ctx['vlm_enhanced'] = True
except Exception as e:
logger.warning(f"表格图片 VLM 懒加载失败: {e}")
if vlm_desc:
ctx['image_description'] = vlm_desc
ctx['vlm_enhanced'] = True
except Exception as e:
chunk_id = ctx.get('meta', {}).get('chunk_id', '?')
logger.warning(f"增强切片失败(chunk_id={chunk_id}): {e}")

View File

@@ -622,7 +622,7 @@ class KnowledgeBaseManager(
try:
from config import get_llm_client, DASHSCOPE_MODEL
client = get_llm_client()
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
return summary.strip() if summary else ""
except Exception as e:
logger.warning(f"生成表格摘要失败: {e}")
@@ -681,13 +681,31 @@ class KnowledgeBaseManager(
]
}
],
max_tokens=512
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token需留足输出空间
)
description = response.choices[0].message.content
# 推理模型兼容content 为空时从 reasoning_content 提取
if not description or not description.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip():
import re
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if cleaned:
logger.info(f"VLM content为空从reasoning_content提取描述: {image_path}")
description = cleaned
else:
description = reasoning.strip()
if not description:
logger.warning(f"VLM 返回空描述: {image_path}")
return ""
# 缓存结果
import hashlib
import re as _re
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
cache_dir = Path('.data/cache/vlm')
cache_dir.mkdir(parents=True, exist_ok=True)