feat(rag): 子章节级图片过滤 + VLM 后台增强 + 意图分析优化

- 图片选择新增 section_path 字段,支持子章节级过滤
- _filter_images_by_answer 增加 primary_sections 参数和叶子节点匹配
- 检索发散检测:primary_leaf_names > 3 时全局阈值+1
- _FIGURE_ANSWER_KEYWORDS 移除单字"图""表",正则图号兜底防误触发
- lazy_enhance 后台增强流程优化
- 意图分析与 LLM 工具层改进

评测:图片选择 F1 从 52.4% 提升至 66.3%(+13.9pp),Precision +16.5pp
This commit is contained in:
lacerate551
2026-06-20 19:26:40 +08:00
parent ee5295cc97
commit 5ba0d782e2
8 changed files with 3402 additions and 3028 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -20,9 +20,13 @@ DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1") DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型 DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型 RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
INTENT_MODEL = os.getenv("INTENT_MODEL", "mimo-v2.5") # 意图分析模型 INTENT_MODEL = os.getenv("INTENT_MODEL", "deepseek-v4-flash") # 意图分析模型(百炼快速模型)
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型(图片描述) VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型(图片描述)
# 百炼 API阿里云 DashScope用于意图分析等轻量任务
BAILIAN_API_KEY = os.getenv("BAILIAN_API_KEY", "")
BAILIAN_BASE_URL = os.getenv("BAILIAN_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1")
# 兼容旧变量名(逐步迁移到 DASHSCOPE_* 命名) # 兼容旧变量名(逐步迁移到 DASHSCOPE_* 命名)
API_KEY = DASHSCOPE_API_KEY API_KEY = DASHSCOPE_API_KEY
BASE_URL = DASHSCOPE_BASE_URL BASE_URL = DASHSCOPE_BASE_URL
@@ -96,12 +100,12 @@ RERANK_DEVICE = os.getenv("RERANK_DEVICE", os.getenv("DEVICE", "auto"))
# ----- 通用问答 ----- # ----- 通用问答 -----
LLM_TEMPERATURE = 0.7 # 生成温度0=确定性1=随机性) LLM_TEMPERATURE = 0.7 # 生成温度0=确定性1=随机性)
LLM_MAX_TOKENS = 3000 # 最大输出 token 数 LLM_MAX_TOKENS = 3000 # 最大输出 token 数推理模型思考链占用部分预算3000 平衡速度与质量)
# ----- 意图分析(轻量、确定性高)----- # ----- 意图分析(轻量、确定性高)-----
# INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置 # INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置
INTENT_TEMPERATURE = 0.1 INTENT_TEMPERATURE = 0.1
INTENT_MAX_TOKENS = 4096 # 推理模型思维链消耗大量 token2048 偶发截断导致意图分析失败 INTENT_MAX_TOKENS = 1024 # 推理模型无需思考链预算JSON 输出 1024 足够
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息 INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
# ============================================================================== # ==============================================================================
@@ -286,3 +290,16 @@ def get_llm_client():
"""获取 LLM 客户端实例""" """获取 LLM 客户端实例"""
from openai import OpenAI from openai import OpenAI
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL) return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
_intent_client = None
def get_intent_client():
"""获取意图分析专用 LLM 客户端(百炼快速模型)"""
global _intent_client
if _intent_client is None:
if not BAILIAN_API_KEY:
raise ValueError("BAILIAN_API_KEY 未配置,请在 .env 中设置")
from openai import OpenAI
_intent_client = OpenAI(api_key=BAILIAN_API_KEY, base_url=BAILIAN_BASE_URL)
return _intent_client

View File

@@ -233,10 +233,10 @@ class IntentAnalyzer:
self._exact_cache_max = 500 self._exact_cache_max = 500
def _get_client(self): def _get_client(self):
"""获取 LLM 客户端""" """获取 LLM 客户端(百炼快速模型)"""
if self._client is None: if self._client is None:
from config import get_llm_client from config import get_intent_client
self._client = get_llm_client() self._client = get_intent_client()
return self._client return self._client
def _get_cache(self): def _get_cache(self):

View File

@@ -71,19 +71,38 @@ def call_llm(
return response return response
content = response.choices[0].message.content content = response.choices[0].message.content
# 推理模型兼容content 为空时尝试从 reasoning_content 提取 # 推理模型兼容mimo-v2.5 等):
# 推理模型思考链消耗大量 token~1000max_tokens 不足时 content 为空,
# 全部输出进入 reasoning_content。此处从思考链中提取有效内容。
if not content or not content.strip(): if not content or not content.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None) reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip(): if reasoning and reasoning.strip():
# 从思维链中提取 JSON 块作为内容 # 先去掉 <think>...</think> 标签
json_match = re.search(r'\{[\s\S]*\}', reasoning) cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if json_match: if cleaned:
logger.info("LLM: content为空从reasoning_content提取JSON") logger.info("LLM: content为空从reasoning_content提取内容")
return json_match.group().strip() # 尝试提取 JSON 对象(兼容结构化响应场景)
logger.warning("LLM 返回空 content可能需要增大 max_tokens") json_match = re.search(r'\{[\s\S]*\}', cleaned)
if json_match:
try:
json.loads(json_match.group())
return json_match.group().strip()
except (json.JSONDecodeError, ValueError):
pass
# 尝试提取 JSON 数组
bracket_match = re.search(r'\[[\s\S]*\]', cleaned)
if bracket_match:
try:
json.loads(bracket_match.group())
return bracket_match.group().strip()
except (json.JSONDecodeError, ValueError):
pass
# 纯文本响应:直接返回清理后的内容
return cleaned
logger.warning("LLM 返回空 content 且 reasoning_content 也无法提取(可能需要增大 max_tokens")
return None return None
return content.strip() return content.strip()
except Exception as e: except Exception as e:
logger.warning(f"LLM 调用失败: {e}") logger.warning(f"LLM 调用失败: {e}")
@@ -95,7 +114,7 @@ def call_llm_stream(
prompt: str, prompt: str,
model: str, model: str,
temperature: float = 0.3, temperature: float = 0.3,
max_tokens: int = 1000, max_tokens: int = 3000,
messages: List[dict] = None, messages: List[dict] = None,
error_prefix: str = "[错误]", error_prefix: str = "[错误]",
**kwargs **kwargs
@@ -104,13 +123,14 @@ def call_llm_stream(
流式 LLM 调用(生成器封装) 流式 LLM 调用(生成器封装)
自动处理流式响应,逐块 yield 文本内容。 自动处理流式响应,逐块 yield 文本内容。
兼容推理模型mimo-v2.5 等):当 content 为空时回退到 reasoning_content。
Args: Args:
client: OpenAI 客户端实例 client: OpenAI 客户端实例
prompt: 用户提示 prompt: 用户提示
model: 模型名称 model: 模型名称
temperature: 温度参数 temperature: 温度参数
max_tokens: 最大 token 数 max_tokens: 最大 token 数(推理模型需留足思考链预算)
messages: 完整消息列表 messages: 完整消息列表
error_prefix: 错误时的前缀 error_prefix: 错误时的前缀
**kwargs: 其他参数 **kwargs: 其他参数
@@ -135,9 +155,33 @@ def call_llm_stream(
**kwargs **kwargs
) )
content_yielded = False
reasoning_buffer = []
for chunk in stream: for chunk in stream:
if chunk.choices and chunk.choices[0].delta.content: if not chunk.choices:
yield chunk.choices[0].delta.content continue
delta = chunk.choices[0].delta
# 正常 content 输出
if hasattr(delta, 'content') and delta.content:
content_yielded = True
yield delta.content
continue
# 推理模型reasoning_content思考链
rc = getattr(delta, 'reasoning_content', None)
if rc:
reasoning_buffer.append(rc)
# 回退content 为空但 reasoning_content 有内容(推理模型 token 不足时)
if not content_yielded and reasoning_buffer:
reasoning_text = ''.join(reasoning_buffer)
# 去掉 <think>...</think> 标签
cleaned = re.sub(r'', '', reasoning_text, flags=re.DOTALL).strip()
if cleaned:
logger.info("流式 LLM: content为空从reasoning_content提取内容")
yield cleaned
except Exception as e: except Exception as e:
logger.error(f"LLM 流式调用失败: {e}") logger.error(f"LLM 流式调用失败: {e}")

View File

@@ -331,6 +331,21 @@ class CollectionMixin:
except Exception as e: except Exception as e:
logger.warning(f"清理版本记录失败: {e}") logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
# 注意:此时 ChromaDB collection 已删除cleanup_image_orphans 会扫描
# 所有剩余 collection仅该 collection 引用的图片会被识别为孤儿
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self)
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存, "
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
if kb_name in self._metadata.get("collections", {}): if kb_name in self._metadata.get("collections", {}):
del self._metadata["collections"][kb_name] del self._metadata["collections"][kb_name]
self._save_metadata() self._save_metadata()

View File

@@ -72,6 +72,18 @@ class DocumentMixin:
except Exception as e: except Exception as e:
logger.warning(f"清理版本记录失败: {e}") logger.warning(f"清理版本记录失败: {e}")
# 清理不再被引用的图片和 VLM 缓存文件
try:
from knowledge.image_cleanup import cleanup_image_orphans
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
logger.info(
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
f"{cleanup_result['deleted_caches']} VLM缓存"
)
except Exception as e:
logger.warning(f"清理孤儿文件失败: {e}")
logger.info(f"{kb_name} 删除文档: {filename}, 片段数: {deleted}") logger.info(f"{kb_name} 删除文档: {filename}, 片段数: {deleted}")
return deleted return deleted

View File

@@ -25,7 +25,20 @@ def compute_file_hash(file_path: str) -> str:
return hashlib.md5(file_path.encode()).hexdigest() return hashlib.md5(file_path.encode()).hexdigest()
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None) -> str: def _get_embedding_model():
"""从 RAGEngine 获取 embedding 模型KnowledgeBaseManager 上没有此属性)"""
try:
from core.engine import get_engine
engine = get_engine()
if not engine._initialized:
engine.initialize()
return engine.embedding_model
except Exception as e:
logger.warning(f"获取 embedding 模型失败: {e}")
return None
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None, defer_chromadb: bool = False) -> str:
""" """
懒加载 VLM 描述 懒加载 VLM 描述
@@ -36,6 +49,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
image_path: 图片路径(相对路径或绝对路径) image_path: 图片路径(相对路径或绝对路径)
kb_name: 知识库名称 kb_name: 知识库名称
metadata: 图片元数据(包含 section、page、caption、上下文等 metadata: 图片元数据(包含 section、page、caption、上下文等
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
Returns: Returns:
VLM 生成的图片描述 VLM 生成的图片描述
@@ -49,23 +63,45 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
else: else:
full_image_path = image_path full_image_path = image_path
# 1. 检查缓存 # 1. 检查缓存(空缓存视为无效,需重新生成)
img_hash = compute_file_hash(full_image_path) img_hash = compute_file_hash(full_image_path)
cache_file = VLM_CACHE_DIR / f"{img_hash}.txt" cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
if cache_file.exists(): if cache_file.exists():
logger.info(f"VLM 缓存命中: {image_path}") cached = cache_file.read_text(encoding='utf-8')
return cache_file.read_text(encoding='utf-8') if len(cached.strip()) >= 5:
logger.info(f"VLM 缓存命中: {image_path}")
return cached
else:
logger.warning(f"VLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {image_path}")
try:
cache_file.unlink()
except OSError:
pass
# 2. 调用 VLM传入元数据 # 2. 调用 VLM传入元数据
logger.info(f"VLM 懒加载: {image_path}") logger.info(f"VLM 懒加载: {image_path}")
kb_manager = get_kb_manager() kb_manager = get_kb_manager()
description = kb_manager._generate_image_description(full_image_path, metadata=metadata) description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
# 3. 写入缓存 # 3. 空描述保护VLM 返回内容过短时不写入缓存和向量库
if not description or len(description.strip()) < 5:
logger.warning(f"VLM 返回描述过短({len(description.strip()) if description else 0}字符),跳过缓存和向量库更新: {image_path}")
return description or ''
# 4. 写入缓存
VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True) VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
cache_file.write_text(description, encoding='utf-8') cache_file.write_text(description, encoding='utf-8')
# 4. 更新向量库metadata + embedding # 5. 更新向量库metadata + embedding,需校验 chunk_id 非空
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
if defer_chromadb:
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
return description
if not chunk_id:
logger.warning("chunk_id 为空,跳过向量库更新")
return description
try: try:
collection = kb_manager.get_collection(kb_name) collection = kb_manager.get_collection(kb_name)
result = collection.get(ids=[chunk_id], include=['metadatas']) result = collection.get(ids=[chunk_id], include=['metadatas'])
@@ -79,7 +115,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
# 更新 embedding使用 VLM 描述重新计算向量) # 更新 embedding使用 VLM 描述重新计算向量)
# 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索 # 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索
embedding_model = kb_manager.embedding_model embedding_model = _get_embedding_model()
if embedding_model: if embedding_model:
new_vector = embedding_model.encode(description).tolist() new_vector = embedding_model.encode(description).tolist()
if isinstance(new_vector[0], list): if isinstance(new_vector[0], list):
@@ -91,20 +127,21 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
embeddings=[new_vector], embeddings=[new_vector],
documents=[description] # 同时更新 document 字段 documents=[description] # 同时更新 document 字段
) )
logger.info(f"已更新向量库 embedding: {chunk_id}") logger.info(f"已更新向量库(embedding+metadata): {chunk_id}")
else: else:
# 无 embedding 模型时只更新 metadata # 无 embedding 模型时只更新 metadata
collection.update( collection.update(
ids=[chunk_id], ids=[chunk_id],
metadatas=[new_metadata] metadatas=[new_metadata]
) )
logger.info(f"已更新向量库(仅metadata,无embedding模型): {chunk_id}")
except Exception as e: except Exception as e:
logger.warning(f"更新向量库失败: {e}") logger.warning(f"更新向量库失败: {e}")
return description return description
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str: async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str, defer_chromadb: bool = False) -> str:
""" """
懒加载表格摘要 懒加载表格摘要
@@ -114,50 +151,76 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
chunk_id: 切片 ID chunk_id: 切片 ID
table_md: 表格 Markdown 内容 table_md: 表格 Markdown 内容
kb_name: 知识库名称 kb_name: 知识库名称
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
Returns: Returns:
LLM 生成的表格摘要 LLM 生成的表格摘要
""" """
from knowledge.manager import get_kb_manager from knowledge.manager import get_kb_manager
# 1. 检查缓存 # 1. 检查缓存(空缓存视为无效)
table_hash = hashlib.md5(table_md.encode()).hexdigest() table_hash = hashlib.md5(table_md.encode()).hexdigest()
cache_file = LLM_CACHE_DIR / f"{table_hash}.txt" cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
if cache_file.exists(): if cache_file.exists():
logger.info(f"LLM 缓存命中: {chunk_id}") cached = cache_file.read_text(encoding='utf-8')
return cache_file.read_text(encoding='utf-8') if len(cached.strip()) >= 5:
logger.info(f"LLM 缓存命中: {chunk_id}")
return cached
else:
logger.warning(f"LLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {chunk_id}")
try:
cache_file.unlink()
except OSError:
pass
# 2. 调用 LLM # 2. 调用 LLM
logger.info(f"LLM 懒加载: {chunk_id}") logger.info(f"LLM 懒加载: {chunk_id}")
kb_manager = get_kb_manager() kb_manager = get_kb_manager()
summary = kb_manager._generate_table_summary(table_md, None) summary = kb_manager._generate_table_summary(table_md, None)
# 空摘要保护
if not summary or len(summary.strip()) < 5:
logger.warning(f"LLM 返回摘要过短,跳过缓存和向量库更新: {chunk_id}")
return summary or ''
# 3. 写入缓存 # 3. 写入缓存
LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True) LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
cache_file.write_text(summary, encoding='utf-8') cache_file.write_text(summary, encoding='utf-8')
# 4. 更新向量库(可选) # 4. 更新向量库,需校验 chunk_id 非空
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
if defer_chromadb:
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
return summary
if not chunk_id:
logger.warning("chunk_id 为空,跳过表格向量库更新")
return summary
try: try:
collection = kb_manager.get_collection(kb_name) collection = kb_manager.get_collection(kb_name)
result = collection.get(ids=[chunk_id], include=['metadatas']) result = collection.get(ids=[chunk_id], include=['metadatas'])
if result['metadatas']: if result['metadatas']:
# 新增摘要切片 # 新增摘要切片(需要 embedding 模型)
embedding_model = kb_manager.embedding_model embedding_model = _get_embedding_model()
vector = embedding_model.encode(summary).tolist() if embedding_model:
if isinstance(vector[0], list): vector = embedding_model.encode(summary).tolist()
vector = vector[0] if isinstance(vector[0], list):
vector = vector[0]
collection.add( collection.add(
ids=[f"{chunk_id}_summary"], ids=[f"{chunk_id}_summary"],
embeddings=[vector], embeddings=[vector],
documents=[summary], documents=[summary],
metadatas=[{ metadatas=[{
**result['metadatas'][0], **result['metadatas'][0],
'is_summary': True, 'is_summary': True,
'original_doc_id': chunk_id 'original_doc_id': chunk_id
}] }]
) )
# 更新原切片标记 logger.info(f"已新增摘要切片(embedding): {chunk_id}_summary")
else:
logger.info(f"跳过摘要切片(无embedding模型): {chunk_id}")
# 更新原切片标记(不依赖 embedding 模型)
collection.update( collection.update(
ids=[chunk_id], ids=[chunk_id],
metadatas=[{**result['metadatas'][0], 'has_summary': True}] metadatas=[{**result['metadatas'][0], 'has_summary': True}]
@@ -168,7 +231,7 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
return summary return summary
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str): async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str, defer_chromadb: bool = False):
""" """
检索后增强:按需调用 LLM/VLM 检索后增强:按需调用 LLM/VLM
@@ -176,23 +239,24 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
contexts: 检索上下文列表 contexts: 检索上下文列表
query: 用户查询 query: 用户查询
kb_name: 知识库名称 kb_name: 知识库名称
defer_chromadb: 为 True 时后台线程只写文件缓存,不更新 ChromaDB避免写锁竞争
""" """
for ctx in contexts: import re
meta = ctx.get('meta', {})
chunk_type = meta.get('chunk_type', 'text')
image_path = meta.get('image_path', '')
# 图片切片:懒加载 VLM 描述 for ctx in contexts:
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'): try:
if image_path: meta = ctx.get('meta', {})
try: chunk_type = meta.get('chunk_type', 'text')
image_path = meta.get('image_path', '')
# 图片切片:懒加载 VLM 描述
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
if image_path:
# 从 doc 字段中提取图号(上下文可能包含"见图2.5"等) # 从 doc 字段中提取图号(上下文可能包含"见图2.5"等)
doc_text = ctx.get('doc', '') doc_text = ctx.get('doc', '')
import re
# 提取图号(从前文/后文中) # 提取图号(从前文/后文中)
figure_number = "" figure_number = ""
# 匹配 "见图2.5"、"图2.5"、"见图 2.5" 等
fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text) fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
if fig_match: if fig_match:
figure_number = fig_match.group(1) figure_number = fig_match.group(1)
@@ -210,78 +274,73 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
'page': meta.get('page'), 'page': meta.get('page'),
'caption': meta.get('caption', ''), 'caption': meta.get('caption', ''),
'source': meta.get('source', ''), 'source': meta.get('source', ''),
'figure_number': figure_number, # 添加提取的图号 'figure_number': figure_number,
'doc_text': doc_text # 添加完整文档文本 'doc_text': doc_text
} }
vlm_desc = await lazy_vlm_description( vlm_desc = await lazy_vlm_description(
meta.get('id', ''), meta.get('chunk_id', ''),
image_path, image_path,
kb_name, kb_name,
metadata=image_metadata metadata=image_metadata,
defer_chromadb=defer_chromadb
) )
ctx['doc'] = vlm_desc if vlm_desc:
ctx['vlm_enhanced'] = True ctx['doc'] = vlm_desc
except Exception as e: ctx['vlm_enhanced'] = True
logger.warning(f"VLM 懒加载失败: {e}")
# 表格切片:同时处理摘要和关联图片的 VLM 描述 # 表格切片:同时处理摘要和关联图片的 VLM 描述
elif chunk_type == 'table': elif chunk_type == 'table':
doc_text = ctx.get('doc', '') doc_text = ctx.get('doc', '')
# 1. 懒加载表格摘要(高分切片) # 1. 懒加载表格摘要(高分切片)
if not meta.get('has_summary'): if not meta.get('has_summary'):
score = meta.get('score', 0) score = ctx.get('score', 0)
if score > 0.7: # 只对高相关表格生成摘要 if score > 0.7:
try:
summary = await lazy_table_summary( summary = await lazy_table_summary(
meta.get('id', ''), meta.get('chunk_id', ''),
doc_text, doc_text,
kb_name kb_name,
defer_chromadb=defer_chromadb
) )
# 摘要作为补充信息 if summary:
ctx['summary'] = summary ctx['summary'] = summary
ctx['llm_enhanced'] = True ctx['llm_enhanced'] = True
except Exception as e:
logger.warning(f"表格摘要懒加载失败: {e}")
# 2. 表格有关联图片时,懒加载 VLM 描述
if image_path and not meta.get('has_vlm_desc'):
try:
import re
# 2. 表格有关联图片时,懒加载 VLM 描述
if image_path and not meta.get('has_vlm_desc'):
# 提取表号(如 "表2.2"、"见表2.1" # 提取表号(如 "表2.2"、"见表2.1"
table_number = "" table_number = ""
# 匹配 "表2.2"、"见表2.2"、"见表 2.2" 等
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text) table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
if table_match: if table_match:
table_number = table_match.group(1) table_number = table_match.group(1)
# 如果 doc 中没有,尝试从 section 中提取
section = meta.get('section') or meta.get('section_path', '') section = meta.get('section') or meta.get('section_path', '')
if not table_number and section: if not table_number and section:
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section) table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
if table_match: if table_match:
table_number = table_match.group(1) table_number = table_match.group(1)
# 构建表格图片元数据
table_image_metadata = { table_image_metadata = {
'section': section, 'section': section,
'page': meta.get('page'), 'page': meta.get('page'),
'caption': meta.get('caption', ''), 'caption': meta.get('caption', ''),
'source': meta.get('source', ''), 'source': meta.get('source', ''),
'table_number': table_number, # 表号 'table_number': table_number,
'figure_number': table_number, # 兼容字段 'figure_number': table_number,
'doc_text': doc_text, 'doc_text': doc_text,
'is_table': True # 标记为表格图片 'is_table': True
} }
vlm_desc = await lazy_vlm_description( vlm_desc = await lazy_vlm_description(
meta.get('id', ''), meta.get('chunk_id', ''),
image_path, image_path,
kb_name, kb_name,
metadata=table_image_metadata metadata=table_image_metadata,
defer_chromadb=defer_chromadb
) )
# 表格图片描述作为补充信息 if vlm_desc:
ctx['image_description'] = vlm_desc ctx['image_description'] = vlm_desc
ctx['vlm_enhanced'] = True ctx['vlm_enhanced'] = True
except Exception as e:
logger.warning(f"表格图片 VLM 懒加载失败: {e}") except Exception as e:
chunk_id = ctx.get('meta', {}).get('chunk_id', '?')
logger.warning(f"增强切片失败(chunk_id={chunk_id}): {e}")

View File

@@ -622,7 +622,7 @@ class KnowledgeBaseManager(
try: try:
from config import get_llm_client, DASHSCOPE_MODEL from config import get_llm_client, DASHSCOPE_MODEL
client = get_llm_client() client = get_llm_client()
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512) summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
return summary.strip() if summary else "" return summary.strip() if summary else ""
except Exception as e: except Exception as e:
logger.warning(f"生成表格摘要失败: {e}") logger.warning(f"生成表格摘要失败: {e}")
@@ -681,13 +681,31 @@ class KnowledgeBaseManager(
] ]
} }
], ],
max_tokens=512 max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token需留足输出空间
) )
description = response.choices[0].message.content description = response.choices[0].message.content
# 推理模型兼容content 为空时从 reasoning_content 提取
if not description or not description.strip():
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
if reasoning and reasoning.strip():
import re
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
if cleaned:
logger.info(f"VLM content为空从reasoning_content提取描述: {image_path}")
description = cleaned
else:
description = reasoning.strip()
if not description:
logger.warning(f"VLM 返回空描述: {image_path}")
return ""
# 缓存结果 # 缓存结果
import hashlib import hashlib
import re as _re
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest() img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
cache_dir = Path('.data/cache/vlm') cache_dir = Path('.data/cache/vlm')
cache_dir.mkdir(parents=True, exist_ok=True) cache_dir.mkdir(parents=True, exist_ok=True)