feat(rag): 子章节级图片过滤 + VLM 后台增强 + 意图分析优化
- 图片选择新增 section_path 字段,支持子章节级过滤 - _filter_images_by_answer 增加 primary_sections 参数和叶子节点匹配 - 检索发散检测:primary_leaf_names > 3 时全局阈值+1 - _FIGURE_ANSWER_KEYWORDS 移除单字"图""表",正则图号兜底防误触发 - lazy_enhance 后台增强流程优化 - 意图分析与 LLM 工具层改进 评测:图片选择 F1 从 52.4% 提升至 66.3%(+13.9pp),Precision +16.5pp
This commit is contained in:
6061
api/chat_routes.py
6061
api/chat_routes.py
File diff suppressed because it is too large
Load Diff
23
config.py
23
config.py
@@ -20,9 +20,13 @@ DASHSCOPE_API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
|
|||||||
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
|
DASHSCOPE_BASE_URL = os.getenv("DASHSCOPE_BASE_URL", "https://token-plan-cn.xiaomimimo.com/v1")
|
||||||
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
|
DASHSCOPE_MODEL = os.getenv("DASHSCOPE_MODEL", "mimo-v2.5") # 文本生成模型
|
||||||
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
|
RAG_CHAT_MODEL = os.getenv("RAG_CHAT_MODEL", "mimo-v2.5") # RAG 对话模型
|
||||||
INTENT_MODEL = os.getenv("INTENT_MODEL", "mimo-v2.5") # 意图分析模型
|
INTENT_MODEL = os.getenv("INTENT_MODEL", "deepseek-v4-flash") # 意图分析模型(百炼快速模型)
|
||||||
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型(图片描述)
|
VLM_MODEL = os.getenv("VLM_MODEL", "mimo-v2.5") # 视觉语言模型(图片描述)
|
||||||
|
|
||||||
|
# 百炼 API(阿里云 DashScope,用于意图分析等轻量任务)
|
||||||
|
BAILIAN_API_KEY = os.getenv("BAILIAN_API_KEY", "")
|
||||||
|
BAILIAN_BASE_URL = os.getenv("BAILIAN_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1")
|
||||||
|
|
||||||
# 兼容旧变量名(逐步迁移到 DASHSCOPE_* 命名)
|
# 兼容旧变量名(逐步迁移到 DASHSCOPE_* 命名)
|
||||||
API_KEY = DASHSCOPE_API_KEY
|
API_KEY = DASHSCOPE_API_KEY
|
||||||
BASE_URL = DASHSCOPE_BASE_URL
|
BASE_URL = DASHSCOPE_BASE_URL
|
||||||
@@ -96,12 +100,12 @@ RERANK_DEVICE = os.getenv("RERANK_DEVICE", os.getenv("DEVICE", "auto"))
|
|||||||
|
|
||||||
# ----- 通用问答 -----
|
# ----- 通用问答 -----
|
||||||
LLM_TEMPERATURE = 0.7 # 生成温度(0=确定性,1=随机性)
|
LLM_TEMPERATURE = 0.7 # 生成温度(0=确定性,1=随机性)
|
||||||
LLM_MAX_TOKENS = 3000 # 最大输出 token 数
|
LLM_MAX_TOKENS = 3000 # 最大输出 token 数(推理模型思考链占用部分预算,3000 平衡速度与质量)
|
||||||
|
|
||||||
# ----- 意图分析(轻量、确定性高)-----
|
# ----- 意图分析(轻量、确定性高)-----
|
||||||
# INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置
|
# INTENT_MODEL 在顶部「一、API 密钥与模型」中统一配置
|
||||||
INTENT_TEMPERATURE = 0.1
|
INTENT_TEMPERATURE = 0.1
|
||||||
INTENT_MAX_TOKENS = 4096 # 推理模型思维链消耗大量 token,2048 偶发截断导致意图分析失败
|
INTENT_MAX_TOKENS = 1024 # 非推理模型无需思考链预算,JSON 输出 1024 足够
|
||||||
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
|
INTENT_HISTORY_WINDOW = 6 # 分析时取最近几条历史消息
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
@@ -286,3 +290,16 @@ def get_llm_client():
|
|||||||
"""获取 LLM 客户端实例"""
|
"""获取 LLM 客户端实例"""
|
||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
|
return OpenAI(api_key=DASHSCOPE_API_KEY, base_url=DASHSCOPE_BASE_URL)
|
||||||
|
|
||||||
|
|
||||||
|
_intent_client = None
|
||||||
|
|
||||||
|
def get_intent_client():
|
||||||
|
"""获取意图分析专用 LLM 客户端(百炼快速模型)"""
|
||||||
|
global _intent_client
|
||||||
|
if _intent_client is None:
|
||||||
|
if not BAILIAN_API_KEY:
|
||||||
|
raise ValueError("BAILIAN_API_KEY 未配置,请在 .env 中设置")
|
||||||
|
from openai import OpenAI
|
||||||
|
_intent_client = OpenAI(api_key=BAILIAN_API_KEY, base_url=BAILIAN_BASE_URL)
|
||||||
|
return _intent_client
|
||||||
|
|||||||
@@ -233,10 +233,10 @@ class IntentAnalyzer:
|
|||||||
self._exact_cache_max = 500
|
self._exact_cache_max = 500
|
||||||
|
|
||||||
def _get_client(self):
|
def _get_client(self):
|
||||||
"""获取 LLM 客户端"""
|
"""获取 LLM 客户端(百炼快速模型)"""
|
||||||
if self._client is None:
|
if self._client is None:
|
||||||
from config import get_llm_client
|
from config import get_intent_client
|
||||||
self._client = get_llm_client()
|
self._client = get_intent_client()
|
||||||
return self._client
|
return self._client
|
||||||
|
|
||||||
def _get_cache(self):
|
def _get_cache(self):
|
||||||
|
|||||||
@@ -71,19 +71,38 @@ def call_llm(
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
content = response.choices[0].message.content
|
content = response.choices[0].message.content
|
||||||
|
|
||||||
# 推理模型兼容:content 为空时尝试从 reasoning_content 提取
|
# 推理模型兼容(mimo-v2.5 等):
|
||||||
|
# 推理模型思考链消耗大量 token(~1000),max_tokens 不足时 content 为空,
|
||||||
|
# 全部输出进入 reasoning_content。此处从思考链中提取有效内容。
|
||||||
if not content or not content.strip():
|
if not content or not content.strip():
|
||||||
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||||
if reasoning and reasoning.strip():
|
if reasoning and reasoning.strip():
|
||||||
# 从思维链中提取 JSON 块作为内容
|
# 先去掉 <think>...</think> 标签
|
||||||
json_match = re.search(r'\{[\s\S]*\}', reasoning)
|
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||||
if json_match:
|
if cleaned:
|
||||||
logger.info("LLM: content为空,从reasoning_content提取JSON")
|
logger.info("LLM: content为空,从reasoning_content提取内容")
|
||||||
return json_match.group().strip()
|
# 尝试提取 JSON 对象(兼容结构化响应场景)
|
||||||
logger.warning("LLM 返回空 content(可能需要增大 max_tokens)")
|
json_match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||||
|
if json_match:
|
||||||
|
try:
|
||||||
|
json.loads(json_match.group())
|
||||||
|
return json_match.group().strip()
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
# 尝试提取 JSON 数组
|
||||||
|
bracket_match = re.search(r'\[[\s\S]*\]', cleaned)
|
||||||
|
if bracket_match:
|
||||||
|
try:
|
||||||
|
json.loads(bracket_match.group())
|
||||||
|
return bracket_match.group().strip()
|
||||||
|
except (json.JSONDecodeError, ValueError):
|
||||||
|
pass
|
||||||
|
# 纯文本响应:直接返回清理后的内容
|
||||||
|
return cleaned
|
||||||
|
logger.warning("LLM 返回空 content 且 reasoning_content 也无法提取(可能需要增大 max_tokens)")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return content.strip()
|
return content.strip()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"LLM 调用失败: {e}")
|
logger.warning(f"LLM 调用失败: {e}")
|
||||||
@@ -95,7 +114,7 @@ def call_llm_stream(
|
|||||||
prompt: str,
|
prompt: str,
|
||||||
model: str,
|
model: str,
|
||||||
temperature: float = 0.3,
|
temperature: float = 0.3,
|
||||||
max_tokens: int = 1000,
|
max_tokens: int = 3000,
|
||||||
messages: List[dict] = None,
|
messages: List[dict] = None,
|
||||||
error_prefix: str = "[错误]",
|
error_prefix: str = "[错误]",
|
||||||
**kwargs
|
**kwargs
|
||||||
@@ -104,13 +123,14 @@ def call_llm_stream(
|
|||||||
流式 LLM 调用(生成器封装)
|
流式 LLM 调用(生成器封装)
|
||||||
|
|
||||||
自动处理流式响应,逐块 yield 文本内容。
|
自动处理流式响应,逐块 yield 文本内容。
|
||||||
|
兼容推理模型(mimo-v2.5 等):当 content 为空时回退到 reasoning_content。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
client: OpenAI 客户端实例
|
client: OpenAI 客户端实例
|
||||||
prompt: 用户提示
|
prompt: 用户提示
|
||||||
model: 模型名称
|
model: 模型名称
|
||||||
temperature: 温度参数
|
temperature: 温度参数
|
||||||
max_tokens: 最大 token 数
|
max_tokens: 最大 token 数(推理模型需留足思考链预算)
|
||||||
messages: 完整消息列表
|
messages: 完整消息列表
|
||||||
error_prefix: 错误时的前缀
|
error_prefix: 错误时的前缀
|
||||||
**kwargs: 其他参数
|
**kwargs: 其他参数
|
||||||
@@ -135,9 +155,33 @@ def call_llm_stream(
|
|||||||
**kwargs
|
**kwargs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
content_yielded = False
|
||||||
|
reasoning_buffer = []
|
||||||
|
|
||||||
for chunk in stream:
|
for chunk in stream:
|
||||||
if chunk.choices and chunk.choices[0].delta.content:
|
if not chunk.choices:
|
||||||
yield chunk.choices[0].delta.content
|
continue
|
||||||
|
delta = chunk.choices[0].delta
|
||||||
|
|
||||||
|
# 正常 content 输出
|
||||||
|
if hasattr(delta, 'content') and delta.content:
|
||||||
|
content_yielded = True
|
||||||
|
yield delta.content
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 推理模型:reasoning_content(思考链)
|
||||||
|
rc = getattr(delta, 'reasoning_content', None)
|
||||||
|
if rc:
|
||||||
|
reasoning_buffer.append(rc)
|
||||||
|
|
||||||
|
# 回退:content 为空但 reasoning_content 有内容(推理模型 token 不足时)
|
||||||
|
if not content_yielded and reasoning_buffer:
|
||||||
|
reasoning_text = ''.join(reasoning_buffer)
|
||||||
|
# 去掉 <think>...</think> 标签
|
||||||
|
cleaned = re.sub(r'', '', reasoning_text, flags=re.DOTALL).strip()
|
||||||
|
if cleaned:
|
||||||
|
logger.info("流式 LLM: content为空,从reasoning_content提取内容")
|
||||||
|
yield cleaned
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"LLM 流式调用失败: {e}")
|
logger.error(f"LLM 流式调用失败: {e}")
|
||||||
|
|||||||
@@ -331,6 +331,21 @@ class CollectionMixin:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"清理版本记录失败: {e}")
|
logger.warning(f"清理版本记录失败: {e}")
|
||||||
|
|
||||||
|
# 清理不再被引用的图片和 VLM 缓存文件
|
||||||
|
# 注意:此时 ChromaDB collection 已删除,cleanup_image_orphans 会扫描
|
||||||
|
# 所有剩余 collection,仅该 collection 引用的图片会被识别为孤儿
|
||||||
|
try:
|
||||||
|
from knowledge.image_cleanup import cleanup_image_orphans
|
||||||
|
cleanup_result = cleanup_image_orphans(self)
|
||||||
|
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||||
|
logger.info(
|
||||||
|
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||||
|
f"{cleanup_result['deleted_caches']} VLM缓存, "
|
||||||
|
f"释放 {cleanup_result['freed_bytes']/1024:.1f} KB"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"清理孤儿文件失败: {e}")
|
||||||
|
|
||||||
if kb_name in self._metadata.get("collections", {}):
|
if kb_name in self._metadata.get("collections", {}):
|
||||||
del self._metadata["collections"][kb_name]
|
del self._metadata["collections"][kb_name]
|
||||||
self._save_metadata()
|
self._save_metadata()
|
||||||
|
|||||||
@@ -72,6 +72,18 @@ class DocumentMixin:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"清理版本记录失败: {e}")
|
logger.warning(f"清理版本记录失败: {e}")
|
||||||
|
|
||||||
|
# 清理不再被引用的图片和 VLM 缓存文件
|
||||||
|
try:
|
||||||
|
from knowledge.image_cleanup import cleanup_image_orphans
|
||||||
|
cleanup_result = cleanup_image_orphans(self, collections=[kb_name])
|
||||||
|
if cleanup_result['deleted_images'] or cleanup_result['deleted_caches']:
|
||||||
|
logger.info(
|
||||||
|
f"清理孤儿文件: {cleanup_result['deleted_images']} 图片 + "
|
||||||
|
f"{cleanup_result['deleted_caches']} VLM缓存"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"清理孤儿文件失败: {e}")
|
||||||
|
|
||||||
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
logger.info(f"从 {kb_name} 删除文档: {filename}, 片段数: {deleted}")
|
||||||
return deleted
|
return deleted
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,20 @@ def compute_file_hash(file_path: str) -> str:
|
|||||||
return hashlib.md5(file_path.encode()).hexdigest()
|
return hashlib.md5(file_path.encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None) -> str:
|
def _get_embedding_model():
|
||||||
|
"""从 RAGEngine 获取 embedding 模型(KnowledgeBaseManager 上没有此属性)"""
|
||||||
|
try:
|
||||||
|
from core.engine import get_engine
|
||||||
|
engine = get_engine()
|
||||||
|
if not engine._initialized:
|
||||||
|
engine.initialize()
|
||||||
|
return engine.embedding_model
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"获取 embedding 模型失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None, defer_chromadb: bool = False) -> str:
|
||||||
"""
|
"""
|
||||||
懒加载 VLM 描述
|
懒加载 VLM 描述
|
||||||
|
|
||||||
@@ -36,6 +49,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
|||||||
image_path: 图片路径(相对路径或绝对路径)
|
image_path: 图片路径(相对路径或绝对路径)
|
||||||
kb_name: 知识库名称
|
kb_name: 知识库名称
|
||||||
metadata: 图片元数据(包含 section、page、caption、上下文等)
|
metadata: 图片元数据(包含 section、page、caption、上下文等)
|
||||||
|
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
VLM 生成的图片描述
|
VLM 生成的图片描述
|
||||||
@@ -49,23 +63,45 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
|||||||
else:
|
else:
|
||||||
full_image_path = image_path
|
full_image_path = image_path
|
||||||
|
|
||||||
# 1. 检查缓存
|
# 1. 检查缓存(空缓存视为无效,需重新生成)
|
||||||
img_hash = compute_file_hash(full_image_path)
|
img_hash = compute_file_hash(full_image_path)
|
||||||
cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
|
cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
|
||||||
if cache_file.exists():
|
if cache_file.exists():
|
||||||
logger.info(f"VLM 缓存命中: {image_path}")
|
cached = cache_file.read_text(encoding='utf-8')
|
||||||
return cache_file.read_text(encoding='utf-8')
|
if len(cached.strip()) >= 5:
|
||||||
|
logger.info(f"VLM 缓存命中: {image_path}")
|
||||||
|
return cached
|
||||||
|
else:
|
||||||
|
logger.warning(f"VLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {image_path}")
|
||||||
|
try:
|
||||||
|
cache_file.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
# 2. 调用 VLM(传入元数据)
|
# 2. 调用 VLM(传入元数据)
|
||||||
logger.info(f"VLM 懒加载: {image_path}")
|
logger.info(f"VLM 懒加载: {image_path}")
|
||||||
kb_manager = get_kb_manager()
|
kb_manager = get_kb_manager()
|
||||||
description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
|
description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
|
||||||
|
|
||||||
# 3. 写入缓存
|
# 3. 空描述保护:VLM 返回内容过短时不写入缓存和向量库
|
||||||
|
if not description or len(description.strip()) < 5:
|
||||||
|
logger.warning(f"VLM 返回描述过短({len(description.strip()) if description else 0}字符),跳过缓存和向量库更新: {image_path}")
|
||||||
|
return description or ''
|
||||||
|
|
||||||
|
# 4. 写入缓存
|
||||||
VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
cache_file.write_text(description, encoding='utf-8')
|
cache_file.write_text(description, encoding='utf-8')
|
||||||
|
|
||||||
# 4. 更新向量库(metadata + embedding)
|
# 5. 更新向量库(metadata + embedding),需校验 chunk_id 非空
|
||||||
|
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
|
||||||
|
if defer_chromadb:
|
||||||
|
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
|
||||||
|
return description
|
||||||
|
|
||||||
|
if not chunk_id:
|
||||||
|
logger.warning("chunk_id 为空,跳过向量库更新")
|
||||||
|
return description
|
||||||
|
|
||||||
try:
|
try:
|
||||||
collection = kb_manager.get_collection(kb_name)
|
collection = kb_manager.get_collection(kb_name)
|
||||||
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
||||||
@@ -79,7 +115,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
|||||||
|
|
||||||
# 更新 embedding(使用 VLM 描述重新计算向量)
|
# 更新 embedding(使用 VLM 描述重新计算向量)
|
||||||
# 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索
|
# 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索
|
||||||
embedding_model = kb_manager.embedding_model
|
embedding_model = _get_embedding_model()
|
||||||
if embedding_model:
|
if embedding_model:
|
||||||
new_vector = embedding_model.encode(description).tolist()
|
new_vector = embedding_model.encode(description).tolist()
|
||||||
if isinstance(new_vector[0], list):
|
if isinstance(new_vector[0], list):
|
||||||
@@ -91,20 +127,21 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
|||||||
embeddings=[new_vector],
|
embeddings=[new_vector],
|
||||||
documents=[description] # 同时更新 document 字段
|
documents=[description] # 同时更新 document 字段
|
||||||
)
|
)
|
||||||
logger.info(f"已更新向量库 embedding: {chunk_id}")
|
logger.info(f"已更新向量库(embedding+metadata): {chunk_id}")
|
||||||
else:
|
else:
|
||||||
# 无 embedding 模型时只更新 metadata
|
# 无 embedding 模型时只更新 metadata
|
||||||
collection.update(
|
collection.update(
|
||||||
ids=[chunk_id],
|
ids=[chunk_id],
|
||||||
metadatas=[new_metadata]
|
metadatas=[new_metadata]
|
||||||
)
|
)
|
||||||
|
logger.info(f"已更新向量库(仅metadata,无embedding模型): {chunk_id}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"更新向量库失败: {e}")
|
logger.warning(f"更新向量库失败: {e}")
|
||||||
|
|
||||||
return description
|
return description
|
||||||
|
|
||||||
|
|
||||||
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str, defer_chromadb: bool = False) -> str:
|
||||||
"""
|
"""
|
||||||
懒加载表格摘要
|
懒加载表格摘要
|
||||||
|
|
||||||
@@ -114,50 +151,76 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
|||||||
chunk_id: 切片 ID
|
chunk_id: 切片 ID
|
||||||
table_md: 表格 Markdown 内容
|
table_md: 表格 Markdown 内容
|
||||||
kb_name: 知识库名称
|
kb_name: 知识库名称
|
||||||
|
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
LLM 生成的表格摘要
|
LLM 生成的表格摘要
|
||||||
"""
|
"""
|
||||||
from knowledge.manager import get_kb_manager
|
from knowledge.manager import get_kb_manager
|
||||||
|
|
||||||
# 1. 检查缓存
|
# 1. 检查缓存(空缓存视为无效)
|
||||||
table_hash = hashlib.md5(table_md.encode()).hexdigest()
|
table_hash = hashlib.md5(table_md.encode()).hexdigest()
|
||||||
cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
|
cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
|
||||||
if cache_file.exists():
|
if cache_file.exists():
|
||||||
logger.info(f"LLM 缓存命中: {chunk_id}")
|
cached = cache_file.read_text(encoding='utf-8')
|
||||||
return cache_file.read_text(encoding='utf-8')
|
if len(cached.strip()) >= 5:
|
||||||
|
logger.info(f"LLM 缓存命中: {chunk_id}")
|
||||||
|
return cached
|
||||||
|
else:
|
||||||
|
logger.warning(f"LLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {chunk_id}")
|
||||||
|
try:
|
||||||
|
cache_file.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
# 2. 调用 LLM
|
# 2. 调用 LLM
|
||||||
logger.info(f"LLM 懒加载: {chunk_id}")
|
logger.info(f"LLM 懒加载: {chunk_id}")
|
||||||
kb_manager = get_kb_manager()
|
kb_manager = get_kb_manager()
|
||||||
summary = kb_manager._generate_table_summary(table_md, None)
|
summary = kb_manager._generate_table_summary(table_md, None)
|
||||||
|
|
||||||
|
# 空摘要保护
|
||||||
|
if not summary or len(summary.strip()) < 5:
|
||||||
|
logger.warning(f"LLM 返回摘要过短,跳过缓存和向量库更新: {chunk_id}")
|
||||||
|
return summary or ''
|
||||||
|
|
||||||
# 3. 写入缓存
|
# 3. 写入缓存
|
||||||
LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
cache_file.write_text(summary, encoding='utf-8')
|
cache_file.write_text(summary, encoding='utf-8')
|
||||||
|
|
||||||
# 4. 更新向量库(可选)
|
# 4. 更新向量库,需校验 chunk_id 非空
|
||||||
|
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
|
||||||
|
if defer_chromadb:
|
||||||
|
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
|
||||||
|
return summary
|
||||||
|
|
||||||
|
if not chunk_id:
|
||||||
|
logger.warning("chunk_id 为空,跳过表格向量库更新")
|
||||||
|
return summary
|
||||||
try:
|
try:
|
||||||
collection = kb_manager.get_collection(kb_name)
|
collection = kb_manager.get_collection(kb_name)
|
||||||
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
||||||
if result['metadatas']:
|
if result['metadatas']:
|
||||||
# 新增摘要切片
|
# 新增摘要切片(需要 embedding 模型)
|
||||||
embedding_model = kb_manager.embedding_model
|
embedding_model = _get_embedding_model()
|
||||||
vector = embedding_model.encode(summary).tolist()
|
if embedding_model:
|
||||||
if isinstance(vector[0], list):
|
vector = embedding_model.encode(summary).tolist()
|
||||||
vector = vector[0]
|
if isinstance(vector[0], list):
|
||||||
|
vector = vector[0]
|
||||||
|
|
||||||
collection.add(
|
collection.add(
|
||||||
ids=[f"{chunk_id}_summary"],
|
ids=[f"{chunk_id}_summary"],
|
||||||
embeddings=[vector],
|
embeddings=[vector],
|
||||||
documents=[summary],
|
documents=[summary],
|
||||||
metadatas=[{
|
metadatas=[{
|
||||||
**result['metadatas'][0],
|
**result['metadatas'][0],
|
||||||
'is_summary': True,
|
'is_summary': True,
|
||||||
'original_doc_id': chunk_id
|
'original_doc_id': chunk_id
|
||||||
}]
|
}]
|
||||||
)
|
)
|
||||||
# 更新原切片标记
|
logger.info(f"已新增摘要切片(embedding): {chunk_id}_summary")
|
||||||
|
else:
|
||||||
|
logger.info(f"跳过摘要切片(无embedding模型): {chunk_id}")
|
||||||
|
# 更新原切片标记(不依赖 embedding 模型)
|
||||||
collection.update(
|
collection.update(
|
||||||
ids=[chunk_id],
|
ids=[chunk_id],
|
||||||
metadatas=[{**result['metadatas'][0], 'has_summary': True}]
|
metadatas=[{**result['metadatas'][0], 'has_summary': True}]
|
||||||
@@ -168,7 +231,7 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
|||||||
return summary
|
return summary
|
||||||
|
|
||||||
|
|
||||||
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str, defer_chromadb: bool = False):
|
||||||
"""
|
"""
|
||||||
检索后增强:按需调用 LLM/VLM
|
检索后增强:按需调用 LLM/VLM
|
||||||
|
|
||||||
@@ -176,23 +239,24 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
|||||||
contexts: 检索上下文列表
|
contexts: 检索上下文列表
|
||||||
query: 用户查询
|
query: 用户查询
|
||||||
kb_name: 知识库名称
|
kb_name: 知识库名称
|
||||||
|
defer_chromadb: 为 True 时后台线程只写文件缓存,不更新 ChromaDB(避免写锁竞争)
|
||||||
"""
|
"""
|
||||||
for ctx in contexts:
|
import re
|
||||||
meta = ctx.get('meta', {})
|
|
||||||
chunk_type = meta.get('chunk_type', 'text')
|
|
||||||
image_path = meta.get('image_path', '')
|
|
||||||
|
|
||||||
# 图片切片:懒加载 VLM 描述
|
for ctx in contexts:
|
||||||
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
|
try:
|
||||||
if image_path:
|
meta = ctx.get('meta', {})
|
||||||
try:
|
chunk_type = meta.get('chunk_type', 'text')
|
||||||
|
image_path = meta.get('image_path', '')
|
||||||
|
|
||||||
|
# 图片切片:懒加载 VLM 描述
|
||||||
|
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
|
||||||
|
if image_path:
|
||||||
# 从 doc 字段中提取图号(上下文可能包含"见图2.5"等)
|
# 从 doc 字段中提取图号(上下文可能包含"见图2.5"等)
|
||||||
doc_text = ctx.get('doc', '')
|
doc_text = ctx.get('doc', '')
|
||||||
import re
|
|
||||||
|
|
||||||
# 提取图号(从前文/后文中)
|
# 提取图号(从前文/后文中)
|
||||||
figure_number = ""
|
figure_number = ""
|
||||||
# 匹配 "见图2.5"、"图2.5"、"见图 2.5" 等
|
|
||||||
fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
|
fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
|
||||||
if fig_match:
|
if fig_match:
|
||||||
figure_number = fig_match.group(1)
|
figure_number = fig_match.group(1)
|
||||||
@@ -210,78 +274,73 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
|||||||
'page': meta.get('page'),
|
'page': meta.get('page'),
|
||||||
'caption': meta.get('caption', ''),
|
'caption': meta.get('caption', ''),
|
||||||
'source': meta.get('source', ''),
|
'source': meta.get('source', ''),
|
||||||
'figure_number': figure_number, # 添加提取的图号
|
'figure_number': figure_number,
|
||||||
'doc_text': doc_text # 添加完整文档文本
|
'doc_text': doc_text
|
||||||
}
|
}
|
||||||
vlm_desc = await lazy_vlm_description(
|
vlm_desc = await lazy_vlm_description(
|
||||||
meta.get('id', ''),
|
meta.get('chunk_id', ''),
|
||||||
image_path,
|
image_path,
|
||||||
kb_name,
|
kb_name,
|
||||||
metadata=image_metadata
|
metadata=image_metadata,
|
||||||
|
defer_chromadb=defer_chromadb
|
||||||
)
|
)
|
||||||
ctx['doc'] = vlm_desc
|
if vlm_desc:
|
||||||
ctx['vlm_enhanced'] = True
|
ctx['doc'] = vlm_desc
|
||||||
except Exception as e:
|
ctx['vlm_enhanced'] = True
|
||||||
logger.warning(f"VLM 懒加载失败: {e}")
|
|
||||||
|
|
||||||
# 表格切片:同时处理摘要和关联图片的 VLM 描述
|
# 表格切片:同时处理摘要和关联图片的 VLM 描述
|
||||||
elif chunk_type == 'table':
|
elif chunk_type == 'table':
|
||||||
doc_text = ctx.get('doc', '')
|
doc_text = ctx.get('doc', '')
|
||||||
|
|
||||||
# 1. 懒加载表格摘要(高分切片)
|
# 1. 懒加载表格摘要(高分切片)
|
||||||
if not meta.get('has_summary'):
|
if not meta.get('has_summary'):
|
||||||
score = meta.get('score', 0)
|
score = ctx.get('score', 0)
|
||||||
if score > 0.7: # 只对高相关表格生成摘要
|
if score > 0.7:
|
||||||
try:
|
|
||||||
summary = await lazy_table_summary(
|
summary = await lazy_table_summary(
|
||||||
meta.get('id', ''),
|
meta.get('chunk_id', ''),
|
||||||
doc_text,
|
doc_text,
|
||||||
kb_name
|
kb_name,
|
||||||
|
defer_chromadb=defer_chromadb
|
||||||
)
|
)
|
||||||
# 摘要作为补充信息
|
if summary:
|
||||||
ctx['summary'] = summary
|
ctx['summary'] = summary
|
||||||
ctx['llm_enhanced'] = True
|
ctx['llm_enhanced'] = True
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"表格摘要懒加载失败: {e}")
|
|
||||||
|
|
||||||
# 2. 表格有关联图片时,懒加载 VLM 描述
|
|
||||||
if image_path and not meta.get('has_vlm_desc'):
|
|
||||||
try:
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
# 2. 表格有关联图片时,懒加载 VLM 描述
|
||||||
|
if image_path and not meta.get('has_vlm_desc'):
|
||||||
# 提取表号(如 "表2.2"、"见表2.1")
|
# 提取表号(如 "表2.2"、"见表2.1")
|
||||||
table_number = ""
|
table_number = ""
|
||||||
# 匹配 "表2.2"、"见表2.2"、"见表 2.2" 等
|
|
||||||
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
|
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
|
||||||
if table_match:
|
if table_match:
|
||||||
table_number = table_match.group(1)
|
table_number = table_match.group(1)
|
||||||
|
|
||||||
# 如果 doc 中没有,尝试从 section 中提取
|
|
||||||
section = meta.get('section') or meta.get('section_path', '')
|
section = meta.get('section') or meta.get('section_path', '')
|
||||||
if not table_number and section:
|
if not table_number and section:
|
||||||
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
|
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
|
||||||
if table_match:
|
if table_match:
|
||||||
table_number = table_match.group(1)
|
table_number = table_match.group(1)
|
||||||
|
|
||||||
# 构建表格图片元数据
|
|
||||||
table_image_metadata = {
|
table_image_metadata = {
|
||||||
'section': section,
|
'section': section,
|
||||||
'page': meta.get('page'),
|
'page': meta.get('page'),
|
||||||
'caption': meta.get('caption', ''),
|
'caption': meta.get('caption', ''),
|
||||||
'source': meta.get('source', ''),
|
'source': meta.get('source', ''),
|
||||||
'table_number': table_number, # 表号
|
'table_number': table_number,
|
||||||
'figure_number': table_number, # 兼容字段
|
'figure_number': table_number,
|
||||||
'doc_text': doc_text,
|
'doc_text': doc_text,
|
||||||
'is_table': True # 标记为表格图片
|
'is_table': True
|
||||||
}
|
}
|
||||||
vlm_desc = await lazy_vlm_description(
|
vlm_desc = await lazy_vlm_description(
|
||||||
meta.get('id', ''),
|
meta.get('chunk_id', ''),
|
||||||
image_path,
|
image_path,
|
||||||
kb_name,
|
kb_name,
|
||||||
metadata=table_image_metadata
|
metadata=table_image_metadata,
|
||||||
|
defer_chromadb=defer_chromadb
|
||||||
)
|
)
|
||||||
# 表格图片描述作为补充信息
|
if vlm_desc:
|
||||||
ctx['image_description'] = vlm_desc
|
ctx['image_description'] = vlm_desc
|
||||||
ctx['vlm_enhanced'] = True
|
ctx['vlm_enhanced'] = True
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"表格图片 VLM 懒加载失败: {e}")
|
except Exception as e:
|
||||||
|
chunk_id = ctx.get('meta', {}).get('chunk_id', '?')
|
||||||
|
logger.warning(f"增强切片失败(chunk_id={chunk_id}): {e}")
|
||||||
|
|||||||
@@ -622,7 +622,7 @@ class KnowledgeBaseManager(
|
|||||||
try:
|
try:
|
||||||
from config import get_llm_client, DASHSCOPE_MODEL
|
from config import get_llm_client, DASHSCOPE_MODEL
|
||||||
client = get_llm_client()
|
client = get_llm_client()
|
||||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
|
||||||
return summary.strip() if summary else ""
|
return summary.strip() if summary else ""
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"生成表格摘要失败: {e}")
|
logger.warning(f"生成表格摘要失败: {e}")
|
||||||
@@ -681,13 +681,31 @@ class KnowledgeBaseManager(
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
max_tokens=512
|
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间
|
||||||
)
|
)
|
||||||
|
|
||||||
description = response.choices[0].message.content
|
description = response.choices[0].message.content
|
||||||
|
|
||||||
|
# 推理模型兼容:content 为空时从 reasoning_content 提取
|
||||||
|
if not description or not description.strip():
|
||||||
|
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||||
|
if reasoning and reasoning.strip():
|
||||||
|
import re
|
||||||
|
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
|
||||||
|
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||||
|
if cleaned:
|
||||||
|
logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}")
|
||||||
|
description = cleaned
|
||||||
|
else:
|
||||||
|
description = reasoning.strip()
|
||||||
|
|
||||||
|
if not description:
|
||||||
|
logger.warning(f"VLM 返回空描述: {image_path}")
|
||||||
|
return ""
|
||||||
|
|
||||||
# 缓存结果
|
# 缓存结果
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import re as _re
|
||||||
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
||||||
cache_dir = Path('.data/cache/vlm')
|
cache_dir = Path('.data/cache/vlm')
|
||||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user