feat(server-release): 从 main 同步子章节级图片过滤 + VLM/LLM 推理模型兼容
- chat_routes: 子章节级图片过滤(section_path 传递、叶子节点匹配、发散检测) - chat_routes: _FIGURE_ANSWER_KEYWORDS 移除单字"图""表",正则图号兜底 - lazy_enhance: defer_chromadb 参数避免后台线程 SQLite 写锁竞争 - lazy_enhance: 缓存内容校验(>=5字)和空结果保护 - llm_utils: reasoning_content 兜底(剥离 <think> 标签)+ 流式 reasoning 支持 - intent_analyzer: SYSTEM_PROMPT 增加追问补全逻辑 - manager: VLM 描述/摘要 max_tokens 提升至 2048 + Windows fcntl 兼容 不改变端口、API 接口和响应数据字段。
This commit is contained in:
@@ -1226,23 +1226,46 @@ def score_image_relevance(query: str, meta: Dict, doc: str = '') -> float:
|
||||
return score
|
||||
|
||||
|
||||
def _filter_images_by_answer(selected_images: List[Dict], answer: str) -> List[Dict]:
|
||||
"""
|
||||
后置图片过滤:根据 LLM 生成的回答内容反向筛选图片。
|
||||
# 图片意图关键词:用于判断查询/回答是否需要图片
|
||||
# 集中管理,便于后续新增文档类型时扩展
|
||||
|
||||
只有图片描述与回答内容有足够关键词重叠时才保留,
|
||||
确保展示的图片与回答内容一致,避免不相关图片干扰用户。
|
||||
# 查询侧关键词(宽松):用户查询中出现这些词表示想看图片
|
||||
_FIGURE_QUERY_KEYWORDS = frozenset([
|
||||
'图', '表', '照片', '图表', '如图', '见表', '示意图',
|
||||
'展示', '示意', '外观', '实物', '结构', '流程图', '架构图',
|
||||
])
|
||||
|
||||
# 回答侧关键词(严格):LLM 回答中出现这些词表示引用了图片
|
||||
# 不含单字"图""表"("力图""企图""表达"等领域词误触发),改用正则图号检测兜底
|
||||
# 不含"图片"(定义类查询的 LLM 回答也会泛化提到"图片",导致负面用例误放行)
|
||||
_FIGURE_ANSWER_KEYWORDS = frozenset([
|
||||
'照片', '图表', '如图', '见图', '见表', '示意图',
|
||||
'流程图', '架构图', '结构图', '实物图', '外观图',
|
||||
])
|
||||
|
||||
|
||||
def _filter_images_by_answer(selected_images: List[Dict], answer: str, query: str = "",
|
||||
primary_sections: List[str] = None) -> List[Dict]:
|
||||
"""
|
||||
后置图片过滤:查询主题一致性校验 + 无图意图早退 + 子章节级过滤。
|
||||
|
||||
确保展示的图片与查询/回答主题一致,过滤 section_cluster_boost 过度召回的
|
||||
不相关章节图片。
|
||||
|
||||
过滤规则:
|
||||
1. 从回答中提取 2 字及以上的中文关键词(jieba 分词)
|
||||
2. 对每张图片,检查其描述(full_description / description)中匹配了多少关键词
|
||||
3. 匹配数 >= 阈值则保留,否则丢弃
|
||||
4. 如果过滤后图片为 0,保留分数最高的 1 张(兜底)
|
||||
5. 用户明确指定图号(如图 2.1)时不过滤
|
||||
1. 候选 <= 1 张时直接返回(无需过滤)
|
||||
2. 无图意图早退:回答和查询都不含图片引用词 → 返回空
|
||||
3. 图号豁免:图片描述包含回答引用的具体图号/表号 → 无条件保留
|
||||
4. 主题一致性:合并查询+回答关键词,图片描述重叠 >= 阈值才保留
|
||||
- 子章节惩罚:图片子章节与主要检索章节不一致时,阈值 +2
|
||||
5. 兜底:过滤后为空且有图片意图 → 保留分数最高的 1 张
|
||||
|
||||
Args:
|
||||
selected_images: select_images 返回的候选图片列表
|
||||
answer: LLM 生成的回答文本
|
||||
query: 用户查询(拼接 retrieval_query + message,覆盖改写丢失的关键词)
|
||||
primary_sections: 主要检索结果的 section_path 列表(来自 top 文本切片),
|
||||
用于子章节级过滤。为空时跳过子章节惩罚,行为与原版一致。
|
||||
|
||||
Returns:
|
||||
过滤后的图片列表
|
||||
@@ -1250,28 +1273,80 @@ def _filter_images_by_answer(selected_images: List[Dict], answer: str) -> List[D
|
||||
if not selected_images or len(selected_images) <= 1:
|
||||
return selected_images
|
||||
|
||||
# 如果回答中提到了具体图号,说明 LLM 认为这些图是相关的,不过滤
|
||||
import re
|
||||
if re.search(r'图\s*\d+\.?\d*', answer):
|
||||
return selected_images
|
||||
|
||||
# 从回答中提取关键词
|
||||
# 提取回答中引用的图号/表号(用于精确豁免,不再一刀切绕过)
|
||||
_mentioned_refs = set()
|
||||
_mentioned_refs.update(re.findall(r'(?:[见如])?图\s*(\d+[\.\-]?\d*)', answer))
|
||||
_mentioned_refs.update(re.findall(r'(?:[见如])?表\s*(\d+[\.\-]?\d*)', answer))
|
||||
|
||||
# 前置无图意图判断:回答和查询都不需要图片时,直接返回空
|
||||
# 解决定义/原则类查询(case 18/19)领域关键词导致图片描述高重叠的问题
|
||||
_has_figure_in_answer = any(kw in answer for kw in _FIGURE_ANSWER_KEYWORDS)
|
||||
if not _has_figure_in_answer:
|
||||
_has_figure_in_answer = bool(re.search(r'[图表]\s*\d+', answer))
|
||||
_has_figure_in_query = any(kw in query for kw in _FIGURE_QUERY_KEYWORDS)
|
||||
if not _has_figure_in_query:
|
||||
_has_figure_in_query = bool(re.search(r'[图表]\s*\d+', query))
|
||||
|
||||
if not _has_figure_in_answer and not _has_figure_in_query:
|
||||
logger.info(f"[图片后置过滤] 无图意图前置检查:回答和查询均不含图片意图,返回空 "
|
||||
f"(候选 {len(selected_images)} 张)")
|
||||
return []
|
||||
|
||||
# 合并查询+回答关键词:查询提供主题焦点,回答提供细节补充
|
||||
# 解决纯回答关键词在长回答(500+字)场景下区分度不足的问题
|
||||
try:
|
||||
import jieba
|
||||
answer_keywords = set(
|
||||
w for w in jieba.lcut(answer)
|
||||
_combined_text = f"{query} {answer}" if query else answer
|
||||
_keywords = set(
|
||||
w for w in jieba.lcut(_combined_text)
|
||||
if len(w) >= 2 and re.search(r'[\u4e00-\u9fff]', w)
|
||||
)
|
||||
except ImportError:
|
||||
# jieba 不可用时回退到 bigram
|
||||
chars = re.findall(r'[\u4e00-\u9fff]', answer)
|
||||
answer_keywords = set(chars[i] + chars[i + 1] for i in range(len(chars) - 1))
|
||||
chars = re.findall(r'[\u4e00-\u9fff]', f"{query} {answer}" if query else answer)
|
||||
_keywords = set(chars[i] + chars[i + 1] for i in range(len(chars) - 1))
|
||||
|
||||
if not answer_keywords:
|
||||
if not _keywords:
|
||||
return selected_images
|
||||
|
||||
# 动态阈值:回答关键词越多,阈值越高(至少匹配 15% 或 2 个关键词,取较小值)
|
||||
threshold = max(2, min(3, round(len(answer_keywords) * 0.15)))
|
||||
# 动态阈值:关键词越多阈值越高,上限 4(比原版略宽松,因子章节惩罚承担更多过滤)
|
||||
threshold = max(2, min(4, round(len(_keywords) * 0.10)))
|
||||
|
||||
def _leaf_section_name(section_path: str) -> str:
|
||||
"""提取 section_path 的叶子节点名称(去编号前缀)"""
|
||||
if not section_path:
|
||||
return ''
|
||||
parts = [p.strip() for p in section_path.split('>') if p.strip()]
|
||||
leaf = parts[-1] if parts else ''
|
||||
# 去掉编号前缀:(2)、(一)、2.3、第1章、一、 等
|
||||
return re.sub(
|
||||
r'^(?:\(\d+\)\s*|([一二三四五六七八九十]+)\s*|\d+[\.\d]*\s*|'
|
||||
r'第\s*\d+\s*章\s*|[一二三四五六七八九十]+、\s*)', '', leaf
|
||||
).strip()
|
||||
|
||||
def _section_leaf_match(img_leaf: str, primary_name: str) -> bool:
|
||||
"""子章节名称匹配:长名称用子串包含,短名称(<3字)只做精确匹配"""
|
||||
if not img_leaf or not primary_name:
|
||||
return False
|
||||
if img_leaf == primary_name:
|
||||
return True
|
||||
if len(img_leaf) >= 3 and len(primary_name) >= 3:
|
||||
return primary_name in img_leaf or img_leaf in primary_name
|
||||
return False
|
||||
|
||||
# 构建主要子章节名称集合(从 top 文本切片的 section_path 提取)
|
||||
_primary_leaf_names = set()
|
||||
if primary_sections:
|
||||
for ps in primary_sections:
|
||||
name = _leaf_section_name(ps)
|
||||
if name and len(name) >= 2:
|
||||
_primary_leaf_names.add(name)
|
||||
|
||||
# 检索发散检测:主要子章节超过 3 个说明 section_cluster_boost 范围过宽
|
||||
# 子章节惩罚退化为无效(几乎所有图片都能匹配某个 primary section)
|
||||
# 此时对所有图片统一加严阈值,避免不相关图片全部通过
|
||||
_scattered_bonus = 1 if len(_primary_leaf_names) > 3 else 0
|
||||
|
||||
filtered = []
|
||||
for img in selected_images:
|
||||
@@ -1281,18 +1356,45 @@ def _filter_images_by_answer(selected_images: List[Dict], answer: str) -> List[D
|
||||
filtered.append(img)
|
||||
continue
|
||||
|
||||
# 计算描述与回答的关键词重叠数
|
||||
overlap = sum(1 for kw in answer_keywords if kw in desc)
|
||||
if overlap >= threshold:
|
||||
# 图号精确豁免:图片描述包含回答引用的具体图号/表号,无条件保留
|
||||
# 防御性逻辑:与 P0 答案对齐互补(P0 用 description 100字截断,此处用 full_description)
|
||||
if _mentioned_refs:
|
||||
_ref_matched = False
|
||||
for ref in _mentioned_refs:
|
||||
_ref_norm = ref.replace('-', '.')
|
||||
if (f"图{_ref_norm}" in desc or f"图 {_ref_norm}" in desc or
|
||||
f"表{_ref_norm}" in desc or f"表 {_ref_norm}" in desc):
|
||||
_ref_matched = True
|
||||
break
|
||||
if _ref_matched:
|
||||
filtered.append(img)
|
||||
continue
|
||||
|
||||
# 主题一致性过滤:图片描述与查询+回答关键词重叠 >= 阈值才保留
|
||||
overlap = sum(1 for kw in _keywords if kw in desc)
|
||||
|
||||
# 阈值计算:基础阈值 + 检索发散加严 + 子章节惩罚
|
||||
effective_threshold = threshold + _scattered_bonus
|
||||
if _primary_leaf_names:
|
||||
img_section_path = img.get('section_path', '')
|
||||
img_leaf = _leaf_section_name(img_section_path)
|
||||
if img_leaf and not any(_section_leaf_match(img_leaf, pn) for pn in _primary_leaf_names):
|
||||
effective_threshold += 2
|
||||
|
||||
if overlap >= effective_threshold:
|
||||
filtered.append(img)
|
||||
|
||||
# 兜底:如果过滤后为空,保留分数最高的 1 张
|
||||
# 兜底:过滤后为空且有图片意图时,保留分数最高的 1 张
|
||||
# 注意:无图意图场景已在上方早退(返回 []),此处兜底仅在有图意图时生效
|
||||
if not filtered and selected_images:
|
||||
filtered = [max(selected_images, key=lambda x: x.get('score', 0))]
|
||||
|
||||
if len(filtered) < len(selected_images):
|
||||
_sub_info = f", 子章节 {len(_primary_leaf_names)} 个" if _primary_leaf_names else ""
|
||||
_scattered_info = f", 发散加严+{_scattered_bonus}" if _scattered_bonus else ""
|
||||
logger.info(f"[图片后置过滤] {len(selected_images)} → {len(filtered)} 张 "
|
||||
f"(回答关键词 {len(answer_keywords)} 个, 阈值 {threshold})")
|
||||
f"(关键词 {len(_keywords)} 个, 阈值 {threshold}{_sub_info}{_scattered_info}, "
|
||||
f"图号豁免 {len(_mentioned_refs)} 个)")
|
||||
|
||||
return filtered
|
||||
|
||||
@@ -1521,10 +1623,12 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
scored_images.append({
|
||||
'score': s,
|
||||
'id': os.path.basename(meta['image_path']),
|
||||
'chunk_id': meta.get('chunk_id', ''),
|
||||
'url': f"/images/{os.path.basename(meta['image_path'])}",
|
||||
'type': meta['chunk_type'],
|
||||
'source': meta.get('source'),
|
||||
'page': meta.get('page'),
|
||||
'section_path': meta.get('section', '') or meta.get('section_path', ''),
|
||||
'description': doc[:100], # 短描述用于 UI 展示
|
||||
'full_description': doc # Bug 6b 修复:完整描述用于 LLM 上下文
|
||||
})
|
||||
@@ -1555,6 +1659,7 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
'type': 'table_image',
|
||||
'source': meta.get('source'),
|
||||
'page': img_page,
|
||||
'section_path': meta.get('section', '') or meta.get('section_path', ''),
|
||||
'description': doc[:100],
|
||||
'full_description': doc
|
||||
})
|
||||
@@ -1600,6 +1705,7 @@ def select_images(contexts: List[Dict], query: str) -> List[Dict]:
|
||||
'type': img_meta.get('chunk_type'),
|
||||
'source': img_meta.get('source'),
|
||||
'page': img_meta.get('page'),
|
||||
'section_path': img_meta.get('section', '') or img_meta.get('section_path', ''),
|
||||
'description': img_doc[:100], # 短描述用于 UI 展示
|
||||
'full_description': img_doc # Bug 6b 修复:完整描述用于 LLM 上下文
|
||||
})
|
||||
@@ -2489,7 +2595,18 @@ def rag():
|
||||
# else: LLM 没有提图号,保留原选择(不再截断到1张)
|
||||
|
||||
# 后置图片过滤:用回答内容反向筛选图片,确保图片与回答一致
|
||||
selected_images = _filter_images_by_answer(selected_images, full_answer_text)
|
||||
# query 拼接 retrieval_query + message,防止意图改写丢失图片意图关键词
|
||||
# 提取主要检索章节路径(top-10 文本切片),用于子章节级图片过滤
|
||||
_primary_sections = list(set(
|
||||
ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||
for ctx in text_contexts[:10]
|
||||
if ctx.get('meta', {}).get('section', '') or ctx.get('meta', {}).get('section_path', '')
|
||||
))
|
||||
selected_images = _filter_images_by_answer(
|
||||
selected_images, full_answer_text,
|
||||
query=f"{retrieval_query} {message}",
|
||||
primary_sections=_primary_sections
|
||||
)
|
||||
|
||||
rich_media = {'images': selected_images, 'tables': [], 'sections': []}
|
||||
|
||||
|
||||
@@ -83,9 +83,16 @@ class IntentAnalyzer:
|
||||
根据对话历史和当前用户消息,输出一个 JSON 对象,包含以下字段:
|
||||
|
||||
1. **rewritten_query**: 改写后的完整问题
|
||||
- 如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
||||
- 例如:"分析一下这两张图片" → "分析一下对话历史中提到的图片"
|
||||
- 如果问题本身已经完整,直接返回原文
|
||||
- **指代消解**:如果问题包含指代(如"这两张图片"、"继续说"),将其改写为完整、独立的问题
|
||||
- 例如:"分析一下这两张图片" → "分析一下对话历史中提到的图片"
|
||||
- **追问补全**:如果问题是省略式追问(省略了上一轮讨论的主题实体),必须补全为完整问题
|
||||
- 判断方法:当前问题缺少主语/宾语,且对话历史中可以推断出省略的实体
|
||||
- 补全方法:从上一轮用户问题中提取主题实体,与追问组合成完整问题
|
||||
- 例如:
|
||||
- 上一轮问"吸烟点C1类是什么区?",追问"有完整表格吗?" → "吸烟点C1类有完整表格吗?"
|
||||
- 上一轮问"三峡工程的投资情况",追问"建设地点在哪?" → "三峡工程的建设地点在哪?"
|
||||
- 上一轮问"货源投放有哪些原则?",追问"具体内容是什么?" → "货源投放原则的具体内容是什么?"
|
||||
- 如果问题本身已经完整且独立,直接返回原文
|
||||
|
||||
2. **use_context**: 布尔值
|
||||
- true: 问题依赖历史对话中的信息,答案已经在历史回答中
|
||||
@@ -105,7 +112,9 @@ class IntentAnalyzer:
|
||||
- 推理类(intent="reasoning"):生成最多2个子查询
|
||||
* 原问题的检索查询
|
||||
* 一个补充角度的检索查询(如原因、背景、影响等),帮助获取更全面的上下文
|
||||
- 其他类(factual/instruction/other):严格只生成1个子查询(原问题)
|
||||
- 其他类(factual/instruction/other):严格只生成1个子查询
|
||||
* 子查询应基于 rewritten_query(改写后的完整问题),而非用户原始输入
|
||||
* 例如:追问"有完整表格吗?"改写为"吸烟点C1类有完整表格吗?"后,子查询应为"吸烟点C1类的完整表格内容"
|
||||
- 不要为同一实体生成语义重叠的查询
|
||||
- 子查询应保持原问题的关键词,长度20-60字符为宜
|
||||
|
||||
|
||||
@@ -72,16 +72,35 @@ def call_llm(
|
||||
|
||||
content = response.choices[0].message.content
|
||||
|
||||
# 推理模型兼容:content 为空时尝试从 reasoning_content 提取
|
||||
# 推理模型兼容(mimo-v2.5 等):
|
||||
# 推理模型思考链消耗大量 token(~1000),max_tokens 不足时 content 为空,
|
||||
# 全部输出进入 reasoning_content。此处从思考链中提取有效内容。
|
||||
if not content or not content.strip():
|
||||
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||
if reasoning and reasoning.strip():
|
||||
# 从思维链中提取 JSON 块作为内容
|
||||
json_match = re.search(r'\{[\s\S]*\}', reasoning)
|
||||
if json_match:
|
||||
logger.info("LLM: content为空,从reasoning_content提取JSON")
|
||||
return json_match.group().strip()
|
||||
logger.warning("LLM 返回空 content(可能需要增大 max_tokens)")
|
||||
# 先去掉 <think>...</think> 标签
|
||||
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||
if cleaned:
|
||||
logger.info("LLM: content为空,从reasoning_content提取内容")
|
||||
# 尝试提取 JSON 对象(兼容结构化响应场景)
|
||||
json_match = re.search(r'\{[\s\S]*\}', cleaned)
|
||||
if json_match:
|
||||
try:
|
||||
json.loads(json_match.group())
|
||||
return json_match.group().strip()
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
# 尝试提取 JSON 数组
|
||||
bracket_match = re.search(r'\[[\s\S]*\]', cleaned)
|
||||
if bracket_match:
|
||||
try:
|
||||
json.loads(bracket_match.group())
|
||||
return bracket_match.group().strip()
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
# 纯文本响应:直接返回清理后的内容
|
||||
return cleaned
|
||||
logger.warning("LLM 返回空 content 且 reasoning_content 也无法提取(可能需要增大 max_tokens)")
|
||||
return None
|
||||
|
||||
return content.strip()
|
||||
@@ -95,7 +114,7 @@ def call_llm_stream(
|
||||
prompt: str,
|
||||
model: str,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 1000,
|
||||
max_tokens: int = 3000,
|
||||
messages: List[dict] = None,
|
||||
error_prefix: str = "[错误]",
|
||||
**kwargs
|
||||
@@ -104,13 +123,14 @@ def call_llm_stream(
|
||||
流式 LLM 调用(生成器封装)
|
||||
|
||||
自动处理流式响应,逐块 yield 文本内容。
|
||||
兼容推理模型(mimo-v2.5 等):当 content 为空时回退到 reasoning_content。
|
||||
|
||||
Args:
|
||||
client: OpenAI 客户端实例
|
||||
prompt: 用户提示
|
||||
model: 模型名称
|
||||
temperature: 温度参数
|
||||
max_tokens: 最大 token 数
|
||||
max_tokens: 最大 token 数(推理模型需留足思考链预算)
|
||||
messages: 完整消息列表
|
||||
error_prefix: 错误时的前缀
|
||||
**kwargs: 其他参数
|
||||
@@ -135,9 +155,33 @@ def call_llm_stream(
|
||||
**kwargs
|
||||
)
|
||||
|
||||
content_yielded = False
|
||||
reasoning_buffer = []
|
||||
|
||||
for chunk in stream:
|
||||
if chunk.choices and chunk.choices[0].delta.content:
|
||||
yield chunk.choices[0].delta.content
|
||||
if not chunk.choices:
|
||||
continue
|
||||
delta = chunk.choices[0].delta
|
||||
|
||||
# 正常 content 输出
|
||||
if hasattr(delta, 'content') and delta.content:
|
||||
content_yielded = True
|
||||
yield delta.content
|
||||
continue
|
||||
|
||||
# 推理模型:reasoning_content(思考链)
|
||||
rc = getattr(delta, 'reasoning_content', None)
|
||||
if rc:
|
||||
reasoning_buffer.append(rc)
|
||||
|
||||
# 回退:content 为空但 reasoning_content 有内容(推理模型 token 不足时)
|
||||
if not content_yielded and reasoning_buffer:
|
||||
reasoning_text = ''.join(reasoning_buffer)
|
||||
# 去掉 <think>...</think> 标签
|
||||
cleaned = re.sub(r'', '', reasoning_text, flags=re.DOTALL).strip()
|
||||
if cleaned:
|
||||
logger.info("流式 LLM: content为空,从reasoning_content提取内容")
|
||||
yield cleaned
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LLM 流式调用失败: {e}")
|
||||
@@ -352,7 +396,7 @@ def quick_yes_no(
|
||||
if keywords is None:
|
||||
keywords = ["是", "需要", "yes", "true"]
|
||||
|
||||
result = call_llm(client, prompt, model, temperature=0, max_tokens=10)
|
||||
result = call_llm(client, prompt, model, temperature=0, max_tokens=128)
|
||||
if result is None:
|
||||
return False
|
||||
|
||||
|
||||
@@ -25,7 +25,20 @@ def compute_file_hash(file_path: str) -> str:
|
||||
return hashlib.md5(file_path.encode()).hexdigest()
|
||||
|
||||
|
||||
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None) -> str:
|
||||
def _get_embedding_model():
|
||||
"""从 RAGEngine 获取 embedding 模型(KnowledgeBaseManager 上没有此属性)"""
|
||||
try:
|
||||
from core.engine import get_engine
|
||||
engine = get_engine()
|
||||
if not engine._initialized:
|
||||
engine.initialize()
|
||||
return engine.embedding_model
|
||||
except Exception as e:
|
||||
logger.warning(f"获取 embedding 模型失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, metadata: dict = None, defer_chromadb: bool = False) -> str:
|
||||
"""
|
||||
懒加载 VLM 描述
|
||||
|
||||
@@ -36,6 +49,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
||||
image_path: 图片路径(相对路径或绝对路径)
|
||||
kb_name: 知识库名称
|
||||
metadata: 图片元数据(包含 section、page、caption、上下文等)
|
||||
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
|
||||
|
||||
Returns:
|
||||
VLM 生成的图片描述
|
||||
@@ -49,23 +63,45 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
||||
else:
|
||||
full_image_path = image_path
|
||||
|
||||
# 1. 检查缓存
|
||||
# 1. 检查缓存(空缓存视为无效,需重新生成)
|
||||
img_hash = compute_file_hash(full_image_path)
|
||||
cache_file = VLM_CACHE_DIR / f"{img_hash}.txt"
|
||||
if cache_file.exists():
|
||||
logger.info(f"VLM 缓存命中: {image_path}")
|
||||
return cache_file.read_text(encoding='utf-8')
|
||||
cached = cache_file.read_text(encoding='utf-8')
|
||||
if len(cached.strip()) >= 5:
|
||||
logger.info(f"VLM 缓存命中: {image_path}")
|
||||
return cached
|
||||
else:
|
||||
logger.warning(f"VLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {image_path}")
|
||||
try:
|
||||
cache_file.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 2. 调用 VLM(传入元数据)
|
||||
logger.info(f"VLM 懒加载: {image_path}")
|
||||
kb_manager = get_kb_manager()
|
||||
description = kb_manager._generate_image_description(full_image_path, metadata=metadata)
|
||||
|
||||
# 3. 写入缓存
|
||||
# 3. 空描述保护:VLM 返回内容过短时不写入缓存和向量库
|
||||
if not description or len(description.strip()) < 5:
|
||||
logger.warning(f"VLM 返回描述过短({len(description.strip()) if description else 0}字符),跳过缓存和向量库更新: {image_path}")
|
||||
return description or ''
|
||||
|
||||
# 4. 写入缓存
|
||||
VLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_file.write_text(description, encoding='utf-8')
|
||||
|
||||
# 4. 更新向量库(metadata + embedding)
|
||||
# 5. 更新向量库(metadata + embedding),需校验 chunk_id 非空
|
||||
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
|
||||
if defer_chromadb:
|
||||
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
|
||||
return description
|
||||
|
||||
if not chunk_id:
|
||||
logger.warning("chunk_id 为空,跳过向量库更新")
|
||||
return description
|
||||
|
||||
try:
|
||||
collection = kb_manager.get_collection(kb_name)
|
||||
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
||||
@@ -79,7 +115,7 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
||||
|
||||
# 更新 embedding(使用 VLM 描述重新计算向量)
|
||||
# 这样 VLM 描述中的关键词(如"发电量")才能参与相似度检索
|
||||
embedding_model = kb_manager.embedding_model
|
||||
embedding_model = _get_embedding_model()
|
||||
if embedding_model:
|
||||
new_vector = embedding_model.encode(description).tolist()
|
||||
if isinstance(new_vector[0], list):
|
||||
@@ -91,20 +127,21 @@ async def lazy_vlm_description(chunk_id: str, image_path: str, kb_name: str, met
|
||||
embeddings=[new_vector],
|
||||
documents=[description] # 同时更新 document 字段
|
||||
)
|
||||
logger.info(f"已更新向量库 embedding: {chunk_id}")
|
||||
logger.info(f"已更新向量库(embedding+metadata): {chunk_id}")
|
||||
else:
|
||||
# 无 embedding 模型时只更新 metadata
|
||||
collection.update(
|
||||
ids=[chunk_id],
|
||||
metadatas=[new_metadata]
|
||||
)
|
||||
logger.info(f"已更新向量库(仅metadata,无embedding模型): {chunk_id}")
|
||||
except Exception as e:
|
||||
logger.warning(f"更新向量库失败: {e}")
|
||||
|
||||
return description
|
||||
|
||||
|
||||
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
||||
async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str, defer_chromadb: bool = False) -> str:
|
||||
"""
|
||||
懒加载表格摘要
|
||||
|
||||
@@ -114,50 +151,76 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
||||
chunk_id: 切片 ID
|
||||
table_md: 表格 Markdown 内容
|
||||
kb_name: 知识库名称
|
||||
defer_chromadb: 为 True 时跳过 ChromaDB 更新(仅写文件缓存),避免后台线程写锁竞争
|
||||
|
||||
Returns:
|
||||
LLM 生成的表格摘要
|
||||
"""
|
||||
from knowledge.manager import get_kb_manager
|
||||
|
||||
# 1. 检查缓存
|
||||
# 1. 检查缓存(空缓存视为无效)
|
||||
table_hash = hashlib.md5(table_md.encode()).hexdigest()
|
||||
cache_file = LLM_CACHE_DIR / f"{table_hash}.txt"
|
||||
if cache_file.exists():
|
||||
logger.info(f"LLM 缓存命中: {chunk_id}")
|
||||
return cache_file.read_text(encoding='utf-8')
|
||||
cached = cache_file.read_text(encoding='utf-8')
|
||||
if len(cached.strip()) >= 5:
|
||||
logger.info(f"LLM 缓存命中: {chunk_id}")
|
||||
return cached
|
||||
else:
|
||||
logger.warning(f"LLM 缓存内容过短({len(cached.strip())}字符),删除并重新生成: {chunk_id}")
|
||||
try:
|
||||
cache_file.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 2. 调用 LLM
|
||||
logger.info(f"LLM 懒加载: {chunk_id}")
|
||||
kb_manager = get_kb_manager()
|
||||
summary = kb_manager._generate_table_summary(table_md, None)
|
||||
|
||||
# 空摘要保护
|
||||
if not summary or len(summary.strip()) < 5:
|
||||
logger.warning(f"LLM 返回摘要过短,跳过缓存和向量库更新: {chunk_id}")
|
||||
return summary or ''
|
||||
|
||||
# 3. 写入缓存
|
||||
LLM_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_file.write_text(summary, encoding='utf-8')
|
||||
|
||||
# 4. 更新向量库(可选)
|
||||
# 4. 更新向量库,需校验 chunk_id 非空
|
||||
# defer_chromadb=True 时跳过(后台线程只写缓存,避免 SQLite 写锁竞争)
|
||||
if defer_chromadb:
|
||||
logger.info(f"延迟 ChromaDB 更新(仅写缓存): {chunk_id}")
|
||||
return summary
|
||||
|
||||
if not chunk_id:
|
||||
logger.warning("chunk_id 为空,跳过表格向量库更新")
|
||||
return summary
|
||||
try:
|
||||
collection = kb_manager.get_collection(kb_name)
|
||||
result = collection.get(ids=[chunk_id], include=['metadatas'])
|
||||
if result['metadatas']:
|
||||
# 新增摘要切片
|
||||
embedding_model = kb_manager.embedding_model
|
||||
vector = embedding_model.encode(summary).tolist()
|
||||
if isinstance(vector[0], list):
|
||||
vector = vector[0]
|
||||
# 新增摘要切片(需要 embedding 模型)
|
||||
embedding_model = _get_embedding_model()
|
||||
if embedding_model:
|
||||
vector = embedding_model.encode(summary).tolist()
|
||||
if isinstance(vector[0], list):
|
||||
vector = vector[0]
|
||||
|
||||
collection.add(
|
||||
ids=[f"{chunk_id}_summary"],
|
||||
embeddings=[vector],
|
||||
documents=[summary],
|
||||
metadatas=[{
|
||||
**result['metadatas'][0],
|
||||
'is_summary': True,
|
||||
'original_doc_id': chunk_id
|
||||
}]
|
||||
)
|
||||
# 更新原切片标记
|
||||
collection.add(
|
||||
ids=[f"{chunk_id}_summary"],
|
||||
embeddings=[vector],
|
||||
documents=[summary],
|
||||
metadatas=[{
|
||||
**result['metadatas'][0],
|
||||
'is_summary': True,
|
||||
'original_doc_id': chunk_id
|
||||
}]
|
||||
)
|
||||
logger.info(f"已新增摘要切片(embedding): {chunk_id}_summary")
|
||||
else:
|
||||
logger.info(f"跳过摘要切片(无embedding模型): {chunk_id}")
|
||||
# 更新原切片标记(不依赖 embedding 模型)
|
||||
collection.update(
|
||||
ids=[chunk_id],
|
||||
metadatas=[{**result['metadatas'][0], 'has_summary': True}]
|
||||
@@ -168,7 +231,7 @@ async def lazy_table_summary(chunk_id: str, table_md: str, kb_name: str) -> str:
|
||||
return summary
|
||||
|
||||
|
||||
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
||||
async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str, defer_chromadb: bool = False):
|
||||
"""
|
||||
检索后增强:按需调用 LLM/VLM
|
||||
|
||||
@@ -176,23 +239,24 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
||||
contexts: 检索上下文列表
|
||||
query: 用户查询
|
||||
kb_name: 知识库名称
|
||||
defer_chromadb: 为 True 时后台线程只写文件缓存,不更新 ChromaDB(避免写锁竞争)
|
||||
"""
|
||||
for ctx in contexts:
|
||||
meta = ctx.get('meta', {})
|
||||
chunk_type = meta.get('chunk_type', 'text')
|
||||
image_path = meta.get('image_path', '')
|
||||
import re
|
||||
|
||||
# 图片切片:懒加载 VLM 描述
|
||||
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
|
||||
if image_path:
|
||||
try:
|
||||
for ctx in contexts:
|
||||
try:
|
||||
meta = ctx.get('meta', {})
|
||||
chunk_type = meta.get('chunk_type', 'text')
|
||||
image_path = meta.get('image_path', '')
|
||||
|
||||
# 图片切片:懒加载 VLM 描述
|
||||
if chunk_type in ('image', 'chart') and not meta.get('has_vlm_desc'):
|
||||
if image_path:
|
||||
# 从 doc 字段中提取图号(上下文可能包含"见图2.5"等)
|
||||
doc_text = ctx.get('doc', '')
|
||||
import re
|
||||
|
||||
# 提取图号(从前文/后文中)
|
||||
figure_number = ""
|
||||
# 匹配 "见图2.5"、"图2.5"、"见图 2.5" 等
|
||||
fig_match = re.search(r'[见如]?图\s*(\d+\.?\d*)', doc_text)
|
||||
if fig_match:
|
||||
figure_number = fig_match.group(1)
|
||||
@@ -210,78 +274,73 @@ async def enhance_retrieved_chunks(contexts: list, query: str, kb_name: str):
|
||||
'page': meta.get('page'),
|
||||
'caption': meta.get('caption', ''),
|
||||
'source': meta.get('source', ''),
|
||||
'figure_number': figure_number, # 添加提取的图号
|
||||
'doc_text': doc_text # 添加完整文档文本
|
||||
'figure_number': figure_number,
|
||||
'doc_text': doc_text
|
||||
}
|
||||
vlm_desc = await lazy_vlm_description(
|
||||
meta.get('id', ''),
|
||||
meta.get('chunk_id', ''),
|
||||
image_path,
|
||||
kb_name,
|
||||
metadata=image_metadata
|
||||
metadata=image_metadata,
|
||||
defer_chromadb=defer_chromadb
|
||||
)
|
||||
ctx['doc'] = vlm_desc
|
||||
ctx['vlm_enhanced'] = True
|
||||
except Exception as e:
|
||||
logger.warning(f"VLM 懒加载失败: {e}")
|
||||
if vlm_desc:
|
||||
ctx['doc'] = vlm_desc
|
||||
ctx['vlm_enhanced'] = True
|
||||
|
||||
# 表格切片:同时处理摘要和关联图片的 VLM 描述
|
||||
elif chunk_type == 'table':
|
||||
doc_text = ctx.get('doc', '')
|
||||
# 表格切片:同时处理摘要和关联图片的 VLM 描述
|
||||
elif chunk_type == 'table':
|
||||
doc_text = ctx.get('doc', '')
|
||||
|
||||
# 1. 懒加载表格摘要(高分切片)
|
||||
if not meta.get('has_summary'):
|
||||
score = meta.get('score', 0)
|
||||
if score > 0.7: # 只对高相关表格生成摘要
|
||||
try:
|
||||
# 1. 懒加载表格摘要(高分切片)
|
||||
if not meta.get('has_summary'):
|
||||
score = ctx.get('score', 0)
|
||||
if score > 0.7:
|
||||
summary = await lazy_table_summary(
|
||||
meta.get('id', ''),
|
||||
meta.get('chunk_id', ''),
|
||||
doc_text,
|
||||
kb_name
|
||||
kb_name,
|
||||
defer_chromadb=defer_chromadb
|
||||
)
|
||||
# 摘要作为补充信息
|
||||
ctx['summary'] = summary
|
||||
ctx['llm_enhanced'] = True
|
||||
except Exception as e:
|
||||
logger.warning(f"表格摘要懒加载失败: {e}")
|
||||
|
||||
# 2. 表格有关联图片时,懒加载 VLM 描述
|
||||
if image_path and not meta.get('has_vlm_desc'):
|
||||
try:
|
||||
import re
|
||||
if summary:
|
||||
ctx['summary'] = summary
|
||||
ctx['llm_enhanced'] = True
|
||||
|
||||
# 2. 表格有关联图片时,懒加载 VLM 描述
|
||||
if image_path and not meta.get('has_vlm_desc'):
|
||||
# 提取表号(如 "表2.2"、"见表2.1")
|
||||
table_number = ""
|
||||
# 匹配 "表2.2"、"见表2.2"、"见表 2.2" 等
|
||||
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', doc_text)
|
||||
if table_match:
|
||||
table_number = table_match.group(1)
|
||||
|
||||
# 如果 doc 中没有,尝试从 section 中提取
|
||||
section = meta.get('section') or meta.get('section_path', '')
|
||||
if not table_number and section:
|
||||
table_match = re.search(r'[见如]?表\s*(\d+\.?\d*)', section)
|
||||
if table_match:
|
||||
table_number = table_match.group(1)
|
||||
|
||||
# 构建表格图片元数据
|
||||
table_image_metadata = {
|
||||
'section': section,
|
||||
'page': meta.get('page'),
|
||||
'caption': meta.get('caption', ''),
|
||||
'source': meta.get('source', ''),
|
||||
'table_number': table_number, # 表号
|
||||
'figure_number': table_number, # 兼容字段
|
||||
'table_number': table_number,
|
||||
'figure_number': table_number,
|
||||
'doc_text': doc_text,
|
||||
'is_table': True # 标记为表格图片
|
||||
'is_table': True
|
||||
}
|
||||
vlm_desc = await lazy_vlm_description(
|
||||
meta.get('id', ''),
|
||||
meta.get('chunk_id', ''),
|
||||
image_path,
|
||||
kb_name,
|
||||
metadata=table_image_metadata
|
||||
metadata=table_image_metadata,
|
||||
defer_chromadb=defer_chromadb
|
||||
)
|
||||
# 表格图片描述作为补充信息
|
||||
ctx['image_description'] = vlm_desc
|
||||
ctx['vlm_enhanced'] = True
|
||||
except Exception as e:
|
||||
logger.warning(f"表格图片 VLM 懒加载失败: {e}")
|
||||
if vlm_desc:
|
||||
ctx['image_description'] = vlm_desc
|
||||
ctx['vlm_enhanced'] = True
|
||||
|
||||
except Exception as e:
|
||||
chunk_id = ctx.get('meta', {}).get('chunk_id', '?')
|
||||
logger.warning(f"增强切片失败(chunk_id={chunk_id}): {e}")
|
||||
|
||||
@@ -33,7 +33,7 @@ try:
|
||||
import fcntl
|
||||
_HAS_FCNTL = True
|
||||
except ImportError:
|
||||
_HAS_FCNTL = False
|
||||
_HAS_FCNTL = False # Windows 环境无 fcntl
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from pathlib import Path
|
||||
import logging
|
||||
@@ -602,7 +602,7 @@ class KnowledgeBaseManager(
|
||||
try:
|
||||
from config import get_llm_client, DASHSCOPE_MODEL
|
||||
client = get_llm_client()
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=2048)
|
||||
return summary.strip() if summary else ""
|
||||
except Exception as e:
|
||||
logger.warning(f"生成表格摘要失败: {e}")
|
||||
@@ -661,13 +661,31 @@ class KnowledgeBaseManager(
|
||||
]
|
||||
}
|
||||
],
|
||||
max_tokens=512
|
||||
max_tokens=2048 # mimo-v2.5 推理模型思考链消耗 ~1000 token,需留足输出空间
|
||||
)
|
||||
|
||||
description = response.choices[0].message.content
|
||||
|
||||
# 推理模型兼容:content 为空时从 reasoning_content 提取
|
||||
if not description or not description.strip():
|
||||
reasoning = getattr(response.choices[0].message, 'reasoning_content', None)
|
||||
if reasoning and reasoning.strip():
|
||||
import re
|
||||
# 尝试从思考链中提取有用文本(去掉 <think> 标签后的内容)
|
||||
cleaned = re.sub(r'', '', reasoning, flags=re.DOTALL).strip()
|
||||
if cleaned:
|
||||
logger.info(f"VLM content为空,从reasoning_content提取描述: {image_path}")
|
||||
description = cleaned
|
||||
else:
|
||||
description = reasoning.strip()
|
||||
|
||||
if not description:
|
||||
logger.warning(f"VLM 返回空描述: {image_path}")
|
||||
return ""
|
||||
|
||||
# 缓存结果
|
||||
import hashlib
|
||||
import re as _re
|
||||
img_hash = hashlib.md5(img_path.read_bytes()).hexdigest()
|
||||
cache_dir = Path('.data/cache/vlm')
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
Reference in New Issue
Block a user