feat(rag): 数据驱动的检索优化与表格/图片处理增强
- 移除硬编码关键词列表，改为数据驱动的图片意图检测 - 新增 _section_similarity() 层级章节相似度函数，替代正则匹配 - 新增 _rescue_table_chunks() 表格救援机制，基于 _in_budget_context 标记 - 修复 _build_context_with_budget 表格组超预算不 break 的问题 - 新增 _filter_images_by_answer() 后置图片过滤 - 表格切片 rerank 分数保护策略（同 section 表格不被误删） - 跨页表格合并逻辑改进（通用标题检测 + 页码不可用降级策略） - 意图分析增强追问补全 + 缓存类型隔离 - 语义缓存 key 加入 collections 防止跨上下文误命中 - 使用 retrieval_query 替代原始 message 进行检索 - engine.py: CloudReranker 模型更新 + 移除 top_n + table 邻居扩展 - LLM prompt 增加表格处理规则和章节路径提示 🤖 Generated with [Qoder][https://qoder.com]
This commit is contained in:
lacerate551
@@ -393,6 +393,11 @@ class KnowledgeBaseManager(
|
||||
if len(chunks) < 2:
|
||||
return chunks
|
||||
|
||||
# 检测页码是否可靠：若所有 chunk 的 page_start 相同（如 Word 文档 page_idx 全为 0），
|
||||
# 则页码信息不可用，需要启用降级合并规则
|
||||
page_values = set(getattr(c, 'page_start', 0) for c in chunks)
|
||||
pages_unavailable = len(page_values) <= 1
|
||||
|
||||
merged_chunks = []
|
||||
i = 0
|
||||
merge_count = 0
|
||||
@@ -405,6 +410,7 @@ class KnowledgeBaseManager(
|
||||
# 查找下一个表格（跳过中间的"续表"文本）
|
||||
next_table_idx = None
|
||||
next_chunk = None
|
||||
intermediate_texts = [] # 收集中间文本用于降级判断
|
||||
|
||||
for j in range(i + 1, min(i + 4, len(chunks))): # 最多向前看3个切片
|
||||
candidate = chunks[j]
|
||||
@@ -419,7 +425,11 @@ class KnowledgeBaseManager(
|
||||
elif candidate_type == 'text' and ('续表' in candidate_title or '续表' in candidate_content):
|
||||
# 遇到"续表"文本，继续查找下一个表格
|
||||
continue
|
||||
elif candidate_type not in ('text',):
|
||||
elif candidate_type == 'text':
|
||||
# 非"续表"文本，收集后停止查找
|
||||
intermediate_texts.append(candidate)
|
||||
break
|
||||
else:
|
||||
# 遇到非文本类型，停止查找
|
||||
break
|
||||
|
||||
@@ -439,24 +449,59 @@ class KnowledgeBaseManager(
|
||||
# 获取内容（用于检测"续表"）
|
||||
next_content = getattr(next_chunk, 'content', '')
|
||||
|
||||
# 通用/无意义标题集合，这些标题不能用于"标题相似"判定
|
||||
_GENERIC_TITLES = {'表格', 'table', '表格', ''}
|
||||
|
||||
# 判断是否为跨页表格
|
||||
is_cross_page = False
|
||||
|
||||
# 规则1: 页码连续（如果页码有效）
|
||||
page_valid = curr_page_end > 0 and next_page_start > 0
|
||||
if page_valid and curr_page_end + 1 == next_page_start:
|
||||
is_cross_page = True
|
||||
# 页码连续时，还需标题匹配或为通用标题才合并
|
||||
# 避免把不同页面上不相关的表格错误合并
|
||||
if curr_title == next_title or curr_title in _GENERIC_TITLES and next_title in _GENERIC_TITLES:
|
||||
is_cross_page = True
|
||||
elif curr_title and next_title:
|
||||
clean_next_r1 = next_title.replace('续表', '').strip()
|
||||
if curr_title in clean_next_r1 or clean_next_r1 in curr_title:
|
||||
is_cross_page = True
|
||||
|
||||
# 规则2: 第二个表格标题或内容包含"续表"
|
||||
elif '续表' in next_title or '续表' in next_content:
|
||||
is_cross_page = True
|
||||
|
||||
# 规则3: 标题相似（去掉"续表"后比较）
|
||||
elif curr_title and next_title:
|
||||
# 排除通用标题（如"表格"），防止把所有标题为"表格"的相邻表格都误合并
|
||||
elif (curr_title and next_title
|
||||
and curr_title not in _GENERIC_TITLES
|
||||
and next_title not in _GENERIC_TITLES):
|
||||
clean_next = next_title.replace('续表', '').strip()
|
||||
if curr_title in clean_next or clean_next in curr_title:
|
||||
if clean_next and (curr_title in clean_next or clean_next in curr_title):
|
||||
is_cross_page = True
|
||||
|
||||
# 规则4（降级）: 页码不可用（如 Word 文档 page_idx 全为 0）
|
||||
# 仅当页码信息缺失时才启用此规则，避免 PDF 正常页码时被误合并
|
||||
if (not is_cross_page
|
||||
and pages_unavailable
|
||||
and curr_title in _GENERIC_TITLES
|
||||
and next_title in _GENERIC_TITLES):
|
||||
# 检查中间文本是否暗示跨页延续（空、短文本、续表标记等）
|
||||
has_separating_content = False
|
||||
for text_chunk in intermediate_texts:
|
||||
tc = (getattr(text_chunk, 'content', '') or '').strip()
|
||||
tt = (getattr(text_chunk, 'title', '') or '').strip()
|
||||
if not tc:
|
||||
continue # 空文本不算分隔
|
||||
if '续表' in tc or '续表' in tt:
|
||||
continue # 续表标记，说明是跨页
|
||||
# 有实质性中间内容（如分类标题"A3类：xxx"），不合并
|
||||
has_separating_content = True
|
||||
break
|
||||
if not has_separating_content:
|
||||
is_cross_page = True
|
||||
logger.debug(f"降级合并(页码不可用): '{curr_title}' + '{next_title}'")
|
||||
|
||||
if is_cross_page:
|
||||
# 执行合并
|
||||
merge_count += 1
|
||||
@@ -469,14 +514,27 @@ class KnowledgeBaseManager(
|
||||
# 合并两个表格的 HTML
|
||||
current.table_html = curr_html + '\n' + next_html
|
||||
|
||||
# 合并 image_path 到 images
|
||||
# 合并 image_path 和嵌入图片到 images
|
||||
curr_img = getattr(current, 'image_path', None)
|
||||
next_img = getattr(next_chunk, 'image_path', None)
|
||||
merged_images = []
|
||||
if curr_img:
|
||||
curr_images = getattr(current, 'images', None) or []
|
||||
next_images = getattr(next_chunk, 'images', None) or []
|
||||
|
||||
# 合并两个表格的所有图片（image_path + 嵌入图片）
|
||||
merged_images = list(curr_images) # 保留当前表格的嵌入图片
|
||||
# 添加 image_path 图片（如果不在列表中）
|
||||
existing_ids = {img.get('id', '') for img in merged_images if isinstance(img, dict)}
|
||||
if curr_img and curr_img not in existing_ids:
|
||||
merged_images.append({'id': curr_img, 'page': curr_page_end})
|
||||
if next_img:
|
||||
existing_ids.add(curr_img)
|
||||
for img in next_images: # 添加下一个表格的嵌入图片
|
||||
img_id = img.get('id', '') if isinstance(img, dict) else ''
|
||||
if img_id and img_id not in existing_ids:
|
||||
merged_images.append(img)
|
||||
existing_ids.add(img_id)
|
||||
if next_img and next_img not in existing_ids:
|
||||
merged_images.append({'id': next_img, 'page': next_page_start})
|
||||
|
||||
if merged_images:
|
||||
current.images = merged_images
|
||||
# 保留第一个图片作为主 image_path
|
||||
@@ -525,7 +583,7 @@ class KnowledgeBaseManager(
|
||||
try:
|
||||
from config import get_llm_client, DASHSCOPE_MODEL
|
||||
client = get_llm_client()
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=100)
|
||||
summary = call_llm(client, prompt, DASHSCOPE_MODEL, max_tokens=512)
|
||||
return summary.strip() if summary else ""
|
||||
except Exception as e:
|
||||
logger.warning(f"生成表格摘要失败: {e}")
|
||||
@@ -584,7 +642,7 @@ class KnowledgeBaseManager(
|
||||
]
|
||||
}
|
||||
],
|
||||
max_tokens=200
|
||||
max_tokens=512
|
||||
)
|
||||
|
||||
description = response.choices[0].message.content
|
||||
|
||||
Reference in New Issue
Block a user