fix(bm25): 修复 BM25 索引覆盖 bug + 综合评测集 v2 + 检索增强
核心修复: - knowledge/base.py: BM25Index.add_documents 从覆盖改为追加+去重， 修复只有最后上传文件的 chunks 保留在 BM25 中的严重 bug (影响: 2.docx/3.docx/PDF 的 755 个 chunk 在 BM25 中完全缺失) 检索增强 (延续上次会话): - core/engine.py: section cluster boost + lexical match exemption - api/chat_routes.py: lexical/cluster rescue 层 + SSE 事件 - core/mmr.py: MMR 去重改进 评测体系: - tests/eval_dataset_v2.json: 62 题综合评测集 (9 种题型×4 文档) - scripts/eval_e2e.py: 推理模型 LLM 评分兼容 + 新数据集格式支持 - scripts/validate_eval_dataset.py: 数据集验证工具 其他: - parsers/mineru_parser.py: 解析器改进
This commit is contained in:
lacerate551
171
core/engine.py
171
core/engine.py
@@ -77,6 +77,10 @@ try:
|
||||
CONTEXT_EXPANSION_MAX_CHUNKS, ENUM_QUERY_DISABLE_TOPK_SHRINK, ENUM_QUERY_MMR_LAMBDA,
|
||||
# Phase 3 扩展精细化
|
||||
EXPANSION_SCORE_THRESHOLD, MAX_EXPANDED_NEIGHBORS,
|
||||
# 章节聚类救援
|
||||
SECTION_CLUSTER_BOOST_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
|
||||
CLUSTER_SEED_FLOOR, CLUSTER_MAX_BOOST_PER_SECTION, CLUSTER_MAX_SECTIONS,
|
||||
CLUSTER_SECTION_PREFIX_LEVELS,
|
||||
# 上下文与生成
|
||||
LLM_TEMPERATURE, LLM_MAX_TOKENS, RECALL_MULTIPLIER,
|
||||
# FAQ 与黑名单
|
||||
@@ -102,10 +106,18 @@ except ImportError:
|
||||
MMR_TOP_K = 30
|
||||
CONTEXT_EXPANSION_ENABLED = True
|
||||
CONTEXT_EXPANSION_BEFORE = 1
|
||||
CONTEXT_EXPANSION_AFTER = 5
|
||||
CONTEXT_EXPANSION_AFTER = 8
|
||||
CONTEXT_EXPANSION_MAX_CHUNKS = 24
|
||||
EXPANSION_SCORE_THRESHOLD = 0.3
|
||||
MAX_EXPANDED_NEIGHBORS = 4
|
||||
MAX_EXPANDED_NEIGHBORS = 8
|
||||
# 章节聚类救援默认值
|
||||
SECTION_CLUSTER_BOOST_ENABLED = True
|
||||
CLUSTER_MIN_MEMBERS = 3
|
||||
CLUSTER_MIN_TYPES = 2
|
||||
CLUSTER_SEED_FLOOR = 0.35
|
||||
CLUSTER_MAX_BOOST_PER_SECTION = 8
|
||||
CLUSTER_MAX_SECTIONS = 3
|
||||
CLUSTER_SECTION_PREFIX_LEVELS = 1
|
||||
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
|
||||
ENUM_QUERY_MMR_LAMBDA = 0.85
|
||||
DYNAMIC_RRF_ENABLED = True
|
||||
@@ -732,11 +744,19 @@ class RAGEngine:
|
||||
# 时间衰减（Time Decay）
|
||||
fused_results = self._apply_time_decay(fused_results)
|
||||
|
||||
# 提前附加 _debug，使聚类提升能写入调试步骤
|
||||
fused_results['_debug'] = _debug
|
||||
|
||||
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
|
||||
if SECTION_CLUSTER_BOOST_ENABLED:
|
||||
fused_results = self._section_cluster_boost(fused_results, query)
|
||||
|
||||
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
|
||||
# Phase 3：仅对高分种子扩展邻居
|
||||
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
|
||||
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
|
||||
min_score=EXPANSION_SCORE_THRESHOLD)
|
||||
min_score=EXPANSION_SCORE_THRESHOLD,
|
||||
query=query)
|
||||
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
|
||||
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})
|
||||
|
||||
@@ -1160,14 +1180,139 @@ class RAGEngine:
|
||||
return None
|
||||
return self.collection
|
||||
|
||||
@staticmethod
|
||||
def _normalize_section_path(section_path: str, levels: int = None) -> str:
|
||||
"""归一化 section_path：取前 N 级路径，容忍 MinerU 标题检测误差。
|
||||
|
||||
例如: "第三章 吸烟场所的功能设置 > 第三条 文明吸烟..." → "第三章 吸烟场所的功能设置"
|
||||
"""
|
||||
if not section_path:
|
||||
return ''
|
||||
if levels is None:
|
||||
levels = CLUSTER_SECTION_PREFIX_LEVELS
|
||||
parts = [p.strip() for p in section_path.split('>')]
|
||||
return ' > '.join(parts[:levels])
|
||||
|
||||
def _section_cluster_boost(self, results: dict, query: str = '') -> dict:
|
||||
"""章节聚类提升：当同一 section 下多个切片（text+table）同时出现在候选集中，
|
||||
即使单个切片 CrossEncoder 分数很低，也将整组提升到种子阈值。
|
||||
|
||||
核心洞察：单个低分切片不可信，但同一 section 多个切片同时出现是强信号。
|
||||
提升后的切片可以作为 _expand_contiguous_chunks 的种子，触发邻居扩展。
|
||||
|
||||
Args:
|
||||
results: rerank 后的检索结果
|
||||
query: 用户查询（用于后续扩展）
|
||||
|
||||
Returns:
|
||||
修改后的 results（distances 被调整，meta 中标记 _cluster_boosted）
|
||||
"""
|
||||
if not results.get('ids') or not results['ids'][0]:
|
||||
return results
|
||||
|
||||
ids = results['ids'][0]
|
||||
metas = results.get('metadatas', [[]])[0]
|
||||
distances = results.get('distances', [[]])[0] if results.get('distances') else None
|
||||
|
||||
if not distances:
|
||||
return results
|
||||
|
||||
# 1. 按 (source, normalized_section) 分组
|
||||
from collections import defaultdict
|
||||
section_groups = defaultdict(list) # key → [(index, meta, dist)]
|
||||
|
||||
for i, (meta, dist) in enumerate(zip(metas, distances)):
|
||||
source = meta.get('source', '')
|
||||
section_path = meta.get('section', '') or meta.get('section_path', '')
|
||||
norm_section = self._normalize_section_path(section_path)
|
||||
if not source or not norm_section:
|
||||
continue
|
||||
key = (source, norm_section)
|
||||
section_groups[key].append((i, meta, dist))
|
||||
|
||||
# 2. 检测聚类信号并提升
|
||||
boost_target_dist = 1.0 - CLUSTER_SEED_FLOOR # score=0.35 → dist=0.65
|
||||
boosted_sections = []
|
||||
total_boosted = 0
|
||||
|
||||
# 按组成员数降序排列，优先处理最大聚类
|
||||
sorted_groups = sorted(section_groups.items(), key=lambda x: len(x[1]), reverse=True)
|
||||
|
||||
for (source, norm_section), members in sorted_groups:
|
||||
if len(boosted_sections) >= CLUSTER_MAX_SECTIONS:
|
||||
break
|
||||
|
||||
# 聚类信号检测：成员数 >= 阈值 且 类型多样性 >= 阈值
|
||||
chunk_types = set(m[1].get('chunk_type', 'text') for m in members)
|
||||
if len(members) < CLUSTER_MIN_MEMBERS or len(chunk_types) < CLUSTER_MIN_TYPES:
|
||||
continue
|
||||
|
||||
# 提升组内切片分数（仅提升低于阈值的）
|
||||
boost_count = 0
|
||||
for idx, meta, dist in members:
|
||||
if boost_count >= CLUSTER_MAX_BOOST_PER_SECTION:
|
||||
break
|
||||
# 只提升分数低于种子阈值的切片（高分切片不需要）
|
||||
if dist > boost_target_dist:
|
||||
distances[idx] = boost_target_dist
|
||||
meta['_cluster_boosted'] = True
|
||||
boost_count += 1
|
||||
total_boosted += 1
|
||||
|
||||
if boost_count > 0:
|
||||
boosted_sections.append({
|
||||
'source': source,
|
||||
'section': norm_section,
|
||||
'members': len(members),
|
||||
'types': list(chunk_types),
|
||||
'boosted': boost_count
|
||||
})
|
||||
|
||||
# 3. 写 debug 信息
|
||||
if boosted_sections:
|
||||
debug_info = results.get('_debug', {})
|
||||
if 'steps' not in debug_info:
|
||||
debug_info['steps'] = []
|
||||
debug_info['steps'].append({
|
||||
'name': 'section_cluster_boost',
|
||||
'sections': boosted_sections,
|
||||
'total_boosted': total_boosted
|
||||
})
|
||||
results['_debug'] = debug_info
|
||||
logger.info(f"[章节聚类提升] 提升 {total_boosted} 个切片，"
|
||||
f"涉及 {len(boosted_sections)} 个 section: "
|
||||
f"{[s['section'][:30] for s in boosted_sections]}")
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _chunk_lexical_score(chunk_text: str, query: str) -> float:
|
||||
"""计算切片文本与查询的词法重叠度（bigram 命中率），用于辅助种子资格判定。"""
|
||||
if not chunk_text or not query:
|
||||
return 0.0
|
||||
import re
|
||||
clean_q = re.sub(r'[？?！!。，,、；;：:"""\'\s*#`]+', ' ', query).strip()
|
||||
if len(clean_q) < 2:
|
||||
return 0.0
|
||||
bigrams = set()
|
||||
for i in range(len(clean_q) - 1):
|
||||
w = clean_q[i:i+2].strip()
|
||||
if len(w) == 2:
|
||||
bigrams.add(w)
|
||||
if not bigrams:
|
||||
return 0.0
|
||||
matched = sum(1 for w in bigrams if w in chunk_text)
|
||||
return matched / len(bigrams)
|
||||
|
||||
def _expand_contiguous_chunks(self, results: dict, top_k: int = None,
|
||||
min_score: float = 0.0) -> dict:
|
||||
min_score: float = 0.0, query: str = '') -> dict:
|
||||
"""Add same-source same-section neighbor text chunks around strong hits.
|
||||
|
||||
Args:
|
||||
results: 检索结果
|
||||
top_k: 最大切片数
|
||||
min_score: Phase 3 最低分数阈值，仅对 Rerank 分数高于此值的种子扩展
|
||||
query: 查询文本，用于词法匹配辅助种子资格判定
|
||||
"""
|
||||
if not CONTEXT_EXPANSION_ENABLED:
|
||||
return results
|
||||
@@ -1197,7 +1342,7 @@ class RAGEngine:
|
||||
seeds = [
|
||||
(doc_id, doc, meta, dist)
|
||||
for doc_id, doc, meta, dist in items[:base_limit]
|
||||
if meta.get('chunk_type', 'text') == 'text'
|
||||
if (meta.get('chunk_type', 'text') == 'text' or meta.get('_cluster_boosted'))
|
||||
and meta.get('source')
|
||||
and self._to_int(meta.get('chunk_index')) is not None
|
||||
]
|
||||
@@ -1208,8 +1353,12 @@ class RAGEngine:
|
||||
break
|
||||
|
||||
# Phase 3：跳过分数低于阈值的种子（仅当 min_score > 0 时生效）
|
||||
# 词法匹配豁免：CrossEncoder 低分但关键词重叠度高时仍允许作为种子
|
||||
if min_score > 0 and seed_dist < min_score:
|
||||
continue
|
||||
if query and self._chunk_lexical_score(_seed_doc, query) > 0.3:
|
||||
pass # 词法匹配度高，允许作为种子
|
||||
else:
|
||||
continue
|
||||
|
||||
source = seed_meta.get('source')
|
||||
section = seed_meta.get('section', '') or seed_meta.get('section_path', '')
|
||||
@@ -1627,11 +1776,19 @@ class RAGEngine:
|
||||
# 时间衰减
|
||||
fused_results = self._apply_time_decay(fused_results)
|
||||
|
||||
# 提前附加 _debug，使聚类提升能写入调试步骤
|
||||
fused_results['_debug'] = _debug
|
||||
|
||||
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
|
||||
if SECTION_CLUSTER_BOOST_ENABLED:
|
||||
fused_results = self._section_cluster_boost(fused_results, query)
|
||||
|
||||
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
|
||||
# Phase 3：仅对高分种子扩展邻居
|
||||
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
|
||||
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
|
||||
min_score=EXPANSION_SCORE_THRESHOLD)
|
||||
min_score=EXPANSION_SCORE_THRESHOLD,
|
||||
query=query)
|
||||
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
|
||||
if _debug is not None:
|
||||
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})
|
||||
|
||||
Reference in New Issue
Block a user