jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bm25): 修复 BM25 索引覆盖 bug + 综合评测集 v2 + 检索增强

Browse Source 
核心修复:
- knowledge/base.py: BM25Index.add_documents 从覆盖改为追加+去重，
  修复只有最后上传文件的 chunks 保留在 BM25 中的严重 bug
  (影响: 2.docx/3.docx/PDF 的 755 个 chunk 在 BM25 中完全缺失)

检索增强 (延续上次会话):
- core/engine.py: section cluster boost + lexical match exemption
- api/chat_routes.py: lexical/cluster rescue 层 + SSE 事件
- core/mmr.py: MMR 去重改进

评测体系:
- tests/eval_dataset_v2.json: 62 题综合评测集 (9 种题型×4 文档)
- scripts/eval_e2e.py: 推理模型 LLM 评分兼容 + 新数据集格式支持
- scripts/validate_eval_dataset.py: 数据集验证工具

其他:
- parsers/mineru_parser.py: 解析器改进
This commit is contained in:
lacerate551
2026-06-17 17:31:43 +08:00
parent edaef7ad60
commit 8c92657490
8 changed files with 1322 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
core/engine.py
View File

@@ -77,6 +77,10 @@ try:
CONTEXT_EXPANSION_MAX_CHUNKS, ENUM_QUERY_DISABLE_TOPK_SHRINK, ENUM_QUERY_MMR_LAMBDA,
# Phase 3 扩展精细化
EXPANSION_SCORE_THRESHOLD, MAX_EXPANDED_NEIGHBORS,
# 章节聚类救援
SECTION_CLUSTER_BOOST_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
CLUSTER_SEED_FLOOR, CLUSTER_MAX_BOOST_PER_SECTION, CLUSTER_MAX_SECTIONS,
CLUSTER_SECTION_PREFIX_LEVELS,
# 上下文与生成
LLM_TEMPERATURE, LLM_MAX_TOKENS, RECALL_MULTIPLIER,
# FAQ 与黑名单
@@ -102,10 +106,18 @@ except ImportError:
MMR_TOP_K = 30
CONTEXT_EXPANSION_ENABLED = True
CONTEXT_EXPANSION_BEFORE = 1
CONTEXT_EXPANSION_AFTER = 5
CONTEXT_EXPANSION_AFTER = 8
CONTEXT_EXPANSION_MAX_CHUNKS = 24
EXPANSION_SCORE_THRESHOLD = 0.3
MAX_EXPANDED_NEIGHBORS = 4
MAX_EXPANDED_NEIGHBORS = 8
# 章节聚类救援默认值
SECTION_CLUSTER_BOOST_ENABLED = True
CLUSTER_MIN_MEMBERS = 3
CLUSTER_MIN_TYPES = 2
CLUSTER_SEED_FLOOR = 0.35
CLUSTER_MAX_BOOST_PER_SECTION = 8
CLUSTER_MAX_SECTIONS = 3
CLUSTER_SECTION_PREFIX_LEVELS = 1
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
ENUM_QUERY_MMR_LAMBDA = 0.85
DYNAMIC_RRF_ENABLED = True
@@ -732,11 +744,19 @@ class RAGEngine:
# 时间衰减（Time Decay）
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})
@@ -1160,14 +1180,139 @@ class RAGEngine:
return None
return self.collection
@staticmethod
def _normalize_section_path(section_path: str, levels: int = None) -> str:
"""归一化 section_path：取前 N 级路径，容忍 MinerU 标题检测误差。
例如: "第三章 吸烟场所的功能设置 > 第三条 文明吸烟..." → "第三章 吸烟场所的功能设置"
"""
if not section_path:
return ''
if levels is None:
levels = CLUSTER_SECTION_PREFIX_LEVELS
parts = [p.strip() for p in section_path.split('>')]
return ' > '.join(parts[:levels])
def _section_cluster_boost(self, results: dict, query: str = '') -> dict:
"""章节聚类提升：当同一 section 下多个切片（text+table）同时出现在候选集中，
即使单个切片 CrossEncoder 分数很低，也将整组提升到种子阈值。
核心洞察：单个低分切片不可信，但同一 section 多个切片同时出现是强信号。
提升后的切片可以作为 _expand_contiguous_chunks 的种子，触发邻居扩展。
Args:
results: rerank 后的检索结果
query: 用户查询（用于后续扩展）
Returns:
修改后的 results（distances 被调整，meta 中标记 _cluster_boosted）
"""
if not results.get('ids') or not results['ids'][0]:
return results
ids = results['ids'][0]
metas = results.get('metadatas', [[]])[0]
distances = results.get('distances', [[]])[0] if results.get('distances') else None
if not distances:
return results
# 1. 按 (source, normalized_section) 分组
from collections import defaultdict
section_groups = defaultdict(list) # key → [(index, meta, dist)]
for i, (meta, dist) in enumerate(zip(metas, distances)):
source = meta.get('source', '')
section_path = meta.get('section', '') or meta.get('section_path', '')
norm_section = self._normalize_section_path(section_path)
if not source or not norm_section:
continue
key = (source, norm_section)
section_groups[key].append((i, meta, dist))
# 2. 检测聚类信号并提升
boost_target_dist = 1.0 - CLUSTER_SEED_FLOOR # score=0.35 → dist=0.65
boosted_sections = []
total_boosted = 0
# 按组成员数降序排列，优先处理最大聚类
sorted_groups = sorted(section_groups.items(), key=lambda x: len(x[1]), reverse=True)
for (source, norm_section), members in sorted_groups:
if len(boosted_sections) >= CLUSTER_MAX_SECTIONS:
break
# 聚类信号检测：成员数 >= 阈值 且 类型多样性 >= 阈值
chunk_types = set(m[1].get('chunk_type', 'text') for m in members)
if len(members) < CLUSTER_MIN_MEMBERS or len(chunk_types) < CLUSTER_MIN_TYPES:
continue
# 提升组内切片分数（仅提升低于阈值的）
boost_count = 0
for idx, meta, dist in members:
if boost_count >= CLUSTER_MAX_BOOST_PER_SECTION:
break
# 只提升分数低于种子阈值的切片（高分切片不需要）
if dist > boost_target_dist:
distances[idx] = boost_target_dist
meta['_cluster_boosted'] = True
boost_count += 1
total_boosted += 1
if boost_count > 0:
boosted_sections.append({
'source': source,
'section': norm_section,
'members': len(members),
'types': list(chunk_types),
'boosted': boost_count
})
# 3. 写 debug 信息
if boosted_sections:
debug_info = results.get('_debug', {})
if 'steps' not in debug_info:
debug_info['steps'] = []
debug_info['steps'].append({
'name': 'section_cluster_boost',
'sections': boosted_sections,
'total_boosted': total_boosted
})
results['_debug'] = debug_info
logger.info(f"[章节聚类提升] 提升 {total_boosted} 个切片，"
f"涉及 {len(boosted_sections)} 个 section: "
f"{[s['section'][:30] for s in boosted_sections]}")
return results
@staticmethod
def _chunk_lexical_score(chunk_text: str, query: str) -> float:
"""计算切片文本与查询的词法重叠度（bigram 命中率），用于辅助种子资格判定。"""
if not chunk_text or not query:
return 0.0
import re
clean_q = re.sub(r'[？?！!。，,、；;：:"""\'\s*#`]+', ' ', query).strip()
if len(clean_q) < 2:
return 0.0
bigrams = set()
for i in range(len(clean_q) - 1):
w = clean_q[i:i+2].strip()
if len(w) == 2:
bigrams.add(w)
if not bigrams:
return 0.0
matched = sum(1 for w in bigrams if w in chunk_text)
return matched / len(bigrams)
def _expand_contiguous_chunks(self, results: dict, top_k: int = None,
min_score: float = 0.0) -> dict:
min_score: float = 0.0, query: str = '') -> dict:
"""Add same-source same-section neighbor text chunks around strong hits.
Args:
results: 检索结果
top_k: 最大切片数
min_score: Phase 3 最低分数阈值，仅对 Rerank 分数高于此值的种子扩展
query: 查询文本，用于词法匹配辅助种子资格判定
"""
if not CONTEXT_EXPANSION_ENABLED:
return results
@@ -1197,7 +1342,7 @@ class RAGEngine:
seeds = [
(doc_id, doc, meta, dist)
for doc_id, doc, meta, dist in items[:base_limit]
if meta.get('chunk_type', 'text') == 'text'
if (meta.get('chunk_type', 'text') == 'text' or meta.get('_cluster_boosted'))
and meta.get('source')
and self._to_int(meta.get('chunk_index')) is not None
]
@@ -1208,8 +1353,12 @@ class RAGEngine:
break
# Phase 3：跳过分数低于阈值的种子（仅当 min_score > 0 时生效）
# 词法匹配豁免：CrossEncoder 低分但关键词重叠度高时仍允许作为种子
if min_score > 0 and seed_dist < min_score:
continue
if query and self._chunk_lexical_score(_seed_doc, query) > 0.3:
pass # 词法匹配度高，允许作为种子
else:
continue
source = seed_meta.get('source')
section = seed_meta.get('section', '') or seed_meta.get('section_path', '')
@@ -1627,11 +1776,19 @@ class RAGEngine:
# 时间衰减
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
if _debug is not None:
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})

81
core/mmr.py
View File

@@ -108,22 +108,44 @@ def mmr_rerank(
return selected
def _tokenize_words(text: str) -> set:
"""
使用 jieba 分词并过滤噪声，返回有意义的词集合。
过滤规则：
- 去除单字符词（如 "的", "了", "在"）—— 这些是停用词，对区分文档无意义
- 去除纯数字 / 纯标点
- 保留 2 字及以上的实词
"""
import jieba
words = set()
for w in jieba.cut(text):
w = w.strip()
if len(w) >= 2 and not w.isdigit():
words.add(w)
return words
def mmr_filter_by_content(
candidates: List[Dict],
top_k: int = 30,
similarity_threshold: float = 0.9
similarity_threshold: float = 0.85
) -> List[Dict]:
"""
基于内容相似度的去重（简化版，不需要 embedding）
基于 jieba 词级 Jaccard 相似度的去重（不需要 embedding）
与旧版字符级 set(text) 的区别：
- 旧版：set("安全生产管理制度") → {'安','全','生','产',...}，中文文档间字符集合高度重叠
- 新版：jieba 分词 → {"安全生产", "管理制度", ...}，词级集合区分度高
适用于：
- 没有 embedding 的情况
- 快速去重场景
- MMR_USE_EMBEDDING=False 时的快速去重
- 避免 CPU 编码 100+ 文档的 50 秒开销
Args:
candidates: 候选文档列表
top_k: 返回数量
similarity_threshold: 相似度阈值，超过则视为重复
similarity_threshold: 相似度阈值，超过则视为重复（默认 0.85）
Returns:
去重后的候选文档列表
@@ -134,35 +156,42 @@ def mmr_filter_by_content(
if len(candidates) <= top_k:
return candidates
selected = []
remaining = candidates.copy()
# 预分词：对所有候选文档一次性分词，避免重复调用 jieba.cut
word_sets = []
for c in candidates:
content = c.get('content', c.get('document', ''))[:500]
word_sets.append(_tokenize_words(content))
while len(selected) < top_k and remaining:
current = remaining.pop(0)
selected_indices = []
for i in range(len(candidates)):
if len(selected_indices) >= top_k:
break
current_words = word_sets[i]
if not current_words:
# 空内容直接保留
selected_indices.append(i)
continue
# 检查是否与已选内容重复
is_duplicate = False
current_content = current.get('content', current.get('document', ''))[:200]
for j in selected_indices:
selected_words = word_sets[j]
if not selected_words:
continue
for s in selected:
s_content = s.get('content', s.get('document', ''))[:200]
intersection = len(current_words & selected_words)
union = len(current_words | selected_words)
similarity = intersection / union if union > 0 else 0
# 简单的 Jaccard 相似度
words1 = set(current_content)
words2 = set(s_content)
if words1 and words2:
intersection = len(words1 & words2)
union = len(words1 | words2)
similarity = intersection / union if union > 0 else 0
if similarity > similarity_threshold:
is_duplicate = True
break
if similarity > similarity_threshold:
is_duplicate = True
break
if not is_duplicate:
selected.append(current)
selected_indices.append(i)
return selected
return [candidates[i] for i in selected_indices]
# ==================== 测试 ====================