fix(bm25): 修复 BM25 索引覆盖 bug + 综合评测集 v2 + 检索增强

核心修复:
- knowledge/base.py: BM25Index.add_documents 从覆盖改为追加+去重，
  修复只有最后上传文件的 chunks 保留在 BM25 中的严重 bug
  (影响: 2.docx/3.docx/PDF 的 755 个 chunk 在 BM25 中完全缺失)

检索增强 (延续上次会话):
- core/engine.py: section cluster boost + lexical match exemption
- api/chat_routes.py: lexical/cluster rescue 层 + SSE 事件
- core/mmr.py: MMR 去重改进

评测体系:
- tests/eval_dataset_v2.json: 62 题综合评测集 (9 种题型×4 文档)
- scripts/eval_e2e.py: 推理模型 LLM 评分兼容 + 新数据集格式支持
- scripts/validate_eval_dataset.py: 数据集验证工具

其他:
- parsers/mineru_parser.py: 解析器改进

api/chat_routes.py

@@ -493,6 +493,198 @@ def _section_similarity(section_a: str, section_b: str) -> float:
     return overlap / union if union > 0 else 0.0
 
 
+def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
+                          min_score: float) -> List[Dict]:
+    """
+    词法匹配救援：当切片文本精确包含查询的核心关键词但 CrossEncoder 评分很低时，
+    将其分数提升至保底值，并同时救援同 source 下 chunk_index 相邻的切片。
+
+    适用场景：
+    1. 某个 section 只有一个正确切片（无法触发聚类救援）
+    2. 枚举类问题的 header 切片被词法匹配救援后，其后续子条目也应被一并保留
+
+    Args:
+        contexts: 全部上下文切片
+        retrieval_query: 检索查询
+        min_score: 最低分数阈值
+
+    Returns:
+        修改后的 contexts
+    """
+    if not contexts or not retrieval_query:
+        return contexts
+
+    import re
+    from config import CLUSTER_RESCUE_FLOOR
+
+    # 清理查询：去除 markdown 格式和标点
+    clean_query = re.sub(r'\*+|#+|`', '', retrieval_query)
+    clean_query = re.sub(r'[？?！!。，,、；;：:"""\'\s]+', ' ', clean_query).strip()
+
+    if len(clean_query) < 2:
+        return contexts
+
+    # 提取查询中的有意义 bigram（连续两字组）
+    query_bigrams = set()
+    for i in range(len(clean_query) - 1):
+        w = clean_query[i:i+2].strip()
+        if len(w) == 2:
+            query_bigrams.add(w)
+
+    if not query_bigrams:
+        return contexts
+
+    # Phase 1: 词法匹配救援——找到高分匹配的切片
+    rescued_seeds = []  # [(source, chunk_index)]
+    for ctx in contexts:
+        if ctx.get('score', 0) >= min_score:
+            continue
+
+        doc = ctx.get('doc', '') or ''
+        meta = ctx.get('meta', {})
+        section = meta.get('section', '') or meta.get('section_path', '')
+        combined = doc + ' ' + section
+
+        matched = sum(1 for w in query_bigrams if w in combined)
+        match_ratio = matched / len(query_bigrams)
+
+        if match_ratio > 0.35:
+            ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
+            source = meta.get('source', '')
+            chunk_index = meta.get('chunk_index')
+            if source and chunk_index is not None:
+                try:
+                    rescued_seeds.append((source, int(chunk_index)))
+                except (ValueError, TypeError):
+                    pass
+
+    # Phase 2: 邻居救援——对每个被词法匹配救援的种子，
+    # 同时救援同 source 下 chunk_index 后续相邻的切片（枚举子条目）
+    if rescued_seeds:
+        for source, seed_idx in rescued_seeds:
+            for ctx in contexts:
+                if ctx.get('score', 0) >= min_score:
+                    continue
+                meta = ctx.get('meta', {})
+                if meta.get('source') != source:
+                    continue
+                try:
+                    n_idx = int(meta.get('chunk_index', -1))
+                except (ValueError, TypeError):
+                    continue
+                # 救援种子后续 8 个相邻切片（覆盖大多数枚举/条款模式）
+                if seed_idx < n_idx <= seed_idx + 8:
+                    ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
+
+    return contexts
+
+
+def _normalize_section_for_rescue(section_path: str, levels: int = 2) -> str:
+    """归一化 section_path：取前 N 级路径用于分组"""
+    if not section_path:
+        return ''
+    parts = [p.strip() for p in section_path.split('>')]
+    return ' > '.join(parts[:levels])
+
+
+def _rescue_section_cluster(contexts: List[Dict], retrieval_query: str,
+                            min_score: float) -> List[Dict]:
+    """
+    路由层章节聚类救援：当某个 section 有多个候选切片但全部低于 min_score 时，
+    给该 section 的切片分配保底分数，使其通过后续的 min_score 过滤。
+
+    作为引擎层 _section_cluster_boost 的二次安全网，防止极端情况下所有正确切片被过滤。
+
+    Args:
+        contexts: engine 返回的全部上下文切片（每个含 doc, meta, score）
+        retrieval_query: 改写后的检索查询
+        min_score: 当前的最低分数阈值
+
+    Returns:
+        修改后的 contexts（部分切片 score 被提升至保底分数）
+    """
+    if not contexts:
+        return contexts
+
+    from config import (
+        CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, CLUSTER_RESCUE_FLOOR,
+        CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION,
+        CLUSTER_SECTION_PREFIX_LEVELS,
+    )
+
+    # 1. 按 (source, normalized_section) 分组
+    from collections import defaultdict
+    section_groups = defaultdict(list)  # key → [index_in_contexts]
+
+    for i, ctx in enumerate(contexts):
+        meta = ctx.get('meta', {})
+        source = meta.get('source', '')
+        section_path = meta.get('section', '') or meta.get('section_path', '')
+        norm_section = _normalize_section_for_rescue(section_path, CLUSTER_SECTION_PREFIX_LEVELS)
+        if not source or not norm_section:
+            continue
+        key = (source, norm_section)
+        section_groups[key].append(i)
+
+    # 2. 检测"全灭 section"并计算聚类强度
+    rescue_candidates = []  # (strength, key, member_indices)
+
+    for key, indices in section_groups.items():
+        if len(indices) < CLUSTER_MIN_MEMBERS:
+            continue
+
+        # 检查是否所有成员都低于 min_score（"全灭"）
+        scores = [contexts[i].get('score', 0) for i in indices]
+        if any(s >= min_score for s in scores):
+            continue  # 已有成员通过阈值，无需救援
+
+        # 计算聚类强度
+        chunk_types = set(contexts[i].get('meta', {}).get('chunk_type', 'text') for i in indices)
+        type_diversity = len(chunk_types)
+        if type_diversity < CLUSTER_MIN_TYPES:
+            continue  # 类型不够多样，可能是噪音
+
+        # 查询与 section_path 的字符重叠率
+        source, norm_section = key
+        query_chars = set(retrieval_query)
+        section_chars = set(norm_section)
+        overlap = len(query_chars & section_chars) / max(len(query_chars), 1)
+
+        # 聚类强度 = 成员数 × 类型多样性 × (1 + 查询匹配度)
+        strength = len(indices) * type_diversity * (1.0 + overlap)
+        rescue_candidates.append((strength, key, indices))
+
+    # 3. 按强度降序，救援 top-N section
+    rescue_candidates.sort(key=lambda x: x[0], reverse=True)
+
+    rescued_sections = []
+    total_rescued = 0
+
+    for strength, key, indices in rescue_candidates[:CLUSTER_MAX_SECTIONS]:
+        # 对组内切片分配保底分数（仅低于 min_score 的）
+        rescue_count = 0
+        for idx in indices:
+            if rescue_count >= CLUSTER_MAX_RESCUE_PER_SECTION:
+                break
+            ctx = contexts[idx]
+            if ctx.get('score', 0) < min_score:
+                ctx['score'] = CLUSTER_RESCUE_FLOOR
+                rescue_count += 1
+                total_rescued += 1
+
+        if rescue_count > 0:
+            source, norm_section = key
+            rescued_sections.append({
+                'source': source,
+                'section': norm_section,
+                'members': len(indices),
+                'rescued': rescue_count,
+                'strength': round(strength, 2)
+            })
+
+    return contexts
+
+
 def _rescue_table_chunks(contexts: List[Dict], context_text: str,
                        retrieval_query: str, max_rescue_chars: int = 3000) -> str:
     """
@@ -1731,7 +1923,10 @@ def rag():
         DIRECT_CONTEXT_MAX_CHARS,
         RERANK_CONTEXT_MIN_SCORE,
         CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT,
-        CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD
+        CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD,
+        SECTION_CLUSTER_RESCUE_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
+        CLUSTER_RESCUE_FLOOR, CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION,
+        CLUSTER_SECTION_PREFIX_LEVELS
     )
 
     data = request.json or {}
@@ -1973,6 +2168,11 @@ def rag():
                 debug_info = search_result.get('_debug', {})
                 yield f"data: {json.dumps({'type': 'retrieval_debug', 'data': {'steps': debug_info.get('steps', []), 'collections_searched': collections, 'total_candidates': len(search_result.get('ids', [[]])[0])}}, ensure_ascii=False)}\n\n"
 
+                # 提取章节聚类提升事件（引擎层）单独发送
+                for step in debug_info.get('steps', []):
+                    if step.get('name') == 'section_cluster_boost':
+                        yield f"data: {json.dumps({'type': 'section_cluster_boost', 'data': step}, ensure_ascii=False)}\n\n"
+
             # 提取上下文
             contexts = []
             sources = []
@@ -2241,6 +2441,32 @@ def rag():
 
             # 4. 构建 prompt（Phase 6：LLM 图片感知）
             # Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争
+
+            # 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时，
+            # 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景）
+            contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
+
+            # 章节聚类救援（路由层安全网）：在 min_score 过滤前，
+            # 检测"全灭 section"并分配保底分数
+            _rescue_debug = None
+            if SECTION_CLUSTER_RESCUE_ENABLED:
+                _before_scores = {id(ctx): ctx.get('score', 0) for ctx in contexts}
+                contexts = _rescue_section_cluster(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
+                # 统计救援结果
+                _rescued = [ctx for ctx in contexts if id(ctx) in _before_scores
+                           and ctx.get('score', 0) > _before_scores[id(ctx)]]
+                if _rescued and IS_DEV:
+                    _rescue_sections = set()
+                    for ctx in _rescued:
+                        meta = ctx.get('meta', {})
+                        sp = meta.get('section', '') or meta.get('section_path', '')
+                        _rescue_sections.add(_normalize_section_for_rescue(sp, CLUSTER_SECTION_PREFIX_LEVELS))
+                    _rescue_debug = {
+                        'rescued_count': len(_rescued),
+                        'rescued_sections': list(_rescue_sections)
+                    }
+                    yield f"data: {json.dumps({'type': 'section_cluster_rescue', 'data': _rescue_debug}, ensure_ascii=False)}\n\n"
+
             text_contexts = _order_text_contexts_for_prompt(contexts, retrieval_query, MAX_CONTEXT_CHUNKS,
                                                              min_score=RERANK_CONTEXT_MIN_SCORE)
             # Phase 2：按字符预算构建上下文