feat(server-release): 从 main 迁移检索增强（章节聚类提升 + BM25 追加修复）

从 main 分支选择性提取检索质量增强代码，不含端口/API 配置变更：

- knowledge/base.py: BM25Index.add_documents 改为追加+去重模式，
  修复多文件上传后仅保留最后文件切片的覆盖 bug
- core/engine.py: 章节聚类提升（_section_cluster_boost）、词法匹配
  辅助种子资格（_chunk_lexical_score）、扩展参数优化
  （CONTEXT_EXPANSION_AFTER 5→8, MAX_EXPANDED_NEIGHBORS 4→8）
- api/chat_routes.py: 路由层词法匹配救援（_rescue_lexical_match）、
  章节聚类救援安全网（_rescue_section_cluster）

部署后需重建 BM25 索引：KnowledgeBaseManager().rebuild_bm25_index('public_kb')

api/chat_routes.py

@@ -292,6 +292,198 @@ def _process_table_doc(doc: str, meta: Dict) -> str:
     return doc
 
 
+def _rescue_lexical_match(contexts: List[Dict], retrieval_query: str,
+                          min_score: float) -> List[Dict]:
+    """
+    词法匹配救援：当切片文本精确包含查询的核心关键词但 CrossEncoder 评分很低时，
+    将其分数提升至保底值，并同时救援同 source 下 chunk_index 相邻的切片。
+
+    适用场景：
+    1. 某个 section 只有一个正确切片（无法触发聚类救援）
+    2. 枚举类问题的 header 切片被词法匹配救援后，其后续子条目也应被一并保留
+
+    Args:
+        contexts: 全部上下文切片
+        retrieval_query: 检索查询
+        min_score: 最低分数阈值
+
+    Returns:
+        修改后的 contexts
+    """
+    if not contexts or not retrieval_query:
+        return contexts
+
+    import re
+    from config import CLUSTER_RESCUE_FLOOR
+
+    # 清理查询：去除 markdown 格式和标点
+    clean_query = re.sub(r'\*+|#+|`', '', retrieval_query)
+    clean_query = re.sub(r'[？?！!。，,、；;：:"""\'\s]+', ' ', clean_query).strip()
+
+    if len(clean_query) < 2:
+        return contexts
+
+    # 提取查询中的有意义 bigram（连续两字组）
+    query_bigrams = set()
+    for i in range(len(clean_query) - 1):
+        w = clean_query[i:i+2].strip()
+        if len(w) == 2:
+            query_bigrams.add(w)
+
+    if not query_bigrams:
+        return contexts
+
+    # Phase 1: 词法匹配救援——找到高分匹配的切片
+    rescued_seeds = []  # [(source, chunk_index)]
+    for ctx in contexts:
+        if ctx.get('score', 0) >= min_score:
+            continue
+
+        doc = ctx.get('doc', '') or ''
+        meta = ctx.get('meta', {})
+        section = meta.get('section', '') or meta.get('section_path', '')
+        combined = doc + ' ' + section
+
+        matched = sum(1 for w in query_bigrams if w in combined)
+        match_ratio = matched / len(query_bigrams)
+
+        if match_ratio > 0.35:
+            ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
+            source = meta.get('source', '')
+            chunk_index = meta.get('chunk_index')
+            if source and chunk_index is not None:
+                try:
+                    rescued_seeds.append((source, int(chunk_index)))
+                except (ValueError, TypeError):
+                    pass
+
+    # Phase 2: 邻居救援——对每个被词法匹配救援的种子，
+    # 同时救援同 source 下 chunk_index 后续相邻的切片（枚举子条目）
+    if rescued_seeds:
+        for source, seed_idx in rescued_seeds:
+            for ctx in contexts:
+                if ctx.get('score', 0) >= min_score:
+                    continue
+                meta = ctx.get('meta', {})
+                if meta.get('source') != source:
+                    continue
+                try:
+                    n_idx = int(meta.get('chunk_index', -1))
+                except (ValueError, TypeError):
+                    continue
+                # 救援种子后续 8 个相邻切片（覆盖大多数枚举/条款模式）
+                if seed_idx < n_idx <= seed_idx + 8:
+                    ctx['score'] = max(ctx.get('score', 0), CLUSTER_RESCUE_FLOOR)
+
+    return contexts
+
+
+def _normalize_section_for_rescue(section_path: str, levels: int = 2) -> str:
+    """归一化 section_path：取前 N 级路径用于分组"""
+    if not section_path:
+        return ''
+    parts = [p.strip() for p in section_path.split('>')]
+    return ' > '.join(parts[:levels])
+
+
+def _rescue_section_cluster(contexts: List[Dict], retrieval_query: str,
+                            min_score: float) -> List[Dict]:
+    """
+    路由层章节聚类救援：当某个 section 有多个候选切片但全部低于 min_score 时，
+    给该 section 的切片分配保底分数，使其通过后续的 min_score 过滤。
+
+    作为引擎层 _section_cluster_boost 的二次安全网，防止极端情况下所有正确切片被过滤。
+
+    Args:
+        contexts: engine 返回的全部上下文切片（每个含 doc, meta, score）
+        retrieval_query: 改写后的检索查询
+        min_score: 当前的最低分数阈值
+
+    Returns:
+        修改后的 contexts（部分切片 score 被提升至保底分数）
+    """
+    if not contexts:
+        return contexts
+
+    from config import (
+        CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES, CLUSTER_RESCUE_FLOOR,
+        CLUSTER_MAX_SECTIONS, CLUSTER_MAX_RESCUE_PER_SECTION,
+        CLUSTER_SECTION_PREFIX_LEVELS,
+    )
+
+    # 1. 按 (source, normalized_section) 分组
+    from collections import defaultdict
+    section_groups = defaultdict(list)  # key → [index_in_contexts]
+
+    for i, ctx in enumerate(contexts):
+        meta = ctx.get('meta', {})
+        source = meta.get('source', '')
+        section_path = meta.get('section', '') or meta.get('section_path', '')
+        norm_section = _normalize_section_for_rescue(section_path, CLUSTER_SECTION_PREFIX_LEVELS)
+        if not source or not norm_section:
+            continue
+        key = (source, norm_section)
+        section_groups[key].append(i)
+
+    # 2. 检测"全灭 section"并计算聚类强度
+    rescue_candidates = []  # (strength, key, member_indices)
+
+    for key, indices in section_groups.items():
+        if len(indices) < CLUSTER_MIN_MEMBERS:
+            continue
+
+        # 检查是否所有成员都低于 min_score（"全灭"）
+        scores = [contexts[i].get('score', 0) for i in indices]
+        if any(s >= min_score for s in scores):
+            continue  # 已有成员通过阈值，无需救援
+
+        # 计算聚类强度
+        chunk_types = set(contexts[i].get('meta', {}).get('chunk_type', 'text') for i in indices)
+        type_diversity = len(chunk_types)
+        if type_diversity < CLUSTER_MIN_TYPES:
+            continue  # 类型不够多样，可能是噪音
+
+        # 查询与 section_path 的字符重叠率
+        source, norm_section = key
+        query_chars = set(retrieval_query)
+        section_chars = set(norm_section)
+        overlap = len(query_chars & section_chars) / max(len(query_chars), 1)
+
+        # 聚类强度 = 成员数 × 类型多样性 × (1 + 查询匹配度)
+        strength = len(indices) * type_diversity * (1.0 + overlap)
+        rescue_candidates.append((strength, key, indices))
+
+    # 3. 按强度降序，救援 top-N section
+    rescue_candidates.sort(key=lambda x: x[0], reverse=True)
+
+    rescued_sections = []
+    total_rescued = 0
+
+    for strength, key, indices in rescue_candidates[:CLUSTER_MAX_SECTIONS]:
+        # 对组内切片分配保底分数（仅低于 min_score 的）
+        rescue_count = 0
+        for idx in indices:
+            if rescue_count >= CLUSTER_MAX_RESCUE_PER_SECTION:
+                break
+            ctx = contexts[idx]
+            if ctx.get('score', 0) < min_score:
+                ctx['score'] = CLUSTER_RESCUE_FLOOR
+                rescue_count += 1
+                total_rescued += 1
+
+        if rescue_count > 0:
+            source, norm_section = key
+            rescued_sections.append({
+                'source': source,
+                'section': norm_section,
+                'members': len(indices),
+                'rescued': rescue_count,
+                'strength': round(strength, 2)
+            })
+
+    return contexts
+
+
 def _order_text_contexts_for_prompt(contexts: List[Dict], query: str, max_chunks: int,
                                     min_score: float = 0.0) -> List[Dict]:
     """
@@ -1635,7 +1827,8 @@ def rag():
         DIRECT_CONTEXT_MAX_CHARS,
         RERANK_CONTEXT_MIN_SCORE,
         CONTEXT_MAX_CHARS, CONTEXT_SOFT_LIMIT,
-        CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD
+        CONFIDENCE_WARN_THRESHOLD, CONFIDENCE_CAUTION_THRESHOLD,
+        SECTION_CLUSTER_RESCUE_ENABLED
     )
 
     data = request.json or {}
@@ -2140,6 +2333,16 @@ def rag():
 
             # 4. 构建 prompt（Phase 6：LLM 图片感知）
             # Bug 1 修复：文本切片用于 top 5 名额竞争，图片描述不参与竞争
+
+            # 词法匹配救援：当切片文本精确包含查询关键词但 CrossEncoder 评分低时，
+            # 提升分数使其通过 min_score 过滤（适用于独立切片无法触发聚类救援的场景）
+            contexts = _rescue_lexical_match(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
+
+            # 章节聚类救援（路由层安全网）：在 min_score 过滤前，
+            # 检测"全灭 section"并分配保底分数
+            if SECTION_CLUSTER_RESCUE_ENABLED:
+                contexts = _rescue_section_cluster(contexts, retrieval_query, RERANK_CONTEXT_MIN_SCORE)
+
             text_contexts = _order_text_contexts_for_prompt(contexts, message, MAX_CONTEXT_CHUNKS,
                                                              min_score=RERANK_CONTEXT_MIN_SCORE)
             # Phase 2：按字符预算构建上下文