jhzhang/rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server-release): 从 main 迁移检索增强（章节聚类提升 + BM25 追加修复）

Browse Source 
从 main 分支选择性提取检索质量增强代码，不含端口/API 配置变更：

- knowledge/base.py: BM25Index.add_documents 改为追加+去重模式，
  修复多文件上传后仅保留最后文件切片的覆盖 bug
- core/engine.py: 章节聚类提升（_section_cluster_boost）、词法匹配
  辅助种子资格（_chunk_lexical_score）、扩展参数优化
  （CONTEXT_EXPANSION_AFTER 5→8, MAX_EXPANDED_NEIGHBORS 4→8）
- api/chat_routes.py: 路由层词法匹配救援（_rescue_lexical_match）、
  章节聚类救援安全网（_rescue_section_cluster）

部署后需重建 BM25 索引：KnowledgeBaseManager().rebuild_bm25_index('public_kb')
This commit is contained in:
lacerate551
2026-06-17 17:49:36 +08:00
parent fc11b11dda
commit 87f3fae1aa
3 changed files with 390 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
core/engine.py
View File

@@ -77,6 +77,10 @@ try:
CONTEXT_EXPANSION_MAX_CHUNKS, ENUM_QUERY_DISABLE_TOPK_SHRINK, ENUM_QUERY_MMR_LAMBDA,
# Phase 3 扩展精细化
EXPANSION_SCORE_THRESHOLD, MAX_EXPANDED_NEIGHBORS,
# 章节聚类救援
SECTION_CLUSTER_BOOST_ENABLED, CLUSTER_MIN_MEMBERS, CLUSTER_MIN_TYPES,
CLUSTER_SEED_FLOOR, CLUSTER_MAX_BOOST_PER_SECTION, CLUSTER_MAX_SECTIONS,
CLUSTER_SECTION_PREFIX_LEVELS,
# 上下文与生成
LLM_TEMPERATURE, LLM_MAX_TOKENS, RECALL_MULTIPLIER,
# FAQ 与黑名单
@@ -102,10 +106,18 @@ except ImportError:
MMR_TOP_K = 30
CONTEXT_EXPANSION_ENABLED = True
CONTEXT_EXPANSION_BEFORE = 1
CONTEXT_EXPANSION_AFTER = 5
CONTEXT_EXPANSION_AFTER = 8
CONTEXT_EXPANSION_MAX_CHUNKS = 24
EXPANSION_SCORE_THRESHOLD = 0.3
MAX_EXPANDED_NEIGHBORS = 4
MAX_EXPANDED_NEIGHBORS = 8
# 章节聚类救援默认值
SECTION_CLUSTER_BOOST_ENABLED = True
CLUSTER_MIN_MEMBERS = 3
CLUSTER_MIN_TYPES = 2
CLUSTER_SEED_FLOOR = 0.35
CLUSTER_MAX_BOOST_PER_SECTION = 8
CLUSTER_MAX_SECTIONS = 3
CLUSTER_SECTION_PREFIX_LEVELS = 1
ENUM_QUERY_DISABLE_TOPK_SHRINK = True
ENUM_QUERY_MMR_LAMBDA = 0.85
DYNAMIC_RRF_ENABLED = True
@@ -733,11 +745,19 @@ class RAGEngine:
# 时间衰减（Time Decay）
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})
@@ -1162,13 +1182,14 @@ class RAGEngine:
return self.collection
def _expand_contiguous_chunks(self, results: dict, top_k: int = None,
min_score: float = 0.0) -> dict:
min_score: float = 0.0, query: str = '') -> dict:
"""Add same-source same-section neighbor text chunks around strong hits.
Args:
results: 检索结果
top_k: 最大切片数
min_score: Phase 3 最低分数阈值，仅对 Rerank 分数高于此值的种子扩展
query: 查询文本，用于词法匹配辅助种子资格判定
"""
if not CONTEXT_EXPANSION_ENABLED:
return results
@@ -1198,7 +1219,7 @@ class RAGEngine:
seeds = [
(doc_id, doc, meta, dist)
for doc_id, doc, meta, dist in items[:base_limit]
if meta.get('chunk_type', 'text') == 'text'
if (meta.get('chunk_type', 'text') == 'text' or meta.get('_cluster_boosted'))
and meta.get('source')
and self._to_int(meta.get('chunk_index')) is not None
]
@@ -1210,7 +1231,11 @@ class RAGEngine:
# Phase 3：跳过分数低于阈值的种子（仅当 min_score > 0 时生效）
if min_score > 0 and seed_dist < min_score:
continue
# 词法匹配豁免：CrossEncoder 低分但关键词重叠度高时仍允许作为种子
if query and self._chunk_lexical_score(_seed_doc, query) > 0.3:
pass # 词法匹配度高，允许作为种子
else:
continue
source = seed_meta.get('source')
section = seed_meta.get('section', '') or seed_meta.get('section_path', '')
@@ -1316,6 +1341,130 @@ class RAGEngine:
expanded[key] = results[key]
return expanded
@staticmethod
def _normalize_section_path(section_path: str, levels: int = None) -> str:
"""归一化 section_path：取前 N 级路径，容忍 MinerU 标题检测误差。
例如: "第三章 吸烟场所的功能设置 > 第三条 文明吸烟..." → "第三章 吸烟场所的功能设置"
"""
if not section_path:
return ''
if levels is None:
levels = CLUSTER_SECTION_PREFIX_LEVELS
parts = [p.strip() for p in section_path.split('>')]
return ' > '.join(parts[:levels])
def _section_cluster_boost(self, results: dict, query: str = '') -> dict:
"""章节聚类提升：当同一 section 下多个切片（text+table）同时出现在候选集中，
即使单个切片 CrossEncoder 分数很低，也将整组提升到种子阈值。
核心洞察：单个低分切片不可信，但同一 section 多个切片同时出现是强信号。
提升后的切片可以作为 _expand_contiguous_chunks 的种子，触发邻居扩展。
Args:
results: rerank 后的检索结果
query: 用户查询（用于后续扩展）
Returns:
修改后的 results（distances 被调整，meta 中标记 _cluster_boosted）
"""
if not results.get('ids') or not results['ids'][0]:
return results
ids = results['ids'][0]
metas = results.get('metadatas', [[]])[0]
distances = results.get('distances', [[]])[0] if results.get('distances') else None
if not distances:
return results
# 1. 按 (source, normalized_section) 分组
from collections import defaultdict
section_groups = defaultdict(list) # key → [(index, meta, dist)]
for i, (meta, dist) in enumerate(zip(metas, distances)):
source = meta.get('source', '')
section_path = meta.get('section', '') or meta.get('section_path', '')
norm_section = self._normalize_section_path(section_path)
if not source or not norm_section:
continue
key = (source, norm_section)
section_groups[key].append((i, meta, dist))
# 2. 检测聚类信号并提升
boost_target_dist = 1.0 - CLUSTER_SEED_FLOOR # score=0.35 → dist=0.65
boosted_sections = []
total_boosted = 0
# 按组成员数降序排列，优先处理最大聚类
sorted_groups = sorted(section_groups.items(), key=lambda x: len(x[1]), reverse=True)
for (source, norm_section), members in sorted_groups:
if len(boosted_sections) >= CLUSTER_MAX_SECTIONS:
break
# 聚类信号检测：成员数 >= 阈值 且 类型多样性 >= 阈值
chunk_types = set(m[1].get('chunk_type', 'text') for m in members)
if len(members) < CLUSTER_MIN_MEMBERS or len(chunk_types) < CLUSTER_MIN_TYPES:
continue
# 提升组内切片分数（仅提升低于阈值的）
boost_count = 0
for idx, meta, dist in members:
if boost_count >= CLUSTER_MAX_BOOST_PER_SECTION:
break
# 只提升分数低于种子阈值的切片（高分切片不需要）
if dist > boost_target_dist:
distances[idx] = boost_target_dist
meta['_cluster_boosted'] = True
boost_count += 1
total_boosted += 1
if boost_count > 0:
boosted_sections.append({
'source': source,
'section': norm_section,
'members': len(members),
'types': list(chunk_types),
'boosted': boost_count
})
# 3. 写 debug 信息
if boosted_sections:
debug_info = results.get('_debug', {})
if 'steps' not in debug_info:
debug_info['steps'] = []
debug_info['steps'].append({
'name': 'section_cluster_boost',
'sections': boosted_sections,
'total_boosted': total_boosted
})
results['_debug'] = debug_info
logger.info(f"[章节聚类提升] 提升 {total_boosted} 个切片，"
f"涉及 {len(boosted_sections)} 个 section: "
f"{[s['section'][:30] for s in boosted_sections]}")
return results
@staticmethod
def _chunk_lexical_score(chunk_text: str, query: str) -> float:
"""计算切片文本与查询的词法重叠度（bigram 命中率），用于辅助种子资格判定。"""
if not chunk_text or not query:
return 0.0
import re
clean_q = re.sub(r'[？?！!。，,、；;：:"""\'\s*#`]+', ' ', query).strip()
if len(clean_q) < 2:
return 0.0
bigrams = set()
for i in range(len(clean_q) - 1):
w = clean_q[i:i+2].strip()
if len(w) == 2:
bigrams.add(w)
if not bigrams:
return 0.0
matched = sum(1 for w in bigrams if w in chunk_text)
return matched / len(bigrams)
def _search_with_sub_queries(
self, original_query, sub_queries, top_k=5, allowed_levels=None,
role=None, department=None, collections=None, source_filter=None
@@ -1627,11 +1776,19 @@ class RAGEngine:
# 时间衰减
fused_results = self._apply_time_decay(fused_results)
# 提前附加 _debug，使聚类提升能写入调试步骤
fused_results['_debug'] = _debug
# ========== 章节聚类提升：在扩展前将低分但聚类的切片提升至种子阈值 ==========
if SECTION_CLUSTER_BOOST_ENABLED:
fused_results = self._section_cluster_boost(fused_results, query)
# ========== 上下文扩展：补充强命中切片周围的连续文本（rerank 之后，防止被截断）==========
# Phase 3：仅对高分种子扩展邻居
before_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
fused_results = self._expand_contiguous_chunks(fused_results, top_k=top_k,
min_score=EXPANSION_SCORE_THRESHOLD)
min_score=EXPANSION_SCORE_THRESHOLD,
query=query)
after_exp = len(fused_results['ids'][0]) if fused_results.get('ids') else 0
if _debug is not None:
_debug['steps'].append({'name': 'context_expansion', 'before': before_exp, 'after': after_exp})